1   /**
2    * Copyright (c) 2000-2007 Liferay, Inc. All rights reserved.
3    *
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   *
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.lucene;
24  
25  import com.liferay.portal.SystemException;
26  import com.liferay.portal.kernel.search.Hits;
27  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
28  import com.liferay.portal.kernel.util.StringMaker;
29  import com.liferay.portal.kernel.util.StringPool;
30  import com.liferay.portal.kernel.util.Validator;
31  import com.liferay.portal.spring.hibernate.HibernateUtil;
32  import com.liferay.portal.util.PropsUtil;
33  import com.liferay.util.CollectionFactory;
34  import com.liferay.util.FileUtil;
35  import com.liferay.util.dao.DataAccess;
36  import com.liferay.util.lucene.HitsImpl;
37  import com.liferay.util.lucene.KeywordsUtil;
38  
39  import java.io.IOException;
40  
41  import java.sql.Connection;
42  import java.sql.DatabaseMetaData;
43  import java.sql.ResultSet;
44  import java.sql.Statement;
45  
46  import java.util.Map;
47  import java.util.regex.Pattern;
48  
49  import javax.sql.DataSource;
50  
51  import org.apache.commons.logging.Log;
52  import org.apache.commons.logging.LogFactory;
53  import org.apache.lucene.analysis.Analyzer;
54  import org.apache.lucene.analysis.WhitespaceAnalyzer;
55  import org.apache.lucene.index.IndexReader;
56  import org.apache.lucene.index.IndexWriter;
57  import org.apache.lucene.index.Term;
58  import org.apache.lucene.queryParser.ParseException;
59  import org.apache.lucene.queryParser.QueryParser;
60  import org.apache.lucene.search.BooleanClause;
61  import org.apache.lucene.search.BooleanQuery;
62  import org.apache.lucene.search.IndexSearcher;
63  import org.apache.lucene.search.Query;
64  import org.apache.lucene.search.Searcher;
65  import org.apache.lucene.search.TermQuery;
66  import org.apache.lucene.store.Directory;
67  import org.apache.lucene.store.FSDirectory;
68  import org.apache.lucene.store.RAMDirectory;
69  import org.apache.lucene.store.jdbc.JdbcDirectory;
70  import org.apache.lucene.store.jdbc.JdbcStoreException;
71  import org.apache.lucene.store.jdbc.dialect.Dialect;
72  import org.apache.lucene.store.jdbc.lock.JdbcLock;
73  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
74  
75  /**
76   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
77   *
78   * @author Brian Wing Shun Chan
79   * @author Harry Mark
80   *
81   */
82  public class LuceneUtil {
83  
84      public static final Pattern TERM_END_PATTERN =
85          Pattern.compile("(\\w{4,}?)\\b");
86  
87      public static void acquireLock(long companyId) {
88          try {
89              _instance._sharedWriter.acquireLock(companyId, true);
90          }
91          catch (InterruptedException ie) {
92              _log.error(ie);
93          }
94      }
95  
96      public static void addExactTerm(
97              BooleanQuery booleanQuery, String field, long number)
98          throws ParseException {
99  
100         addExactTerm(booleanQuery, field, String.valueOf(number));
101     }
102 
103     public static void addExactTerm(
104             BooleanQuery booleanQuery, String field, String text)
105         throws ParseException {
106 
107         //text = KeywordsUtil.escape(text);
108 
109         Query query = new TermQuery(new Term(field, text));
110 
111         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
112     }
113 
114     public static void addTerm(
115             BooleanQuery booleanQuery, String field, long number)
116         throws ParseException {
117 
118         addTerm(booleanQuery, field, String.valueOf(number));
119     }
120 
121     public static void addTerm(
122             BooleanQuery booleanQuery, String field, String text)
123         throws ParseException {
124 
125         if (Validator.isNotNull(text)) {
126             //Matcher matcher = TERM_END_PATTERN.matcher(text);
127 
128             // Add wildcard to the end of every word 4 chars or longer
129 
130             //text = matcher.replaceAll("$1*");
131 
132             // Add fuzzy query to the end of every word 4 chars or longer
133 
134             //text = text + " " + matcher.replaceAll("$1~");
135 
136             // Remove invalid hints
137 
138             //text = StringUtil.replace(text, "**", "*");
139             //text = StringUtil.replace(text, "*~", "*");
140             //text = StringUtil.replace(text, "~~", "~");
141             //text = StringUtil.replace(text, "~*", "~");
142 
143             QueryParser queryParser = new QueryParser(
144                 field, LuceneUtil.getAnalyzer());
145 
146             try {
147                 Query query = queryParser.parse(text);
148 
149                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
150             }
151             catch(ParseException pe) {
152                 if (_log.isDebugEnabled()) {
153                     _log.debug(
154                         "ParseException thrown, reverting to literal search",
155                         pe);
156                 }
157 
158                 text = KeywordsUtil.escape(text);
159 
160                 Query query = queryParser.parse(text);
161 
162                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
163             }
164         }
165     }
166 
167     public static void addRequiredTerm(
168         BooleanQuery booleanQuery, String field, long number) {
169 
170         addRequiredTerm(booleanQuery, field, String.valueOf(number));
171     }
172 
173     public static void addRequiredTerm(
174         BooleanQuery booleanQuery, String field, String text) {
175 
176         //text = KeywordsUtil.escape(text);
177 
178         Term term = new Term(field, text);
179         TermQuery termQuery = new TermQuery(term);
180 
181         booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
182     }
183 
184     public static void checkLuceneDir(long companyId) {
185         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
186 
187         IndexWriter writer = null;
188 
189         // Lucene does not properly release its lock on the index when
190         // IndexWriter throws an exception
191 
192         try {
193             if (luceneDir.fileExists("segments.gen")) {
194                 writer = new IndexWriter(
195                     luceneDir, LuceneUtil.getAnalyzer(), false);
196             }
197             else {
198                 writer = new IndexWriter(
199                     luceneDir, LuceneUtil.getAnalyzer(), true);
200             }
201         }
202         catch (IOException ioe) {
203             _log.error(ioe);
204         }
205         finally {
206             if (writer != null) {
207                 try {
208                     writer.close();
209                 }
210                 catch (IOException ioe) {
211                     _log.error(ioe);
212                 }
213             }
214         }
215     }
216 
217     public static void closeSearcher(Searcher searcher) {
218         try {
219             if (searcher != null){
220                 searcher.close();
221             }
222         }
223         catch (Exception e) {
224         }
225     }
226 
227     public static Hits closeSearcher(
228             Searcher searcher, String keywords, Exception e)
229         throws SystemException {
230 
231         closeSearcher(searcher);
232 
233         if (e instanceof BooleanQuery.TooManyClauses ||
234             e instanceof ParseException) {
235 
236             _log.error("Parsing keywords " + keywords, e);
237 
238             return new HitsImpl();
239         }
240         else {
241             throw new SystemException(e);
242         }
243     }
244 
245     public static void delete(long companyId) {
246         _instance._delete(companyId);
247     }
248 
249     public static void deleteDocuments(long companyId, Term term)
250         throws IOException {
251 
252         try {
253             _instance._sharedWriter.deleteDocuments(companyId, term);
254         }
255         catch (InterruptedException ie) {
256             _log.error(ie);
257         }
258     }
259 
260     public static Analyzer getAnalyzer() {
261         return _instance._getAnalyzer();
262     }
263 
264     public static Directory getLuceneDir(long companyId) {
265         return _instance._getLuceneDir(companyId);
266     }
267 
268     public static IndexReader getReader(long companyId) throws IOException {
269         return IndexReader.open(getLuceneDir(companyId));
270     }
271 
272     public static IndexSearcher getSearcher(long companyId)
273         throws IOException {
274 
275         return new IndexSearcher(getLuceneDir(companyId));
276     }
277 
278     public static IndexWriter getWriter(long companyId) throws IOException {
279         return getWriter(companyId, false);
280     }
281 
282     public static IndexWriter getWriter(long companyId, boolean create)
283         throws IOException {
284 
285         return _instance._sharedWriter.getWriter(companyId, create);
286     }
287 
288     public static void releaseLock(long companyId) {
289         _instance._sharedWriter.releaseLock(companyId);
290     }
291 
292     public static void write(long companyId) throws IOException {
293         _instance._sharedWriter.write(companyId);
294     }
295 
296     public static void write(IndexWriter writer) throws IOException {
297         _instance._sharedWriter.write(writer);
298     }
299 
300     private LuceneUtil() {
301         String analyzerName = PropsUtil.get(PropsUtil.LUCENE_ANALYZER);
302 
303         if (Validator.isNotNull(analyzerName)) {
304             try {
305                 _analyzerClass = Class.forName(analyzerName);
306             }
307             catch (Exception e) {
308                 _log.error(e);
309             }
310         }
311 
312         // Dialect
313 
314         if (PropsUtil.get(PropsUtil.LUCENE_STORE_TYPE).equals(
315                 _LUCENE_STORE_TYPE_JDBC)) {
316 
317             Connection con = null;
318 
319             try {
320                 con = HibernateUtil.getConnection();
321 
322                 String url = con.getMetaData().getURL();
323 
324                 int x = url.indexOf(":");
325                 int y = url.indexOf(":", x + 1);
326 
327                 String urlPrefix = url.substring(x + 1, y);
328 
329                 String dialectClass = PropsUtil.get(
330                     PropsUtil.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
331 
332                 if (dialectClass != null) {
333                     if (_log.isDebugEnabled()) {
334                         _log.debug("JDBC class implementation " + dialectClass);
335                     }
336                 }
337                 else {
338                     if (_log.isDebugEnabled()) {
339                         _log.debug("JDBC class implementation is null");
340                     }
341                 }
342 
343                 if (dialectClass != null) {
344                     _dialect =
345                         (Dialect)Class.forName(dialectClass).newInstance();
346                 }
347             }
348             catch (Exception e) {
349                 _log.error(e);
350             }
351             finally{
352                 DataAccess.cleanUp(con);
353             }
354 
355             if (_dialect == null) {
356                 _log.error("No JDBC dialect found");
357             }
358         }
359     }
360 
361     public void _delete(long companyId) {
362         String storeType = PropsUtil.get(PropsUtil.LUCENE_STORE_TYPE);
363 
364         if (_log.isDebugEnabled()) {
365             _log.debug("Lucene store type " + storeType);
366         }
367 
368         if (storeType.equals(_LUCENE_STORE_TYPE_FILE)) {
369             _deleteFile(companyId);
370         }
371         else if (storeType.equals(_LUCENE_STORE_TYPE_JDBC)) {
372             _deleteJdbc(companyId);
373         }
374         else if (storeType.equals(_LUCENE_STORE_TYPE_RAM)) {
375             _deleteRam(companyId);
376         }
377         else {
378             throw new RuntimeException("Invalid store type " + storeType);
379         }
380     }
381 
382     private void _deleteFile(long companyId) {
383         String path = _getPath(companyId);
384 
385         try {
386             Directory directory = FSDirectory.getDirectory(path, false);
387 
388             directory.close();
389         }
390         catch (Exception e) {
391             if (_log.isWarnEnabled()) {
392                 _log.warn("Could not close directory " + path);
393             }
394         }
395 
396         FileUtil.deltree(path);
397     }
398 
399     private void _deleteJdbc(long companyId) {
400         String tableName = _getTableName(companyId);
401 
402         try {
403             Directory directory = (Directory)_jdbcDirectories.remove(tableName);
404 
405             if (directory != null) {
406                 directory.close();
407             }
408         }
409         catch (Exception e) {
410             if (_log.isWarnEnabled()) {
411                 _log.warn("Could not close directory " + tableName);
412             }
413         }
414 
415         Connection con = null;
416         Statement s = null;
417 
418         try {
419             con = HibernateUtil.getConnection();
420 
421             s = con.createStatement();
422 
423             s.executeUpdate("DELETE FROM " + tableName);
424         }
425         catch (Exception e) {
426             if (_log.isWarnEnabled()) {
427                 _log.warn("Could not truncate " + tableName);
428             }
429         }
430         finally {
431             DataAccess.cleanUp(con, s);
432         }
433     }
434 
435     private void _deleteRam(long companyId) {
436     }
437 
438     private Analyzer _getAnalyzer() {
439         try {
440             return (Analyzer)_analyzerClass.newInstance();
441         }
442         catch (Exception e) {
443             throw new RuntimeException(e);
444         }
445     }
446 
447     private Directory _getLuceneDir(long companyId) {
448         String storeType = PropsUtil.get(PropsUtil.LUCENE_STORE_TYPE);
449 
450         if (_log.isDebugEnabled()) {
451             _log.debug("Lucene store type " + storeType);
452         }
453 
454         if (storeType.equals(_LUCENE_STORE_TYPE_FILE)) {
455             return _getLuceneDirFile(companyId);
456         }
457         else if (storeType.equals(_LUCENE_STORE_TYPE_JDBC)) {
458             return _getLuceneDirJdbc(companyId);
459         }
460         else if (storeType.equals(_LUCENE_STORE_TYPE_RAM)) {
461             return _getLuceneDirRam(companyId);
462         }
463         else {
464             throw new RuntimeException("Invalid store type " + storeType);
465         }
466     }
467 
468     private Directory _getLuceneDirFile(long companyId) {
469         Directory directory = null;
470 
471         String path = _getPath(companyId);
472 
473         try {
474             directory = FSDirectory.getDirectory(path, false);
475         }
476         catch (IOException ioe1) {
477             try {
478                 if (directory != null) {
479                     directory.close();
480                 }
481 
482                 directory = FSDirectory.getDirectory(path, true);
483             }
484             catch (IOException ioe2) {
485                 throw new RuntimeException(ioe2);
486             }
487         }
488 
489         return directory;
490     }
491 
492     private Directory _getLuceneDirJdbc(long companyId) {
493         JdbcDirectory directory = null;
494 
495         ClassLoader contextClassLoader =
496             Thread.currentThread().getContextClassLoader();
497 
498         try {
499             Thread.currentThread().setContextClassLoader(
500                 PortalClassLoaderUtil.getClassLoader());
501 
502             String tableName = _getTableName(companyId);
503 
504             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
505 
506             if (directory != null) {
507                 return directory;
508             }
509 
510             try {
511                 DataSource ds = HibernateUtil.getDataSource();
512 
513                 directory = new JdbcDirectory(ds, _dialect, tableName);
514 
515                 _jdbcDirectories.put(tableName, directory);
516 
517                 if (!directory.tableExists()) {
518                     directory.create();
519                 }
520             }
521             catch (IOException ioe) {
522                 throw new RuntimeException(ioe);
523             }
524             catch (UnsupportedOperationException uoe) {
525                 if (_log.isWarnEnabled()) {
526                     _log.warn(
527                         "Database doesn't support the ability to check " +
528                             "whether a table exists");
529                 }
530 
531                 _manuallyCreateJdbcDirectory(directory, tableName);
532             }
533         }
534         finally {
535             Thread.currentThread().setContextClassLoader(contextClassLoader);
536         }
537 
538         return directory;
539     }
540 
541     private Directory _getLuceneDirRam(long companyId) {
542         String path = _getPath(companyId);
543 
544         Directory directory = (Directory)_ramDirectories.get(path);
545 
546         if (directory == null) {
547             directory = new RAMDirectory();
548 
549             _ramDirectories.put(path, directory);
550         }
551 
552         return directory;
553     }
554 
555     private String _getPath(long companyId) {
556         StringMaker sm = new StringMaker();
557 
558         sm.append(PropsUtil.get(PropsUtil.LUCENE_DIR));
559         sm.append(companyId);
560         sm.append(StringPool.SLASH);
561 
562         return sm.toString();
563     }
564 
565     private String _getTableName(long companyId) {
566         return _LUCENE_TABLE_PREFIX + companyId;
567     }
568 
569     private void _manuallyCreateJdbcDirectory(
570         JdbcDirectory directory, String tableName) {
571 
572         // LEP-2181
573 
574         Connection con = null;
575         ResultSet rs = null;
576 
577         try {
578             con = HibernateUtil.getConnection();
579 
580             // Check if table exists
581 
582             DatabaseMetaData metaData = con.getMetaData();
583 
584             rs = metaData.getTables(null, null, tableName, null);
585 
586             if (!rs.next()) {
587                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
588 
589                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
590 
591                 Class lockClass = directory.getSettings().getLockClass();
592 
593                 JdbcLock jdbcLock = null;
594 
595                 try {
596                     jdbcLock = (JdbcLock)lockClass.newInstance();
597                 }
598                 catch (Exception e) {
599                     throw new JdbcStoreException(
600                         "Failed to create lock class " + lockClass);
601                 }
602 
603                 jdbcLock.initializeDatabase(directory);
604             }
605         }
606         catch (Exception e) {
607             if (_log.isWarnEnabled()) {
608                 _log.warn("Could not create " + tableName);
609             }
610         }
611         finally {
612             DataAccess.cleanUp(con, null, rs);
613         }
614     }
615 
616     private static final String _LUCENE_STORE_TYPE_FILE = "file";
617 
618     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
619 
620     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
621 
622     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
623 
624     private static Log _log = LogFactory.getLog(LuceneUtil.class);
625 
626     private static LuceneUtil _instance = new LuceneUtil();
627 
628     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
629     private Class _analyzerClass = WhitespaceAnalyzer.class;
630     private Dialect _dialect;
631     private Map _jdbcDirectories = CollectionFactory.getSyncHashMap();
632     private Map _ramDirectories = CollectionFactory.getSyncHashMap();
633 
634 }