1   /**
2    * Copyright (c) 2000-2009 Liferay, Inc. All rights reserved.
3    *
4    * The contents of this file are subject to the terms of the Liferay Enterprise
5    * Subscription License ("License"). You may not use this file except in
6    * compliance with the License. You can obtain a copy of the License by
7    * contacting Liferay, Inc. See the License for the specific language governing
8    * permissions and limitations under the License, including but not limited to
9    * distribution rights of the Software.
10   *
11   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
12   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
13   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
14   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
15   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
16   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
17   * SOFTWARE.
18   */
19  
20  package com.liferay.portal.search.lucene;
21  
22  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
23  import com.liferay.portal.kernel.log.Log;
24  import com.liferay.portal.kernel.log.LogFactoryUtil;
25  import com.liferay.portal.kernel.search.SearchEngineUtil;
26  import com.liferay.portal.kernel.util.FileUtil;
27  import com.liferay.portal.kernel.util.InfrastructureUtil;
28  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
29  import com.liferay.portal.kernel.util.StringPool;
30  import com.liferay.portal.kernel.util.StringUtil;
31  import com.liferay.portal.kernel.util.Validator;
32  import com.liferay.portal.util.PropsKeys;
33  import com.liferay.portal.util.PropsUtil;
34  import com.liferay.portal.util.PropsValues;
35  import com.liferay.util.lucene.KeywordsUtil;
36  
37  import java.io.IOException;
38  
39  import java.sql.Connection;
40  import java.sql.DatabaseMetaData;
41  import java.sql.ResultSet;
42  import java.sql.Statement;
43  
44  import java.util.Date;
45  import java.util.Map;
46  import java.util.concurrent.ConcurrentHashMap;
47  
48  import javax.sql.DataSource;
49  
50  import org.apache.lucene.analysis.Analyzer;
51  import org.apache.lucene.analysis.WhitespaceAnalyzer;
52  import org.apache.lucene.document.Document;
53  import org.apache.lucene.index.IndexReader;
54  import org.apache.lucene.index.IndexWriter;
55  import org.apache.lucene.index.Term;
56  import org.apache.lucene.queryParser.ParseException;
57  import org.apache.lucene.queryParser.QueryParser;
58  import org.apache.lucene.search.BooleanClause;
59  import org.apache.lucene.search.BooleanQuery;
60  import org.apache.lucene.search.IndexSearcher;
61  import org.apache.lucene.search.Query;
62  import org.apache.lucene.search.TermQuery;
63  import org.apache.lucene.search.WildcardQuery;
64  import org.apache.lucene.store.Directory;
65  import org.apache.lucene.store.FSDirectory;
66  import org.apache.lucene.store.RAMDirectory;
67  import org.apache.lucene.store.jdbc.JdbcDirectory;
68  import org.apache.lucene.store.jdbc.JdbcStoreException;
69  import org.apache.lucene.store.jdbc.dialect.Dialect;
70  import org.apache.lucene.store.jdbc.lock.JdbcLock;
71  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
72  
73  /**
74   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
75   *
76   * @author Brian Wing Shun Chan
77   * @author Harry Mark
78   *
79   */
80  public class LuceneUtil {
81  
82      public static void acquireLock(long companyId) {
83          try {
84              _instance._sharedWriter.acquireLock(companyId, true);
85          }
86          catch (InterruptedException ie) {
87              _log.error(ie);
88          }
89      }
90  
91      public static void addDate(Document doc, String field, Date value) {
92          doc.add(LuceneFields.getDate(field, value));
93      }
94  
95      public static void addExactTerm(
96          BooleanQuery booleanQuery, String field, boolean value) {
97  
98          addExactTerm(booleanQuery, field, String.valueOf(value));
99      }
100 
101     public static void addExactTerm(
102         BooleanQuery booleanQuery, String field, Boolean value) {
103 
104         addExactTerm(booleanQuery, field, String.valueOf(value));
105     }
106 
107     public static void addExactTerm(
108         BooleanQuery booleanQuery, String field, double value) {
109 
110         addExactTerm(booleanQuery, field, String.valueOf(value));
111     }
112 
113     public static void addExactTerm(
114         BooleanQuery booleanQuery, String field, Double value) {
115 
116         addExactTerm(booleanQuery, field, String.valueOf(value));
117     }
118 
119     public static void addExactTerm(
120         BooleanQuery booleanQuery, String field, int value) {
121 
122         addExactTerm(booleanQuery, field, String.valueOf(value));
123     }
124 
125     public static void addExactTerm(
126         BooleanQuery booleanQuery, String field, Integer value) {
127 
128         addExactTerm(booleanQuery, field, String.valueOf(value));
129     }
130 
131     public static void addExactTerm(
132         BooleanQuery booleanQuery, String field, long value) {
133 
134         addExactTerm(booleanQuery, field, String.valueOf(value));
135     }
136 
137     public static void addExactTerm(
138         BooleanQuery booleanQuery, String field, Long value) {
139 
140         addExactTerm(booleanQuery, field, String.valueOf(value));
141     }
142 
143     public static void addExactTerm(
144         BooleanQuery booleanQuery, String field, short value) {
145 
146         addExactTerm(booleanQuery, field, String.valueOf(value));
147     }
148 
149     public static void addExactTerm(
150         BooleanQuery booleanQuery, String field, Short value) {
151 
152         addExactTerm(booleanQuery, field, String.valueOf(value));
153     }
154 
155     public static void addExactTerm(
156         BooleanQuery booleanQuery, String field, String value) {
157 
158         //text = KeywordsUtil.escape(value);
159 
160         Query query = new TermQuery(new Term(field, value));
161 
162         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
163     }
164 
165     public static void addRequiredTerm(
166         BooleanQuery booleanQuery, String field, boolean value) {
167 
168         addRequiredTerm(booleanQuery, field, String.valueOf(value));
169     }
170 
171     public static void addRequiredTerm(
172         BooleanQuery booleanQuery, String field, Boolean value) {
173 
174         addRequiredTerm(booleanQuery, field, String.valueOf(value));
175     }
176 
177     public static void addRequiredTerm(
178         BooleanQuery booleanQuery, String field, double value) {
179 
180         addRequiredTerm(booleanQuery, field, String.valueOf(value));
181     }
182 
183     public static void addRequiredTerm(
184         BooleanQuery booleanQuery, String field, Double value) {
185 
186         addRequiredTerm(booleanQuery, field, String.valueOf(value));
187     }
188 
189     public static void addRequiredTerm(
190         BooleanQuery booleanQuery, String field, int value) {
191 
192         addRequiredTerm(booleanQuery, field, String.valueOf(value));
193     }
194 
195     public static void addRequiredTerm(
196         BooleanQuery booleanQuery, String field, Integer value) {
197 
198         addRequiredTerm(booleanQuery, field, String.valueOf(value));
199     }
200 
201     public static void addRequiredTerm(
202         BooleanQuery booleanQuery, String field, long value) {
203 
204         addRequiredTerm(booleanQuery, field, String.valueOf(value));
205     }
206 
207     public static void addRequiredTerm(
208         BooleanQuery booleanQuery, String field, Long value) {
209 
210         addRequiredTerm(booleanQuery, field, String.valueOf(value));
211     }
212 
213     public static void addRequiredTerm(
214         BooleanQuery booleanQuery, String field, short value) {
215 
216         addRequiredTerm(booleanQuery, field, String.valueOf(value));
217     }
218 
219     public static void addRequiredTerm(
220         BooleanQuery booleanQuery, String field, Short value) {
221 
222         addRequiredTerm(booleanQuery, field, String.valueOf(value));
223     }
224 
225     public static void addRequiredTerm(
226         BooleanQuery booleanQuery, String field, String value) {
227 
228         addRequiredTerm(booleanQuery, field, value, false);
229     }
230 
231     public static void addRequiredTerm(
232         BooleanQuery booleanQuery, String field, String value, boolean like) {
233 
234         if (like) {
235             value = StringUtil.replace(
236                 value, StringPool.PERCENT, StringPool.STAR);
237 
238             value = value.toLowerCase();
239 
240             WildcardQuery wildcardQuery = new WildcardQuery(
241                 new Term(field, value));
242 
243             booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
244         }
245         else {
246             //text = KeywordsUtil.escape(value);
247 
248             Term term = new Term(field, value);
249             TermQuery termQuery = new TermQuery(term);
250 
251             booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
252         }
253     }
254 
255     public static void addTerm(
256             BooleanQuery booleanQuery, String field, long value)
257         throws ParseException {
258 
259         addTerm(booleanQuery, field, String.valueOf(value));
260     }
261 
262     public static void addTerm(
263             BooleanQuery booleanQuery, String field, String value)
264         throws ParseException {
265 
266         addTerm(booleanQuery, field, value, false);
267     }
268 
269     public static void addTerm(
270             BooleanQuery booleanQuery, String field, String value,
271             boolean like)
272         throws ParseException {
273 
274         if (Validator.isNull(value)) {
275             return;
276         }
277 
278         if (like) {
279             value = value.toLowerCase();
280 
281             StringBuilder sb = new StringBuilder();
282 
283             sb.append(value);
284             sb.append(StringPool.STAR);
285 
286             WildcardQuery wildcardQuery = new WildcardQuery(
287                 new Term(field, sb.toString()));
288 
289             booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
290         }
291         else {
292             QueryParser queryParser = new QueryParser(
293                 field, LuceneUtil.getAnalyzer());
294 
295             try {
296                 Query query = queryParser.parse(value);
297 
298                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
299             }
300             catch (ParseException pe) {
301                 if (_log.isDebugEnabled()) {
302                     _log.debug(
303                         "ParseException thrown, reverting to literal search",
304                         pe);
305                 }
306 
307                 value = KeywordsUtil.escape(value);
308 
309                 Query query = queryParser.parse(value);
310 
311                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
312             }
313         }
314     }
315 
316     public static void checkLuceneDir(long companyId) {
317         if (SearchEngineUtil.isIndexReadOnly()) {
318             return;
319         }
320 
321         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
322 
323         try {
324 
325             // LEP-6078
326 
327             if (luceneDir.fileExists("write.lock")) {
328                 luceneDir.deleteFile("write.lock");
329             }
330         }
331         catch (IOException ioe) {
332             _log.error("Unable to clear write lock", ioe);
333         }
334 
335         IndexWriter writer = null;
336 
337         // Lucene does not properly release its lock on the index when
338         // IndexWriter throws an exception
339 
340         try {
341             if (luceneDir.fileExists("segments.gen")) {
342                 writer = new IndexWriter(
343                     luceneDir, LuceneUtil.getAnalyzer(), false);
344             }
345             else {
346                 writer = new IndexWriter(
347                     luceneDir, LuceneUtil.getAnalyzer(), true);
348             }
349         }
350         catch (IOException ioe) {
351             _log.error("Check Lucene directory failed for " + companyId, ioe);
352         }
353         finally {
354             if (writer != null) {
355                 try {
356                     writer.close();
357                 }
358                 catch (IOException ioe) {
359                     _log.error(ioe);
360                 }
361             }
362         }
363     }
364 
365     public static void delete(long companyId) {
366         _instance._delete(companyId);
367     }
368 
369     public static void deleteDocuments(long companyId, Term term)
370         throws IOException {
371 
372         try {
373             _instance._sharedWriter.deleteDocuments(companyId, term);
374         }
375         catch (InterruptedException ie) {
376             _log.error(ie);
377         }
378     }
379 
380     public static Analyzer getAnalyzer() {
381         return _instance._getAnalyzer();
382     }
383 
384     public static FSDirectory getDirectory(String path, boolean create)
385         throws IOException {
386 
387         return FSDirectory.getDirectory(path, false);
388     }
389 
390     public static Directory getLuceneDir(long companyId) {
391         return _instance._getLuceneDir(companyId);
392     }
393 
394     public static IndexReader getReader(long companyId) throws IOException {
395         return IndexReader.open(getLuceneDir(companyId));
396     }
397 
398     public static IndexSearcher getSearcher(long companyId)
399         throws IOException {
400 
401         return new IndexSearcher(getLuceneDir(companyId));
402     }
403 
404     public static IndexWriter getWriter(long companyId) throws IOException {
405         return getWriter(companyId, false);
406     }
407 
408     public static IndexWriter getWriter(long companyId, boolean create)
409         throws IOException {
410 
411         return _instance._sharedWriter.getWriter(companyId, create);
412     }
413 
414     public static void releaseLock(long companyId) {
415         _instance._sharedWriter.releaseLock(companyId);
416     }
417 
418     public static void write(long companyId) {
419         _instance._sharedWriter.write(companyId);
420     }
421 
422     public static void write(IndexWriter writer) throws IOException {
423         _instance._sharedWriter.write(writer);
424     }
425 
426     private LuceneUtil() {
427         String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
428 
429         if (Validator.isNotNull(analyzerName)) {
430             try {
431                 _analyzerClass = Class.forName(analyzerName);
432             }
433             catch (Exception e) {
434                 _log.error(e);
435             }
436         }
437 
438         // Dialect
439 
440         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
441             Connection con = null;
442 
443             try {
444                 con = DataAccess.getConnection();
445 
446                 String url = con.getMetaData().getURL();
447 
448                 int x = url.indexOf(":");
449                 int y = url.indexOf(":", x + 1);
450 
451                 String urlPrefix = url.substring(x + 1, y);
452 
453                 String dialectClass = PropsUtil.get(
454                     PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
455 
456                 if (dialectClass != null) {
457                     if (_log.isDebugEnabled()) {
458                         _log.debug("JDBC class implementation " + dialectClass);
459                     }
460                 }
461                 else {
462                     if (_log.isDebugEnabled()) {
463                         _log.debug("JDBC class implementation is null");
464                     }
465                 }
466 
467                 if (dialectClass != null) {
468                     _dialect =
469                         (Dialect)Class.forName(dialectClass).newInstance();
470                 }
471             }
472             catch (Exception e) {
473                 _log.error(e);
474             }
475             finally{
476                 DataAccess.cleanUp(con);
477             }
478 
479             if (_dialect == null) {
480                 _log.error("No JDBC dialect found");
481             }
482         }
483     }
484 
485     public void _delete(long companyId) {
486         if (SearchEngineUtil.isIndexReadOnly()) {
487             return;
488         }
489 
490         if (_log.isDebugEnabled()) {
491             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
492         }
493 
494         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
495             _deleteFile(companyId);
496         }
497         else if (PropsValues.LUCENE_STORE_TYPE.equals(
498                     _LUCENE_STORE_TYPE_JDBC)) {
499 
500             _deleteJdbc(companyId);
501         }
502         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
503             _deleteRam(companyId);
504         }
505         else {
506             throw new RuntimeException(
507                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
508         }
509     }
510 
511     private void _deleteFile(long companyId) {
512         String path = _getPath(companyId);
513 
514         try {
515             Directory directory = getDirectory(path, false);
516 
517             directory.close();
518         }
519         catch (Exception e) {
520             if (_log.isWarnEnabled()) {
521                 _log.warn("Could not close directory " + path);
522             }
523         }
524 
525         FileUtil.deltree(path);
526     }
527 
528     private void _deleteJdbc(long companyId) {
529         String tableName = _getTableName(companyId);
530 
531         try {
532             Directory directory = _jdbcDirectories.remove(tableName);
533 
534             if (directory != null) {
535                 directory.close();
536             }
537         }
538         catch (Exception e) {
539             if (_log.isWarnEnabled()) {
540                 _log.warn("Could not close directory " + tableName);
541             }
542         }
543 
544         Connection con = null;
545         Statement s = null;
546 
547         try {
548             con = DataAccess.getConnection();
549 
550             s = con.createStatement();
551 
552             s.executeUpdate("DELETE FROM " + tableName);
553         }
554         catch (Exception e) {
555             if (_log.isWarnEnabled()) {
556                 _log.warn("Could not truncate " + tableName);
557             }
558         }
559         finally {
560             DataAccess.cleanUp(con, s);
561         }
562     }
563 
564     private void _deleteRam(long companyId) {
565     }
566 
567     private Analyzer _getAnalyzer() {
568         try {
569             return (Analyzer)_analyzerClass.newInstance();
570         }
571         catch (Exception e) {
572             throw new RuntimeException(e);
573         }
574     }
575 
576     private Directory _getLuceneDir(long companyId) {
577         if (_log.isDebugEnabled()) {
578             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
579         }
580 
581         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
582             return _getLuceneDirFile(companyId);
583         }
584         else if (PropsValues.LUCENE_STORE_TYPE.equals(
585                     _LUCENE_STORE_TYPE_JDBC)) {
586 
587             return _getLuceneDirJdbc(companyId);
588         }
589         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
590             return _getLuceneDirRam(companyId);
591         }
592         else {
593             throw new RuntimeException(
594                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
595         }
596     }
597 
598     private Directory _getLuceneDirFile(long companyId) {
599         Directory directory = null;
600 
601         String path = _getPath(companyId);
602 
603         try {
604             directory = getDirectory(path, false);
605         }
606         catch (IOException ioe1) {
607             try {
608                 if (directory != null) {
609                     directory.close();
610                 }
611 
612                 directory = getDirectory(path, true);
613             }
614             catch (IOException ioe2) {
615                 throw new RuntimeException(ioe2);
616             }
617         }
618 
619         return directory;
620     }
621 
622     private Directory _getLuceneDirJdbc(long companyId) {
623         JdbcDirectory directory = null;
624 
625         Thread currentThread = Thread.currentThread();
626 
627         ClassLoader contextClassLoader = currentThread.getContextClassLoader();
628 
629         try {
630             currentThread.setContextClassLoader(
631                 PortalClassLoaderUtil.getClassLoader());
632 
633             String tableName = _getTableName(companyId);
634 
635             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
636 
637             if (directory != null) {
638                 return directory;
639             }
640 
641             try {
642                 DataSource ds = InfrastructureUtil.getDataSource();
643 
644                 directory = new JdbcDirectory(ds, _dialect, tableName);
645 
646                 _jdbcDirectories.put(tableName, directory);
647 
648                 if (!directory.tableExists()) {
649                     directory.create();
650                 }
651             }
652             catch (IOException ioe) {
653                 throw new RuntimeException(ioe);
654             }
655             catch (UnsupportedOperationException uoe) {
656                 if (_log.isWarnEnabled()) {
657                     _log.warn(
658                         "Database doesn't support the ability to check " +
659                             "whether a table exists");
660                 }
661 
662                 _manuallyCreateJdbcDirectory(directory, tableName);
663             }
664         }
665         finally {
666             currentThread.setContextClassLoader(contextClassLoader);
667         }
668 
669         return directory;
670     }
671 
672     private Directory _getLuceneDirRam(long companyId) {
673         String path = _getPath(companyId);
674 
675         Directory directory = _ramDirectories.get(path);
676 
677         if (directory == null) {
678             directory = new RAMDirectory();
679 
680             _ramDirectories.put(path, directory);
681         }
682 
683         return directory;
684     }
685 
686     private String _getPath(long companyId) {
687         StringBuilder sb = new StringBuilder();
688 
689         sb.append(PropsValues.LUCENE_DIR);
690         sb.append(companyId);
691         sb.append(StringPool.SLASH);
692 
693         return sb.toString();
694     }
695 
696     private String _getTableName(long companyId) {
697         return _LUCENE_TABLE_PREFIX + companyId;
698     }
699 
700     private void _manuallyCreateJdbcDirectory(
701         JdbcDirectory directory, String tableName) {
702 
703         // LEP-2181
704 
705         Connection con = null;
706         ResultSet rs = null;
707 
708         try {
709             con = DataAccess.getConnection();
710 
711             // Check if table exists
712 
713             DatabaseMetaData metaData = con.getMetaData();
714 
715             rs = metaData.getTables(null, null, tableName, null);
716 
717             if (!rs.next()) {
718                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
719 
720                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
721 
722                 Class<?> lockClass = directory.getSettings().getLockClass();
723 
724                 JdbcLock jdbcLock = null;
725 
726                 try {
727                     jdbcLock = (JdbcLock)lockClass.newInstance();
728                 }
729                 catch (Exception e) {
730                     throw new JdbcStoreException(
731                         "Failed to create lock class " + lockClass);
732                 }
733 
734                 jdbcLock.initializeDatabase(directory);
735             }
736         }
737         catch (Exception e) {
738             if (_log.isWarnEnabled()) {
739                 _log.warn("Could not create " + tableName);
740             }
741         }
742         finally {
743             DataAccess.cleanUp(con, null, rs);
744         }
745     }
746 
747     private static final String _LUCENE_STORE_TYPE_FILE = "file";
748 
749     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
750 
751     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
752 
753     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
754 
755     private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
756 
757     private static LuceneUtil _instance = new LuceneUtil();
758 
759     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
760     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
761     private Dialect _dialect;
762     private Map<String, Directory> _jdbcDirectories =
763         new ConcurrentHashMap<String, Directory>();
764     private Map<String, Directory> _ramDirectories =
765         new ConcurrentHashMap<String, Directory>();
766 
767 }