1   /**
2    * Copyright (c) 2000-2008 Liferay, Inc. All rights reserved.
3    *
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   *
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.lucene;
24  
25  import com.liferay.portal.SystemException;
26  import com.liferay.portal.kernel.search.Hits;
27  import com.liferay.portal.kernel.util.GetterUtil;
28  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
29  import com.liferay.portal.kernel.util.StringMaker;
30  import com.liferay.portal.kernel.util.StringPool;
31  import com.liferay.portal.kernel.util.Validator;
32  import com.liferay.portal.spring.hibernate.HibernateUtil;
33  import com.liferay.portal.util.PropsUtil;
34  import com.liferay.portal.util.PropsValues;
35  import com.liferay.util.FileUtil;
36  import com.liferay.util.dao.DataAccess;
37  import com.liferay.util.lucene.HitsImpl;
38  import com.liferay.util.lucene.KeywordsUtil;
39  
40  import java.io.IOException;
41  
42  import java.sql.Connection;
43  import java.sql.DatabaseMetaData;
44  import java.sql.ResultSet;
45  import java.sql.Statement;
46  
47  import java.util.Date;
48  import java.util.Map;
49  import java.util.concurrent.ConcurrentHashMap;
50  import java.util.regex.Pattern;
51  
52  import javax.sql.DataSource;
53  
54  import org.apache.commons.logging.Log;
55  import org.apache.commons.logging.LogFactory;
56  import org.apache.lucene.analysis.Analyzer;
57  import org.apache.lucene.analysis.WhitespaceAnalyzer;
58  import org.apache.lucene.document.Document;
59  import org.apache.lucene.index.IndexReader;
60  import org.apache.lucene.index.IndexWriter;
61  import org.apache.lucene.index.Term;
62  import org.apache.lucene.queryParser.ParseException;
63  import org.apache.lucene.queryParser.QueryParser;
64  import org.apache.lucene.search.BooleanClause;
65  import org.apache.lucene.search.BooleanQuery;
66  import org.apache.lucene.search.IndexSearcher;
67  import org.apache.lucene.search.Query;
68  import org.apache.lucene.search.Searcher;
69  import org.apache.lucene.search.TermQuery;
70  import org.apache.lucene.store.Directory;
71  import org.apache.lucene.store.FSDirectory;
72  import org.apache.lucene.store.RAMDirectory;
73  import org.apache.lucene.store.jdbc.JdbcDirectory;
74  import org.apache.lucene.store.jdbc.JdbcStoreException;
75  import org.apache.lucene.store.jdbc.dialect.Dialect;
76  import org.apache.lucene.store.jdbc.lock.JdbcLock;
77  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
78  
79  /**
80   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
81   *
82   * @author Brian Wing Shun Chan
83   * @author Harry Mark
84   *
85   */
86  public class LuceneUtil {
87  
88      public static boolean INDEX_READ_ONLY = GetterUtil.getBoolean(
89          PropsUtil.get(PropsUtil.INDEX_READ_ONLY));
90  
91      public static final Pattern TERM_END_PATTERN =
92          Pattern.compile("(\\w{4,}?)\\b");
93  
94      public static void acquireLock(long companyId) {
95          try {
96              _instance._sharedWriter.acquireLock(companyId, true);
97          }
98          catch (InterruptedException ie) {
99              _log.error(ie);
100         }
101     }
102 
103     public static void addDate(Document doc, String field, Date value) {
104         doc.add(LuceneFields.getDate(field, value));
105     }
106 
107     public static void addExactTerm(
108             BooleanQuery booleanQuery, String field, long value)
109         throws ParseException {
110 
111         addExactTerm(booleanQuery, field, String.valueOf(value));
112     }
113 
114     public static void addExactTerm(
115             BooleanQuery booleanQuery, String field, String text)
116         throws ParseException {
117 
118         //text = KeywordsUtil.escape(text);
119 
120         Query query = new TermQuery(new Term(field, text));
121 
122         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
123     }
124 
125     public static void addKeyword(Document doc, String field, double value) {
126         addKeyword(doc, field, String.valueOf(value));
127     }
128 
129     public static void addKeyword(Document doc, String field, long value) {
130         addKeyword(doc, field, String.valueOf(value));
131     }
132 
133     public static void addKeyword(Document doc, String field, String value) {
134         if (Validator.isNotNull(value)) {
135             doc.add(LuceneFields.getKeyword(field, value));
136         }
137     }
138 
139     public static void addKeyword(Document doc, String field, String[] values) {
140         if (values == null) {
141             return;
142         }
143 
144         for (int i = 0; i < values.length; i++) {
145             addKeyword(doc, field, values[i]);
146         }
147     }
148 
149     public static void addRequiredTerm(
150         BooleanQuery booleanQuery, String field, long value) {
151 
152         addRequiredTerm(booleanQuery, field, String.valueOf(value));
153     }
154 
155     public static void addRequiredTerm(
156         BooleanQuery booleanQuery, String field, String text) {
157 
158         //text = KeywordsUtil.escape(text);
159 
160         Term term = new Term(field, text);
161         TermQuery termQuery = new TermQuery(term);
162 
163         booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
164     }
165 
166     public static void addModifiedDate(Document doc) {
167         doc.add(LuceneFields.getDate(LuceneFields.MODIFIED));
168     }
169 
170     public static void addTerm(
171             BooleanQuery booleanQuery, String field, long value)
172         throws ParseException {
173 
174         addTerm(booleanQuery, field, String.valueOf(value));
175     }
176 
177     public static void addTerm(
178             BooleanQuery booleanQuery, String field, String text)
179         throws ParseException {
180 
181         if (Validator.isNotNull(text)) {
182             //Matcher matcher = TERM_END_PATTERN.matcher(text);
183 
184             // Add wildcard to the end of every word 4 chars or longer
185 
186             //text = matcher.replaceAll("$1*");
187 
188             // Add fuzzy query to the end of every word 4 chars or longer
189 
190             //text = text + " " + matcher.replaceAll("$1~");
191 
192             // Remove invalid hints
193 
194             //text = StringUtil.replace(text, "**", "*");
195             //text = StringUtil.replace(text, "*~", "*");
196             //text = StringUtil.replace(text, "~~", "~");
197             //text = StringUtil.replace(text, "~*", "~");
198 
199             QueryParser queryParser = new QueryParser(
200                 field, LuceneUtil.getAnalyzer());
201 
202             try {
203                 Query query = queryParser.parse(text);
204 
205                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
206             }
207             catch(ParseException pe) {
208                 if (_log.isDebugEnabled()) {
209                     _log.debug(
210                         "ParseException thrown, reverting to literal search",
211                         pe);
212                 }
213 
214                 text = KeywordsUtil.escape(text);
215 
216                 Query query = queryParser.parse(text);
217 
218                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
219             }
220         }
221     }
222 
223     public static void addText(Document doc, String field, long value) {
224         addText(doc, field, String.valueOf(value));
225     }
226 
227     public static void addText(Document doc, String field, String value) {
228         if (Validator.isNotNull(value)) {
229             doc.add(LuceneFields.getText(field, value));
230         }
231     }
232 
233     public static void checkLuceneDir(long companyId) {
234         if (INDEX_READ_ONLY) {
235             return;
236         }
237 
238         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
239 
240         IndexWriter writer = null;
241 
242         // Lucene does not properly release its lock on the index when
243         // IndexWriter throws an exception
244 
245         try {
246             if (luceneDir.fileExists("segments.gen")) {
247                 writer = new IndexWriter(
248                     luceneDir, LuceneUtil.getAnalyzer(), false);
249             }
250             else {
251                 writer = new IndexWriter(
252                     luceneDir, LuceneUtil.getAnalyzer(), true);
253             }
254         }
255         catch (IOException ioe) {
256             _log.error(ioe);
257         }
258         finally {
259             if (writer != null) {
260                 try {
261                     writer.close();
262                 }
263                 catch (IOException ioe) {
264                     _log.error(ioe);
265                 }
266             }
267         }
268     }
269 
270     public static void closeSearcher(Searcher searcher) {
271         try {
272             if (searcher != null){
273                 searcher.close();
274             }
275         }
276         catch (Exception e) {
277         }
278     }
279 
280     public static Hits closeSearcher(
281             Searcher searcher, String keywords, Exception e)
282         throws SystemException {
283 
284         closeSearcher(searcher);
285 
286         if (e instanceof BooleanQuery.TooManyClauses ||
287             e instanceof ParseException) {
288 
289             _log.error("Parsing keywords " + keywords, e);
290 
291             return new HitsImpl();
292         }
293         else {
294             throw new SystemException(e);
295         }
296     }
297 
298     public static void delete(long companyId) {
299         _instance._delete(companyId);
300     }
301 
302     public static void deleteDocuments(long companyId, Term term)
303         throws IOException {
304 
305         try {
306             _instance._sharedWriter.deleteDocuments(companyId, term);
307         }
308         catch (InterruptedException ie) {
309             _log.error(ie);
310         }
311     }
312 
313     public static Analyzer getAnalyzer() {
314         return _instance._getAnalyzer();
315     }
316 
317     public static Directory getLuceneDir(long companyId) {
318         return _instance._getLuceneDir(companyId);
319     }
320 
321     public static IndexReader getReader(long companyId) throws IOException {
322         return IndexReader.open(getLuceneDir(companyId));
323     }
324 
325     public static IndexSearcher getSearcher(long companyId)
326         throws IOException {
327 
328         return new IndexSearcher(getLuceneDir(companyId));
329     }
330 
331     public static IndexWriter getWriter(long companyId) throws IOException {
332         return getWriter(companyId, false);
333     }
334 
335     public static IndexWriter getWriter(long companyId, boolean create)
336         throws IOException {
337 
338         return _instance._sharedWriter.getWriter(companyId, create);
339     }
340 
341     public static void releaseLock(long companyId) {
342         _instance._sharedWriter.releaseLock(companyId);
343     }
344 
345     public static void write(long companyId) throws IOException {
346         _instance._sharedWriter.write(companyId);
347     }
348 
349     public static void write(IndexWriter writer) throws IOException {
350         _instance._sharedWriter.write(writer);
351     }
352 
353     private LuceneUtil() {
354         String analyzerName = PropsUtil.get(PropsUtil.LUCENE_ANALYZER);
355 
356         if (Validator.isNotNull(analyzerName)) {
357             try {
358                 _analyzerClass = Class.forName(analyzerName);
359             }
360             catch (Exception e) {
361                 _log.error(e);
362             }
363         }
364 
365         // Dialect
366 
367         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
368             Connection con = null;
369 
370             try {
371                 con = HibernateUtil.getConnection();
372 
373                 String url = con.getMetaData().getURL();
374 
375                 int x = url.indexOf(":");
376                 int y = url.indexOf(":", x + 1);
377 
378                 String urlPrefix = url.substring(x + 1, y);
379 
380                 String dialectClass = PropsUtil.get(
381                     PropsUtil.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
382 
383                 if (dialectClass != null) {
384                     if (_log.isDebugEnabled()) {
385                         _log.debug("JDBC class implementation " + dialectClass);
386                     }
387                 }
388                 else {
389                     if (_log.isDebugEnabled()) {
390                         _log.debug("JDBC class implementation is null");
391                     }
392                 }
393 
394                 if (dialectClass != null) {
395                     _dialect =
396                         (Dialect)Class.forName(dialectClass).newInstance();
397                 }
398             }
399             catch (Exception e) {
400                 _log.error(e);
401             }
402             finally{
403                 DataAccess.cleanUp(con);
404             }
405 
406             if (_dialect == null) {
407                 _log.error("No JDBC dialect found");
408             }
409         }
410     }
411 
412     public void _delete(long companyId) {
413         if (LuceneUtil.INDEX_READ_ONLY) {
414             return;
415         }
416 
417         if (_log.isDebugEnabled()) {
418             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
419         }
420 
421         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
422             _deleteFile(companyId);
423         }
424         else if (PropsValues.LUCENE_STORE_TYPE.equals(
425                     _LUCENE_STORE_TYPE_JDBC)) {
426 
427             _deleteJdbc(companyId);
428         }
429         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
430             _deleteRam(companyId);
431         }
432         else {
433             throw new RuntimeException(
434                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
435         }
436     }
437 
438     private void _deleteFile(long companyId) {
439         String path = _getPath(companyId);
440 
441         try {
442             Directory directory = FSDirectory.getDirectory(path, false);
443 
444             directory.close();
445         }
446         catch (Exception e) {
447             if (_log.isWarnEnabled()) {
448                 _log.warn("Could not close directory " + path);
449             }
450         }
451 
452         FileUtil.deltree(path);
453     }
454 
455     private void _deleteJdbc(long companyId) {
456         String tableName = _getTableName(companyId);
457 
458         try {
459             Directory directory = _jdbcDirectories.remove(tableName);
460 
461             if (directory != null) {
462                 directory.close();
463             }
464         }
465         catch (Exception e) {
466             if (_log.isWarnEnabled()) {
467                 _log.warn("Could not close directory " + tableName);
468             }
469         }
470 
471         Connection con = null;
472         Statement s = null;
473 
474         try {
475             con = HibernateUtil.getConnection();
476 
477             s = con.createStatement();
478 
479             s.executeUpdate("DELETE FROM " + tableName);
480         }
481         catch (Exception e) {
482             if (_log.isWarnEnabled()) {
483                 _log.warn("Could not truncate " + tableName);
484             }
485         }
486         finally {
487             DataAccess.cleanUp(con, s);
488         }
489     }
490 
491     private void _deleteRam(long companyId) {
492     }
493 
494     private Analyzer _getAnalyzer() {
495         try {
496             return (Analyzer)_analyzerClass.newInstance();
497         }
498         catch (Exception e) {
499             throw new RuntimeException(e);
500         }
501     }
502 
503     private Directory _getLuceneDir(long companyId) {
504         if (_log.isDebugEnabled()) {
505             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
506         }
507 
508         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
509             return _getLuceneDirFile(companyId);
510         }
511         else if (PropsValues.LUCENE_STORE_TYPE.equals(
512                     _LUCENE_STORE_TYPE_JDBC)) {
513 
514             return _getLuceneDirJdbc(companyId);
515         }
516         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
517             return _getLuceneDirRam(companyId);
518         }
519         else {
520             throw new RuntimeException(
521                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
522         }
523     }
524 
525     private Directory _getLuceneDirFile(long companyId) {
526         Directory directory = null;
527 
528         String path = _getPath(companyId);
529 
530         try {
531             directory = FSDirectory.getDirectory(path, false);
532         }
533         catch (IOException ioe1) {
534             try {
535                 if (directory != null) {
536                     directory.close();
537                 }
538 
539                 directory = FSDirectory.getDirectory(path, true);
540             }
541             catch (IOException ioe2) {
542                 throw new RuntimeException(ioe2);
543             }
544         }
545 
546         return directory;
547     }
548 
549     private Directory _getLuceneDirJdbc(long companyId) {
550         JdbcDirectory directory = null;
551 
552         ClassLoader contextClassLoader =
553             Thread.currentThread().getContextClassLoader();
554 
555         try {
556             Thread.currentThread().setContextClassLoader(
557                 PortalClassLoaderUtil.getClassLoader());
558 
559             String tableName = _getTableName(companyId);
560 
561             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
562 
563             if (directory != null) {
564                 return directory;
565             }
566 
567             try {
568                 DataSource ds = HibernateUtil.getDataSource();
569 
570                 directory = new JdbcDirectory(ds, _dialect, tableName);
571 
572                 _jdbcDirectories.put(tableName, directory);
573 
574                 if (!directory.tableExists()) {
575                     directory.create();
576                 }
577             }
578             catch (IOException ioe) {
579                 throw new RuntimeException(ioe);
580             }
581             catch (UnsupportedOperationException uoe) {
582                 if (_log.isWarnEnabled()) {
583                     _log.warn(
584                         "Database doesn't support the ability to check " +
585                             "whether a table exists");
586                 }
587 
588                 _manuallyCreateJdbcDirectory(directory, tableName);
589             }
590         }
591         finally {
592             Thread.currentThread().setContextClassLoader(contextClassLoader);
593         }
594 
595         return directory;
596     }
597 
598     private Directory _getLuceneDirRam(long companyId) {
599         String path = _getPath(companyId);
600 
601         Directory directory = _ramDirectories.get(path);
602 
603         if (directory == null) {
604             directory = new RAMDirectory();
605 
606             _ramDirectories.put(path, directory);
607         }
608 
609         return directory;
610     }
611 
612     private String _getPath(long companyId) {
613         StringMaker sm = new StringMaker();
614 
615         sm.append(PropsValues.LUCENE_DIR);
616         sm.append(companyId);
617         sm.append(StringPool.SLASH);
618 
619         return sm.toString();
620     }
621 
622     private String _getTableName(long companyId) {
623         return _LUCENE_TABLE_PREFIX + companyId;
624     }
625 
626     private void _manuallyCreateJdbcDirectory(
627         JdbcDirectory directory, String tableName) {
628 
629         // LEP-2181
630 
631         Connection con = null;
632         ResultSet rs = null;
633 
634         try {
635             con = HibernateUtil.getConnection();
636 
637             // Check if table exists
638 
639             DatabaseMetaData metaData = con.getMetaData();
640 
641             rs = metaData.getTables(null, null, tableName, null);
642 
643             if (!rs.next()) {
644                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
645 
646                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
647 
648                 Class<?> lockClass = directory.getSettings().getLockClass();
649 
650                 JdbcLock jdbcLock = null;
651 
652                 try {
653                     jdbcLock = (JdbcLock)lockClass.newInstance();
654                 }
655                 catch (Exception e) {
656                     throw new JdbcStoreException(
657                         "Failed to create lock class " + lockClass);
658                 }
659 
660                 jdbcLock.initializeDatabase(directory);
661             }
662         }
663         catch (Exception e) {
664             if (_log.isWarnEnabled()) {
665                 _log.warn("Could not create " + tableName);
666             }
667         }
668         finally {
669             DataAccess.cleanUp(con, null, rs);
670         }
671     }
672 
673     private static final String _LUCENE_STORE_TYPE_FILE = "file";
674 
675     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
676 
677     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
678 
679     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
680 
681     private static Log _log = LogFactory.getLog(LuceneUtil.class);
682 
683     private static LuceneUtil _instance = new LuceneUtil();
684 
685     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
686     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
687     private Dialect _dialect;
688     private Map<String, Directory> _jdbcDirectories =
689         new ConcurrentHashMap<String, Directory>();
690     private Map<String, Directory> _ramDirectories =
691         new ConcurrentHashMap<String, Directory>();
692 
693 }