1   /**
2    * Copyright (c) 2000-2008 Liferay, Inc. All rights reserved.
3    *
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   *
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.search.lucene;
24  
25  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26  import com.liferay.portal.kernel.jndi.PortalJNDIUtil;
27  import com.liferay.portal.kernel.search.Field;
28  import com.liferay.portal.kernel.util.FileUtil;
29  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
30  import com.liferay.portal.kernel.util.StringPool;
31  import com.liferay.portal.kernel.util.Validator;
32  import com.liferay.portal.util.PropsKeys;
33  import com.liferay.portal.util.PropsUtil;
34  import com.liferay.portal.util.PropsValues;
35  import com.liferay.util.lucene.KeywordsUtil;
36  
37  import java.io.IOException;
38  
39  import java.sql.Connection;
40  import java.sql.DatabaseMetaData;
41  import java.sql.ResultSet;
42  import java.sql.Statement;
43  
44  import java.util.Date;
45  import java.util.Map;
46  import java.util.concurrent.ConcurrentHashMap;
47  
48  import javax.naming.NamingException;
49  
50  import javax.sql.DataSource;
51  
52  import org.apache.commons.logging.Log;
53  import org.apache.commons.logging.LogFactory;
54  import org.apache.lucene.analysis.Analyzer;
55  import org.apache.lucene.analysis.WhitespaceAnalyzer;
56  import org.apache.lucene.document.Document;
57  import org.apache.lucene.index.IndexReader;
58  import org.apache.lucene.index.IndexWriter;
59  import org.apache.lucene.index.Term;
60  import org.apache.lucene.queryParser.ParseException;
61  import org.apache.lucene.queryParser.QueryParser;
62  import org.apache.lucene.search.BooleanClause;
63  import org.apache.lucene.search.BooleanQuery;
64  import org.apache.lucene.search.IndexSearcher;
65  import org.apache.lucene.search.Query;
66  import org.apache.lucene.search.TermQuery;
67  import org.apache.lucene.store.Directory;
68  import org.apache.lucene.store.FSDirectory;
69  import org.apache.lucene.store.RAMDirectory;
70  import org.apache.lucene.store.jdbc.JdbcDirectory;
71  import org.apache.lucene.store.jdbc.JdbcStoreException;
72  import org.apache.lucene.store.jdbc.dialect.Dialect;
73  import org.apache.lucene.store.jdbc.lock.JdbcLock;
74  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75  
76  /**
77   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
78   *
79   * @author Brian Wing Shun Chan
80   * @author Harry Mark
81   *
82   */
83  public class LuceneUtil {
84  
85      public static void acquireLock(long companyId) {
86          try {
87              _instance._sharedWriter.acquireLock(companyId, true);
88          }
89          catch (InterruptedException ie) {
90              _log.error(ie);
91          }
92      }
93  
94      public static void addDate(Document doc, String field, Date value) {
95          doc.add(LuceneFields.getDate(field, value));
96      }
97  
98      public static void addExactTerm(
99          BooleanQuery booleanQuery, String field, long value) {
100 
101         addExactTerm(booleanQuery, field, String.valueOf(value));
102     }
103 
104     public static void addExactTerm(
105         BooleanQuery booleanQuery, String field, String value) {
106 
107         //text = KeywordsUtil.escape(value);
108 
109         Query query = new TermQuery(new Term(field, value));
110 
111         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
112     }
113 
114     public static void addKeyword(Document doc, String field, double value) {
115         addKeyword(doc, field, String.valueOf(value));
116     }
117 
118     public static void addKeyword(Document doc, String field, long value) {
119         addKeyword(doc, field, String.valueOf(value));
120     }
121 
122     public static void addKeyword(Document doc, String field, String value) {
123         if (Validator.isNotNull(value)) {
124             doc.add(LuceneFields.getKeyword(field, value));
125         }
126     }
127 
128     public static void addKeyword(Document doc, String field, String[] values) {
129         if (values == null) {
130             return;
131         }
132 
133         for (int i = 0; i < values.length; i++) {
134             addKeyword(doc, field, values[i]);
135         }
136     }
137 
138     public static void addRequiredTerm(
139         BooleanQuery booleanQuery, String field, long value) {
140 
141         addRequiredTerm(booleanQuery, field, String.valueOf(value));
142     }
143 
144     public static void addRequiredTerm(
145         BooleanQuery booleanQuery, String field, String value) {
146 
147         //text = KeywordsUtil.escape(value);
148 
149         Term term = new Term(field, value);
150         TermQuery termQuery = new TermQuery(term);
151 
152         booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
153     }
154 
155     public static void addModifiedDate(Document doc) {
156         doc.add(LuceneFields.getDate(Field.MODIFIED));
157     }
158 
159     public static void addTerm(
160             BooleanQuery booleanQuery, String field, long value)
161         throws ParseException {
162 
163         addTerm(booleanQuery, field, String.valueOf(value));
164     }
165 
166     public static void addTerm(
167             BooleanQuery booleanQuery, String field, String value)
168         throws ParseException {
169 
170         if (Validator.isNotNull(value)) {
171             QueryParser queryParser = new QueryParser(
172                 field, LuceneUtil.getAnalyzer());
173 
174             try {
175                 Query query = queryParser.parse(value);
176 
177                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
178             }
179             catch (ParseException pe) {
180                 if (_log.isDebugEnabled()) {
181                     _log.debug(
182                         "ParseException thrown, reverting to literal search",
183                         pe);
184                 }
185 
186                 value = KeywordsUtil.escape(value);
187 
188                 Query query = queryParser.parse(value);
189 
190                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
191             }
192         }
193     }
194 
195     public static void addText(Document doc, String field, long value) {
196         addText(doc, field, String.valueOf(value));
197     }
198 
199     public static void addText(Document doc, String field, String value) {
200         if (Validator.isNotNull(value)) {
201             doc.add(LuceneFields.getText(field, value));
202         }
203     }
204 
205     public static void checkLuceneDir(long companyId) {
206         if (PropsValues.INDEX_READ_ONLY) {
207             return;
208         }
209 
210         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
211 
212         try {
213 
214             // LEP-6078
215 
216             if (luceneDir.fileExists("write.lock")) {
217                 luceneDir.deleteFile("write.lock");
218             }
219         }
220         catch (IOException ioe) {
221             _log.error("Unable to clear write lock", ioe);
222         }
223 
224         IndexWriter writer = null;
225 
226         // Lucene does not properly release its lock on the index when
227         // IndexWriter throws an exception
228 
229         try {
230             if (luceneDir.fileExists("segments.gen")) {
231                 writer = new IndexWriter(
232                     luceneDir, LuceneUtil.getAnalyzer(), false);
233             }
234             else {
235                 writer = new IndexWriter(
236                     luceneDir, LuceneUtil.getAnalyzer(), true);
237             }
238         }
239         catch (IOException ioe) {
240             _log.error("Check Lucene directory failed for " + companyId, ioe);
241         }
242         finally {
243             if (writer != null) {
244                 try {
245                     writer.close();
246                 }
247                 catch (IOException ioe) {
248                     _log.error(ioe);
249                 }
250             }
251         }
252     }
253 
254     public static void delete(long companyId) {
255         _instance._delete(companyId);
256     }
257 
258     public static void deleteDocuments(long companyId, Term term)
259         throws IOException {
260 
261         try {
262             _instance._sharedWriter.deleteDocuments(companyId, term);
263         }
264         catch (InterruptedException ie) {
265             _log.error(ie);
266         }
267     }
268 
269     public static Analyzer getAnalyzer() {
270         return _instance._getAnalyzer();
271     }
272 
273     public static Directory getLuceneDir(long companyId) {
274         return _instance._getLuceneDir(companyId);
275     }
276 
277     public static IndexReader getReader(long companyId) throws IOException {
278         return IndexReader.open(getLuceneDir(companyId));
279     }
280 
281     public static IndexSearcher getSearcher(long companyId)
282         throws IOException {
283 
284         return new IndexSearcher(getLuceneDir(companyId));
285     }
286 
287     public static IndexWriter getWriter(long companyId) throws IOException {
288         return getWriter(companyId, false);
289     }
290 
291     public static IndexWriter getWriter(long companyId, boolean create)
292         throws IOException {
293 
294         return _instance._sharedWriter.getWriter(companyId, create);
295     }
296 
297     public static void releaseLock(long companyId) {
298         _instance._sharedWriter.releaseLock(companyId);
299     }
300 
301     public static void write(long companyId) {
302         _instance._sharedWriter.write(companyId);
303     }
304 
305     public static void write(IndexWriter writer) throws IOException {
306         _instance._sharedWriter.write(writer);
307     }
308 
309     private LuceneUtil() {
310         String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
311 
312         if (Validator.isNotNull(analyzerName)) {
313             try {
314                 _analyzerClass = Class.forName(analyzerName);
315             }
316             catch (Exception e) {
317                 _log.error(e);
318             }
319         }
320 
321         // Dialect
322 
323         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
324             Connection con = null;
325 
326             try {
327                 con = DataAccess.getConnection();
328 
329                 String url = con.getMetaData().getURL();
330 
331                 int x = url.indexOf(":");
332                 int y = url.indexOf(":", x + 1);
333 
334                 String urlPrefix = url.substring(x + 1, y);
335 
336                 String dialectClass = PropsUtil.get(
337                     PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
338 
339                 if (dialectClass != null) {
340                     if (_log.isDebugEnabled()) {
341                         _log.debug("JDBC class implementation " + dialectClass);
342                     }
343                 }
344                 else {
345                     if (_log.isDebugEnabled()) {
346                         _log.debug("JDBC class implementation is null");
347                     }
348                 }
349 
350                 if (dialectClass != null) {
351                     _dialect =
352                         (Dialect)Class.forName(dialectClass).newInstance();
353                 }
354             }
355             catch (Exception e) {
356                 _log.error(e);
357             }
358             finally{
359                 DataAccess.cleanUp(con);
360             }
361 
362             if (_dialect == null) {
363                 _log.error("No JDBC dialect found");
364             }
365         }
366     }
367 
368     public void _delete(long companyId) {
369         if (PropsValues.INDEX_READ_ONLY) {
370             return;
371         }
372 
373         if (_log.isDebugEnabled()) {
374             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
375         }
376 
377         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
378             _deleteFile(companyId);
379         }
380         else if (PropsValues.LUCENE_STORE_TYPE.equals(
381                     _LUCENE_STORE_TYPE_JDBC)) {
382 
383             _deleteJdbc(companyId);
384         }
385         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
386             _deleteRam(companyId);
387         }
388         else {
389             throw new RuntimeException(
390                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
391         }
392     }
393 
394     private void _deleteFile(long companyId) {
395         String path = _getPath(companyId);
396 
397         try {
398             Directory directory = FSDirectory.getDirectory(path, false);
399 
400             directory.close();
401         }
402         catch (Exception e) {
403             if (_log.isWarnEnabled()) {
404                 _log.warn("Could not close directory " + path);
405             }
406         }
407 
408         FileUtil.deltree(path);
409     }
410 
411     private void _deleteJdbc(long companyId) {
412         String tableName = _getTableName(companyId);
413 
414         try {
415             Directory directory = _jdbcDirectories.remove(tableName);
416 
417             if (directory != null) {
418                 directory.close();
419             }
420         }
421         catch (Exception e) {
422             if (_log.isWarnEnabled()) {
423                 _log.warn("Could not close directory " + tableName);
424             }
425         }
426 
427         Connection con = null;
428         Statement s = null;
429 
430         try {
431             con = DataAccess.getConnection();
432 
433             s = con.createStatement();
434 
435             s.executeUpdate("DELETE FROM " + tableName);
436         }
437         catch (Exception e) {
438             if (_log.isWarnEnabled()) {
439                 _log.warn("Could not truncate " + tableName);
440             }
441         }
442         finally {
443             DataAccess.cleanUp(con, s);
444         }
445     }
446 
447     private void _deleteRam(long companyId) {
448     }
449 
450     private Analyzer _getAnalyzer() {
451         try {
452             return (Analyzer)_analyzerClass.newInstance();
453         }
454         catch (Exception e) {
455             throw new RuntimeException(e);
456         }
457     }
458 
459     private Directory _getLuceneDir(long companyId) {
460         if (_log.isDebugEnabled()) {
461             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
462         }
463 
464         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
465             return _getLuceneDirFile(companyId);
466         }
467         else if (PropsValues.LUCENE_STORE_TYPE.equals(
468                     _LUCENE_STORE_TYPE_JDBC)) {
469 
470             return _getLuceneDirJdbc(companyId);
471         }
472         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
473             return _getLuceneDirRam(companyId);
474         }
475         else {
476             throw new RuntimeException(
477                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
478         }
479     }
480 
481     private Directory _getLuceneDirFile(long companyId) {
482         Directory directory = null;
483 
484         String path = _getPath(companyId);
485 
486         try {
487             directory = FSDirectory.getDirectory(path, false);
488         }
489         catch (IOException ioe1) {
490             try {
491                 if (directory != null) {
492                     directory.close();
493                 }
494 
495                 directory = FSDirectory.getDirectory(path, true);
496             }
497             catch (IOException ioe2) {
498                 throw new RuntimeException(ioe2);
499             }
500         }
501 
502         return directory;
503     }
504 
505     private Directory _getLuceneDirJdbc(long companyId) {
506         JdbcDirectory directory = null;
507 
508         ClassLoader contextClassLoader =
509             Thread.currentThread().getContextClassLoader();
510 
511         try {
512             Thread.currentThread().setContextClassLoader(
513                 PortalClassLoaderUtil.getClassLoader());
514 
515             String tableName = _getTableName(companyId);
516 
517             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
518 
519             if (directory != null) {
520                 return directory;
521             }
522 
523             try {
524                 DataSource ds = PortalJNDIUtil.getDataSource();
525 
526                 directory = new JdbcDirectory(ds, _dialect, tableName);
527 
528                 _jdbcDirectories.put(tableName, directory);
529 
530                 if (!directory.tableExists()) {
531                     directory.create();
532                 }
533             }
534             catch (IOException ioe) {
535                 throw new RuntimeException(ioe);
536             }
537             catch (NamingException ne) {
538                 throw new RuntimeException(ne);
539             }
540             catch (UnsupportedOperationException uoe) {
541                 if (_log.isWarnEnabled()) {
542                     _log.warn(
543                         "Database doesn't support the ability to check " +
544                             "whether a table exists");
545                 }
546 
547                 _manuallyCreateJdbcDirectory(directory, tableName);
548             }
549         }
550         finally {
551             Thread.currentThread().setContextClassLoader(contextClassLoader);
552         }
553 
554         return directory;
555     }
556 
557     private Directory _getLuceneDirRam(long companyId) {
558         String path = _getPath(companyId);
559 
560         Directory directory = _ramDirectories.get(path);
561 
562         if (directory == null) {
563             directory = new RAMDirectory();
564 
565             _ramDirectories.put(path, directory);
566         }
567 
568         return directory;
569     }
570 
571     private String _getPath(long companyId) {
572         StringBuilder sb = new StringBuilder();
573 
574         sb.append(PropsValues.LUCENE_DIR);
575         sb.append(companyId);
576         sb.append(StringPool.SLASH);
577 
578         return sb.toString();
579     }
580 
581     private String _getTableName(long companyId) {
582         return _LUCENE_TABLE_PREFIX + companyId;
583     }
584 
585     private void _manuallyCreateJdbcDirectory(
586         JdbcDirectory directory, String tableName) {
587 
588         // LEP-2181
589 
590         Connection con = null;
591         ResultSet rs = null;
592 
593         try {
594             con = DataAccess.getConnection();
595 
596             // Check if table exists
597 
598             DatabaseMetaData metaData = con.getMetaData();
599 
600             rs = metaData.getTables(null, null, tableName, null);
601 
602             if (!rs.next()) {
603                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
604 
605                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
606 
607                 Class<?> lockClass = directory.getSettings().getLockClass();
608 
609                 JdbcLock jdbcLock = null;
610 
611                 try {
612                     jdbcLock = (JdbcLock)lockClass.newInstance();
613                 }
614                 catch (Exception e) {
615                     throw new JdbcStoreException(
616                         "Failed to create lock class " + lockClass);
617                 }
618 
619                 jdbcLock.initializeDatabase(directory);
620             }
621         }
622         catch (Exception e) {
623             if (_log.isWarnEnabled()) {
624                 _log.warn("Could not create " + tableName);
625             }
626         }
627         finally {
628             DataAccess.cleanUp(con, null, rs);
629         }
630     }
631 
632     private static final String _LUCENE_STORE_TYPE_FILE = "file";
633 
634     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
635 
636     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
637 
638     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
639 
640     private static Log _log = LogFactory.getLog(LuceneUtil.class);
641 
642     private static LuceneUtil _instance = new LuceneUtil();
643 
644     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
645     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
646     private Dialect _dialect;
647     private Map<String, Directory> _jdbcDirectories =
648         new ConcurrentHashMap<String, Directory>();
649     private Map<String, Directory> _ramDirectories =
650         new ConcurrentHashMap<String, Directory>();
651 
652 }