1
22
23 package com.liferay.portal.search.lucene;
24
25 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26 import com.liferay.portal.kernel.log.Log;
27 import com.liferay.portal.kernel.log.LogFactoryUtil;
28 import com.liferay.portal.kernel.search.SearchEngineUtil;
29 import com.liferay.portal.kernel.util.FileUtil;
30 import com.liferay.portal.kernel.util.InfrastructureUtil;
31 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
32 import com.liferay.portal.kernel.util.StringPool;
33 import com.liferay.portal.kernel.util.StringUtil;
34 import com.liferay.portal.kernel.util.Validator;
35 import com.liferay.portal.util.PropsKeys;
36 import com.liferay.portal.util.PropsUtil;
37 import com.liferay.portal.util.PropsValues;
38 import com.liferay.util.lucene.KeywordsUtil;
39
40 import java.io.IOException;
41
42 import java.sql.Connection;
43 import java.sql.DatabaseMetaData;
44 import java.sql.ResultSet;
45 import java.sql.Statement;
46
47 import java.util.Date;
48 import java.util.Map;
49 import java.util.concurrent.ConcurrentHashMap;
50
51 import javax.sql.DataSource;
52
53 import org.apache.lucene.analysis.Analyzer;
54 import org.apache.lucene.analysis.WhitespaceAnalyzer;
55 import org.apache.lucene.document.Document;
56 import org.apache.lucene.index.IndexReader;
57 import org.apache.lucene.index.IndexWriter;
58 import org.apache.lucene.index.Term;
59 import org.apache.lucene.queryParser.ParseException;
60 import org.apache.lucene.queryParser.QueryParser;
61 import org.apache.lucene.search.BooleanClause;
62 import org.apache.lucene.search.BooleanQuery;
63 import org.apache.lucene.search.IndexSearcher;
64 import org.apache.lucene.search.Query;
65 import org.apache.lucene.search.TermQuery;
66 import org.apache.lucene.search.WildcardQuery;
67 import org.apache.lucene.store.Directory;
68 import org.apache.lucene.store.FSDirectory;
69 import org.apache.lucene.store.RAMDirectory;
70 import org.apache.lucene.store.jdbc.JdbcDirectory;
71 import org.apache.lucene.store.jdbc.JdbcStoreException;
72 import org.apache.lucene.store.jdbc.dialect.Dialect;
73 import org.apache.lucene.store.jdbc.lock.JdbcLock;
74 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75
76
82 public class LuceneUtil {
83
84 public static void acquireLock(long companyId) {
85 try {
86 _instance._sharedWriter.acquireLock(companyId, true);
87 }
88 catch (InterruptedException ie) {
89 _log.error(ie);
90 }
91 }
92
93 public static void addDate(Document doc, String field, Date value) {
94 doc.add(LuceneFields.getDate(field, value));
95 }
96
97 public static void addExactTerm(
98 BooleanQuery booleanQuery, String field, boolean value) {
99
100 addExactTerm(booleanQuery, field, String.valueOf(value));
101 }
102
103 public static void addExactTerm(
104 BooleanQuery booleanQuery, String field, Boolean value) {
105
106 addExactTerm(booleanQuery, field, String.valueOf(value));
107 }
108
109 public static void addExactTerm(
110 BooleanQuery booleanQuery, String field, double value) {
111
112 addExactTerm(booleanQuery, field, String.valueOf(value));
113 }
114
115 public static void addExactTerm(
116 BooleanQuery booleanQuery, String field, Double value) {
117
118 addExactTerm(booleanQuery, field, String.valueOf(value));
119 }
120
121 public static void addExactTerm(
122 BooleanQuery booleanQuery, String field, int value) {
123
124 addExactTerm(booleanQuery, field, String.valueOf(value));
125 }
126
127 public static void addExactTerm(
128 BooleanQuery booleanQuery, String field, Integer value) {
129
130 addExactTerm(booleanQuery, field, String.valueOf(value));
131 }
132
133 public static void addExactTerm(
134 BooleanQuery booleanQuery, String field, long value) {
135
136 addExactTerm(booleanQuery, field, String.valueOf(value));
137 }
138
139 public static void addExactTerm(
140 BooleanQuery booleanQuery, String field, Long value) {
141
142 addExactTerm(booleanQuery, field, String.valueOf(value));
143 }
144
145 public static void addExactTerm(
146 BooleanQuery booleanQuery, String field, short value) {
147
148 addExactTerm(booleanQuery, field, String.valueOf(value));
149 }
150
151 public static void addExactTerm(
152 BooleanQuery booleanQuery, String field, Short value) {
153
154 addExactTerm(booleanQuery, field, String.valueOf(value));
155 }
156
157 public static void addExactTerm(
158 BooleanQuery booleanQuery, String field, String value) {
159
160
162 Query query = new TermQuery(new Term(field, value));
163
164 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
165 }
166
167 public static void addRequiredTerm(
168 BooleanQuery booleanQuery, String field, boolean value) {
169
170 addRequiredTerm(booleanQuery, field, String.valueOf(value));
171 }
172
173 public static void addRequiredTerm(
174 BooleanQuery booleanQuery, String field, Boolean value) {
175
176 addRequiredTerm(booleanQuery, field, String.valueOf(value));
177 }
178
179 public static void addRequiredTerm(
180 BooleanQuery booleanQuery, String field, double value) {
181
182 addRequiredTerm(booleanQuery, field, String.valueOf(value));
183 }
184
185 public static void addRequiredTerm(
186 BooleanQuery booleanQuery, String field, Double value) {
187
188 addRequiredTerm(booleanQuery, field, String.valueOf(value));
189 }
190
191 public static void addRequiredTerm(
192 BooleanQuery booleanQuery, String field, int value) {
193
194 addRequiredTerm(booleanQuery, field, String.valueOf(value));
195 }
196
197 public static void addRequiredTerm(
198 BooleanQuery booleanQuery, String field, Integer value) {
199
200 addRequiredTerm(booleanQuery, field, String.valueOf(value));
201 }
202
203 public static void addRequiredTerm(
204 BooleanQuery booleanQuery, String field, long value) {
205
206 addRequiredTerm(booleanQuery, field, String.valueOf(value));
207 }
208
209 public static void addRequiredTerm(
210 BooleanQuery booleanQuery, String field, Long value) {
211
212 addRequiredTerm(booleanQuery, field, String.valueOf(value));
213 }
214
215 public static void addRequiredTerm(
216 BooleanQuery booleanQuery, String field, short value) {
217
218 addRequiredTerm(booleanQuery, field, String.valueOf(value));
219 }
220
221 public static void addRequiredTerm(
222 BooleanQuery booleanQuery, String field, Short value) {
223
224 addRequiredTerm(booleanQuery, field, String.valueOf(value));
225 }
226
227 public static void addRequiredTerm(
228 BooleanQuery booleanQuery, String field, String value) {
229
230 addRequiredTerm(booleanQuery, field, value, false);
231 }
232
233 public static void addRequiredTerm(
234 BooleanQuery booleanQuery, String field, String value, boolean like) {
235
236 if (like) {
237 value = StringUtil.replace(
238 value, StringPool.PERCENT, StringPool.STAR);
239
240 value = value.toLowerCase();
241
242 WildcardQuery wildcardQuery = new WildcardQuery(
243 new Term(field, value));
244
245 booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
246 }
247 else {
248
250 Term term = new Term(field, value);
251 TermQuery termQuery = new TermQuery(term);
252
253 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
254 }
255 }
256
257 public static void addTerm(
258 BooleanQuery booleanQuery, String field, long value)
259 throws ParseException {
260
261 addTerm(booleanQuery, field, String.valueOf(value));
262 }
263
264 public static void addTerm(
265 BooleanQuery booleanQuery, String field, String value)
266 throws ParseException {
267
268 addTerm(booleanQuery, field, value, false);
269 }
270
271 public static void addTerm(
272 BooleanQuery booleanQuery, String field, String value,
273 boolean like)
274 throws ParseException {
275
276 if (Validator.isNull(value)) {
277 return;
278 }
279
280 if (like) {
281 value = value.toLowerCase();
282
283 StringBuilder sb = new StringBuilder();
284
285 sb.append(StringPool.STAR);
286 sb.append(value);
287 sb.append(StringPool.STAR);
288
289 WildcardQuery wildcardQuery = new WildcardQuery(
290 new Term(field, sb.toString()));
291
292 booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
293 }
294 else {
295 QueryParser queryParser = new QueryParser(
296 field, LuceneUtil.getAnalyzer());
297
298 try {
299 Query query = queryParser.parse(value);
300
301 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
302 }
303 catch (ParseException pe) {
304 if (_log.isDebugEnabled()) {
305 _log.debug(
306 "ParseException thrown, reverting to literal search",
307 pe);
308 }
309
310 value = KeywordsUtil.escape(value);
311
312 Query query = queryParser.parse(value);
313
314 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
315 }
316 }
317 }
318
319 public static void checkLuceneDir(long companyId) {
320 if (SearchEngineUtil.isIndexReadOnly()) {
321 return;
322 }
323
324 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
325
326 try {
327
328
330 if (luceneDir.fileExists("write.lock")) {
331 luceneDir.deleteFile("write.lock");
332 }
333 }
334 catch (IOException ioe) {
335 _log.error("Unable to clear write lock", ioe);
336 }
337
338 IndexWriter writer = null;
339
340
343 try {
344 if (luceneDir.fileExists("segments.gen")) {
345 writer = new IndexWriter(
346 luceneDir, LuceneUtil.getAnalyzer(), false);
347 }
348 else {
349 writer = new IndexWriter(
350 luceneDir, LuceneUtil.getAnalyzer(), true);
351 }
352 }
353 catch (IOException ioe) {
354 _log.error("Check Lucene directory failed for " + companyId, ioe);
355 }
356 finally {
357 if (writer != null) {
358 try {
359 writer.close();
360 }
361 catch (IOException ioe) {
362 _log.error(ioe);
363 }
364 }
365 }
366 }
367
368 public static void delete(long companyId) {
369 _instance._delete(companyId);
370 }
371
372 public static void deleteDocuments(long companyId, Term term)
373 throws IOException {
374
375 try {
376 _instance._sharedWriter.deleteDocuments(companyId, term);
377 }
378 catch (InterruptedException ie) {
379 _log.error(ie);
380 }
381 }
382
383 public static Analyzer getAnalyzer() {
384 return _instance._getAnalyzer();
385 }
386
387 public static Directory getLuceneDir(long companyId) {
388 return _instance._getLuceneDir(companyId);
389 }
390
391 public static IndexReader getReader(long companyId) throws IOException {
392 return IndexReader.open(getLuceneDir(companyId));
393 }
394
395 public static IndexSearcher getSearcher(long companyId)
396 throws IOException {
397
398 return new IndexSearcher(getLuceneDir(companyId));
399 }
400
401 public static IndexWriter getWriter(long companyId) throws IOException {
402 return getWriter(companyId, false);
403 }
404
405 public static IndexWriter getWriter(long companyId, boolean create)
406 throws IOException {
407
408 return _instance._sharedWriter.getWriter(companyId, create);
409 }
410
411 public static void releaseLock(long companyId) {
412 _instance._sharedWriter.releaseLock(companyId);
413 }
414
415 public static void write(long companyId) {
416 _instance._sharedWriter.write(companyId);
417 }
418
419 public static void write(IndexWriter writer) throws IOException {
420 _instance._sharedWriter.write(writer);
421 }
422
423 private LuceneUtil() {
424 String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
425
426 if (Validator.isNotNull(analyzerName)) {
427 try {
428 _analyzerClass = Class.forName(analyzerName);
429 }
430 catch (Exception e) {
431 _log.error(e);
432 }
433 }
434
435
437 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
438 Connection con = null;
439
440 try {
441 con = DataAccess.getConnection();
442
443 String url = con.getMetaData().getURL();
444
445 int x = url.indexOf(":");
446 int y = url.indexOf(":", x + 1);
447
448 String urlPrefix = url.substring(x + 1, y);
449
450 String dialectClass = PropsUtil.get(
451 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
452
453 if (dialectClass != null) {
454 if (_log.isDebugEnabled()) {
455 _log.debug("JDBC class implementation " + dialectClass);
456 }
457 }
458 else {
459 if (_log.isDebugEnabled()) {
460 _log.debug("JDBC class implementation is null");
461 }
462 }
463
464 if (dialectClass != null) {
465 _dialect =
466 (Dialect)Class.forName(dialectClass).newInstance();
467 }
468 }
469 catch (Exception e) {
470 _log.error(e);
471 }
472 finally{
473 DataAccess.cleanUp(con);
474 }
475
476 if (_dialect == null) {
477 _log.error("No JDBC dialect found");
478 }
479 }
480 }
481
482 public void _delete(long companyId) {
483 if (SearchEngineUtil.isIndexReadOnly()) {
484 return;
485 }
486
487 if (_log.isDebugEnabled()) {
488 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
489 }
490
491 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
492 _deleteFile(companyId);
493 }
494 else if (PropsValues.LUCENE_STORE_TYPE.equals(
495 _LUCENE_STORE_TYPE_JDBC)) {
496
497 _deleteJdbc(companyId);
498 }
499 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
500 _deleteRam(companyId);
501 }
502 else {
503 throw new RuntimeException(
504 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
505 }
506 }
507
508 private void _deleteFile(long companyId) {
509 String path = _getPath(companyId);
510
511 try {
512 Directory directory = FSDirectory.getDirectory(path, false);
513
514 directory.close();
515 }
516 catch (Exception e) {
517 if (_log.isWarnEnabled()) {
518 _log.warn("Could not close directory " + path);
519 }
520 }
521
522 FileUtil.deltree(path);
523 }
524
525 private void _deleteJdbc(long companyId) {
526 String tableName = _getTableName(companyId);
527
528 try {
529 Directory directory = _jdbcDirectories.remove(tableName);
530
531 if (directory != null) {
532 directory.close();
533 }
534 }
535 catch (Exception e) {
536 if (_log.isWarnEnabled()) {
537 _log.warn("Could not close directory " + tableName);
538 }
539 }
540
541 Connection con = null;
542 Statement s = null;
543
544 try {
545 con = DataAccess.getConnection();
546
547 s = con.createStatement();
548
549 s.executeUpdate("DELETE FROM " + tableName);
550 }
551 catch (Exception e) {
552 if (_log.isWarnEnabled()) {
553 _log.warn("Could not truncate " + tableName);
554 }
555 }
556 finally {
557 DataAccess.cleanUp(con, s);
558 }
559 }
560
561 private void _deleteRam(long companyId) {
562 }
563
564 private Analyzer _getAnalyzer() {
565 try {
566 return (Analyzer)_analyzerClass.newInstance();
567 }
568 catch (Exception e) {
569 throw new RuntimeException(e);
570 }
571 }
572
573 private Directory _getLuceneDir(long companyId) {
574 if (_log.isDebugEnabled()) {
575 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
576 }
577
578 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
579 return _getLuceneDirFile(companyId);
580 }
581 else if (PropsValues.LUCENE_STORE_TYPE.equals(
582 _LUCENE_STORE_TYPE_JDBC)) {
583
584 return _getLuceneDirJdbc(companyId);
585 }
586 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
587 return _getLuceneDirRam(companyId);
588 }
589 else {
590 throw new RuntimeException(
591 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
592 }
593 }
594
595 private Directory _getLuceneDirFile(long companyId) {
596 Directory directory = null;
597
598 String path = _getPath(companyId);
599
600 try {
601 directory = FSDirectory.getDirectory(path, false);
602 }
603 catch (IOException ioe1) {
604 try {
605 if (directory != null) {
606 directory.close();
607 }
608
609 directory = FSDirectory.getDirectory(path, true);
610 }
611 catch (IOException ioe2) {
612 throw new RuntimeException(ioe2);
613 }
614 }
615
616 return directory;
617 }
618
619 private Directory _getLuceneDirJdbc(long companyId) {
620 JdbcDirectory directory = null;
621
622 Thread currentThread = Thread.currentThread();
623
624 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
625
626 try {
627 currentThread.setContextClassLoader(
628 PortalClassLoaderUtil.getClassLoader());
629
630 String tableName = _getTableName(companyId);
631
632 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
633
634 if (directory != null) {
635 return directory;
636 }
637
638 try {
639 DataSource ds = InfrastructureUtil.getDataSource();
640
641 directory = new JdbcDirectory(ds, _dialect, tableName);
642
643 _jdbcDirectories.put(tableName, directory);
644
645 if (!directory.tableExists()) {
646 directory.create();
647 }
648 }
649 catch (IOException ioe) {
650 throw new RuntimeException(ioe);
651 }
652 catch (UnsupportedOperationException uoe) {
653 if (_log.isWarnEnabled()) {
654 _log.warn(
655 "Database doesn't support the ability to check " +
656 "whether a table exists");
657 }
658
659 _manuallyCreateJdbcDirectory(directory, tableName);
660 }
661 }
662 finally {
663 currentThread.setContextClassLoader(contextClassLoader);
664 }
665
666 return directory;
667 }
668
669 private Directory _getLuceneDirRam(long companyId) {
670 String path = _getPath(companyId);
671
672 Directory directory = _ramDirectories.get(path);
673
674 if (directory == null) {
675 directory = new RAMDirectory();
676
677 _ramDirectories.put(path, directory);
678 }
679
680 return directory;
681 }
682
683 private String _getPath(long companyId) {
684 StringBuilder sb = new StringBuilder();
685
686 sb.append(PropsValues.LUCENE_DIR);
687 sb.append(companyId);
688 sb.append(StringPool.SLASH);
689
690 return sb.toString();
691 }
692
693 private String _getTableName(long companyId) {
694 return _LUCENE_TABLE_PREFIX + companyId;
695 }
696
697 private void _manuallyCreateJdbcDirectory(
698 JdbcDirectory directory, String tableName) {
699
700
702 Connection con = null;
703 ResultSet rs = null;
704
705 try {
706 con = DataAccess.getConnection();
707
708
710 DatabaseMetaData metaData = con.getMetaData();
711
712 rs = metaData.getTables(null, null, tableName, null);
713
714 if (!rs.next()) {
715 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
716
717 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
718
719 Class<?> lockClass = directory.getSettings().getLockClass();
720
721 JdbcLock jdbcLock = null;
722
723 try {
724 jdbcLock = (JdbcLock)lockClass.newInstance();
725 }
726 catch (Exception e) {
727 throw new JdbcStoreException(
728 "Failed to create lock class " + lockClass);
729 }
730
731 jdbcLock.initializeDatabase(directory);
732 }
733 }
734 catch (Exception e) {
735 if (_log.isWarnEnabled()) {
736 _log.warn("Could not create " + tableName);
737 }
738 }
739 finally {
740 DataAccess.cleanUp(con, null, rs);
741 }
742 }
743
744 private static final String _LUCENE_STORE_TYPE_FILE = "file";
745
746 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
747
748 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
749
750 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
751
752 private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
753
754 private static LuceneUtil _instance = new LuceneUtil();
755
756 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
757 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
758 private Dialect _dialect;
759 private Map<String, Directory> _jdbcDirectories =
760 new ConcurrentHashMap<String, Directory>();
761 private Map<String, Directory> _ramDirectories =
762 new ConcurrentHashMap<String, Directory>();
763
764 }