1
19
20 package com.liferay.portal.search.lucene;
21
22 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
23 import com.liferay.portal.kernel.log.Log;
24 import com.liferay.portal.kernel.log.LogFactoryUtil;
25 import com.liferay.portal.kernel.search.SearchEngineUtil;
26 import com.liferay.portal.kernel.util.FileUtil;
27 import com.liferay.portal.kernel.util.InfrastructureUtil;
28 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
29 import com.liferay.portal.kernel.util.StringPool;
30 import com.liferay.portal.kernel.util.StringUtil;
31 import com.liferay.portal.kernel.util.Validator;
32 import com.liferay.portal.util.PropsKeys;
33 import com.liferay.portal.util.PropsUtil;
34 import com.liferay.portal.util.PropsValues;
35 import com.liferay.util.lucene.KeywordsUtil;
36
37 import java.io.IOException;
38
39 import java.sql.Connection;
40 import java.sql.DatabaseMetaData;
41 import java.sql.ResultSet;
42 import java.sql.Statement;
43
44 import java.util.Date;
45 import java.util.Map;
46 import java.util.concurrent.ConcurrentHashMap;
47
48 import javax.sql.DataSource;
49
50 import org.apache.lucene.analysis.Analyzer;
51 import org.apache.lucene.analysis.WhitespaceAnalyzer;
52 import org.apache.lucene.document.Document;
53 import org.apache.lucene.index.IndexReader;
54 import org.apache.lucene.index.IndexWriter;
55 import org.apache.lucene.index.Term;
56 import org.apache.lucene.queryParser.ParseException;
57 import org.apache.lucene.queryParser.QueryParser;
58 import org.apache.lucene.search.BooleanClause;
59 import org.apache.lucene.search.BooleanQuery;
60 import org.apache.lucene.search.IndexSearcher;
61 import org.apache.lucene.search.Query;
62 import org.apache.lucene.search.TermQuery;
63 import org.apache.lucene.search.WildcardQuery;
64 import org.apache.lucene.store.Directory;
65 import org.apache.lucene.store.FSDirectory;
66 import org.apache.lucene.store.RAMDirectory;
67 import org.apache.lucene.store.jdbc.JdbcDirectory;
68 import org.apache.lucene.store.jdbc.JdbcStoreException;
69 import org.apache.lucene.store.jdbc.dialect.Dialect;
70 import org.apache.lucene.store.jdbc.lock.JdbcLock;
71 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
72
73
80 public class LuceneUtil {
81
82 public static void acquireLock(long companyId) {
83 try {
84 _instance._sharedWriter.acquireLock(companyId, true);
85 }
86 catch (InterruptedException ie) {
87 _log.error(ie);
88 }
89 }
90
91 public static void addDate(Document doc, String field, Date value) {
92 doc.add(LuceneFields.getDate(field, value));
93 }
94
95 public static void addExactTerm(
96 BooleanQuery booleanQuery, String field, boolean value) {
97
98 addExactTerm(booleanQuery, field, String.valueOf(value));
99 }
100
101 public static void addExactTerm(
102 BooleanQuery booleanQuery, String field, Boolean value) {
103
104 addExactTerm(booleanQuery, field, String.valueOf(value));
105 }
106
107 public static void addExactTerm(
108 BooleanQuery booleanQuery, String field, double value) {
109
110 addExactTerm(booleanQuery, field, String.valueOf(value));
111 }
112
113 public static void addExactTerm(
114 BooleanQuery booleanQuery, String field, Double value) {
115
116 addExactTerm(booleanQuery, field, String.valueOf(value));
117 }
118
119 public static void addExactTerm(
120 BooleanQuery booleanQuery, String field, int value) {
121
122 addExactTerm(booleanQuery, field, String.valueOf(value));
123 }
124
125 public static void addExactTerm(
126 BooleanQuery booleanQuery, String field, Integer value) {
127
128 addExactTerm(booleanQuery, field, String.valueOf(value));
129 }
130
131 public static void addExactTerm(
132 BooleanQuery booleanQuery, String field, long value) {
133
134 addExactTerm(booleanQuery, field, String.valueOf(value));
135 }
136
137 public static void addExactTerm(
138 BooleanQuery booleanQuery, String field, Long value) {
139
140 addExactTerm(booleanQuery, field, String.valueOf(value));
141 }
142
143 public static void addExactTerm(
144 BooleanQuery booleanQuery, String field, short value) {
145
146 addExactTerm(booleanQuery, field, String.valueOf(value));
147 }
148
149 public static void addExactTerm(
150 BooleanQuery booleanQuery, String field, Short value) {
151
152 addExactTerm(booleanQuery, field, String.valueOf(value));
153 }
154
155 public static void addExactTerm(
156 BooleanQuery booleanQuery, String field, String value) {
157
158
160 Query query = new TermQuery(new Term(field, value));
161
162 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
163 }
164
165 public static void addRequiredTerm(
166 BooleanQuery booleanQuery, String field, boolean value) {
167
168 addRequiredTerm(booleanQuery, field, String.valueOf(value));
169 }
170
171 public static void addRequiredTerm(
172 BooleanQuery booleanQuery, String field, Boolean value) {
173
174 addRequiredTerm(booleanQuery, field, String.valueOf(value));
175 }
176
177 public static void addRequiredTerm(
178 BooleanQuery booleanQuery, String field, double value) {
179
180 addRequiredTerm(booleanQuery, field, String.valueOf(value));
181 }
182
183 public static void addRequiredTerm(
184 BooleanQuery booleanQuery, String field, Double value) {
185
186 addRequiredTerm(booleanQuery, field, String.valueOf(value));
187 }
188
189 public static void addRequiredTerm(
190 BooleanQuery booleanQuery, String field, int value) {
191
192 addRequiredTerm(booleanQuery, field, String.valueOf(value));
193 }
194
195 public static void addRequiredTerm(
196 BooleanQuery booleanQuery, String field, Integer value) {
197
198 addRequiredTerm(booleanQuery, field, String.valueOf(value));
199 }
200
201 public static void addRequiredTerm(
202 BooleanQuery booleanQuery, String field, long value) {
203
204 addRequiredTerm(booleanQuery, field, String.valueOf(value));
205 }
206
207 public static void addRequiredTerm(
208 BooleanQuery booleanQuery, String field, Long value) {
209
210 addRequiredTerm(booleanQuery, field, String.valueOf(value));
211 }
212
213 public static void addRequiredTerm(
214 BooleanQuery booleanQuery, String field, short value) {
215
216 addRequiredTerm(booleanQuery, field, String.valueOf(value));
217 }
218
219 public static void addRequiredTerm(
220 BooleanQuery booleanQuery, String field, Short value) {
221
222 addRequiredTerm(booleanQuery, field, String.valueOf(value));
223 }
224
225 public static void addRequiredTerm(
226 BooleanQuery booleanQuery, String field, String value) {
227
228 addRequiredTerm(booleanQuery, field, value, false);
229 }
230
231 public static void addRequiredTerm(
232 BooleanQuery booleanQuery, String field, String value, boolean like) {
233
234 if (like) {
235 value = StringUtil.replace(
236 value, StringPool.PERCENT, StringPool.STAR);
237
238 value = value.toLowerCase();
239
240 WildcardQuery wildcardQuery = new WildcardQuery(
241 new Term(field, value));
242
243 booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
244 }
245 else {
246
248 Term term = new Term(field, value);
249 TermQuery termQuery = new TermQuery(term);
250
251 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
252 }
253 }
254
255 public static void addTerm(
256 BooleanQuery booleanQuery, String field, long value)
257 throws ParseException {
258
259 addTerm(booleanQuery, field, String.valueOf(value));
260 }
261
262 public static void addTerm(
263 BooleanQuery booleanQuery, String field, String value)
264 throws ParseException {
265
266 addTerm(booleanQuery, field, value, false);
267 }
268
269 public static void addTerm(
270 BooleanQuery booleanQuery, String field, String value,
271 boolean like)
272 throws ParseException {
273
274 if (Validator.isNull(value)) {
275 return;
276 }
277
278 if (like) {
279 value = value.toLowerCase();
280
281 StringBuilder sb = new StringBuilder();
282
283 sb.append(value);
284 sb.append(StringPool.STAR);
285
286 WildcardQuery wildcardQuery = new WildcardQuery(
287 new Term(field, sb.toString()));
288
289 booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
290 }
291 else {
292 QueryParser queryParser = new QueryParser(
293 field, LuceneUtil.getAnalyzer());
294
295 try {
296 Query query = queryParser.parse(value);
297
298 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
299 }
300 catch (ParseException pe) {
301 if (_log.isDebugEnabled()) {
302 _log.debug(
303 "ParseException thrown, reverting to literal search",
304 pe);
305 }
306
307 value = KeywordsUtil.escape(value);
308
309 Query query = queryParser.parse(value);
310
311 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
312 }
313 }
314 }
315
316 public static void checkLuceneDir(long companyId) {
317 if (SearchEngineUtil.isIndexReadOnly()) {
318 return;
319 }
320
321 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
322
323 try {
324
325
327 if (luceneDir.fileExists("write.lock")) {
328 luceneDir.deleteFile("write.lock");
329 }
330 }
331 catch (IOException ioe) {
332 _log.error("Unable to clear write lock", ioe);
333 }
334
335 IndexWriter writer = null;
336
337
340 try {
341 if (luceneDir.fileExists("segments.gen")) {
342 writer = new IndexWriter(
343 luceneDir, LuceneUtil.getAnalyzer(), false);
344 }
345 else {
346 writer = new IndexWriter(
347 luceneDir, LuceneUtil.getAnalyzer(), true);
348 }
349 }
350 catch (IOException ioe) {
351 _log.error("Check Lucene directory failed for " + companyId, ioe);
352 }
353 finally {
354 if (writer != null) {
355 try {
356 writer.close();
357 }
358 catch (IOException ioe) {
359 _log.error(ioe);
360 }
361 }
362 }
363 }
364
365 public static void delete(long companyId) {
366 _instance._delete(companyId);
367 }
368
369 public static void deleteDocuments(long companyId, Term term)
370 throws IOException {
371
372 try {
373 _instance._sharedWriter.deleteDocuments(companyId, term);
374 }
375 catch (InterruptedException ie) {
376 _log.error(ie);
377 }
378 }
379
380 public static Analyzer getAnalyzer() {
381 return _instance._getAnalyzer();
382 }
383
384 public static FSDirectory getDirectory(String path, boolean create)
385 throws IOException {
386
387 return FSDirectory.getDirectory(path, false);
388 }
389
390 public static Directory getLuceneDir(long companyId) {
391 return _instance._getLuceneDir(companyId);
392 }
393
394 public static IndexReader getReader(long companyId) throws IOException {
395 return IndexReader.open(getLuceneDir(companyId));
396 }
397
398 public static IndexSearcher getSearcher(long companyId)
399 throws IOException {
400
401 return new IndexSearcher(getLuceneDir(companyId));
402 }
403
404 public static IndexWriter getWriter(long companyId) throws IOException {
405 return getWriter(companyId, false);
406 }
407
408 public static IndexWriter getWriter(long companyId, boolean create)
409 throws IOException {
410
411 return _instance._sharedWriter.getWriter(companyId, create);
412 }
413
414 public static void releaseLock(long companyId) {
415 _instance._sharedWriter.releaseLock(companyId);
416 }
417
418 public static void write(long companyId) {
419 _instance._sharedWriter.write(companyId);
420 }
421
422 public static void write(IndexWriter writer) throws IOException {
423 _instance._sharedWriter.write(writer);
424 }
425
426 private LuceneUtil() {
427 String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
428
429 if (Validator.isNotNull(analyzerName)) {
430 try {
431 _analyzerClass = Class.forName(analyzerName);
432 }
433 catch (Exception e) {
434 _log.error(e);
435 }
436 }
437
438
440 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
441 Connection con = null;
442
443 try {
444 con = DataAccess.getConnection();
445
446 String url = con.getMetaData().getURL();
447
448 int x = url.indexOf(":");
449 int y = url.indexOf(":", x + 1);
450
451 String urlPrefix = url.substring(x + 1, y);
452
453 String dialectClass = PropsUtil.get(
454 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
455
456 if (dialectClass != null) {
457 if (_log.isDebugEnabled()) {
458 _log.debug("JDBC class implementation " + dialectClass);
459 }
460 }
461 else {
462 if (_log.isDebugEnabled()) {
463 _log.debug("JDBC class implementation is null");
464 }
465 }
466
467 if (dialectClass != null) {
468 _dialect =
469 (Dialect)Class.forName(dialectClass).newInstance();
470 }
471 }
472 catch (Exception e) {
473 _log.error(e);
474 }
475 finally{
476 DataAccess.cleanUp(con);
477 }
478
479 if (_dialect == null) {
480 _log.error("No JDBC dialect found");
481 }
482 }
483 }
484
485 public void _delete(long companyId) {
486 if (SearchEngineUtil.isIndexReadOnly()) {
487 return;
488 }
489
490 if (_log.isDebugEnabled()) {
491 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
492 }
493
494 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
495 _deleteFile(companyId);
496 }
497 else if (PropsValues.LUCENE_STORE_TYPE.equals(
498 _LUCENE_STORE_TYPE_JDBC)) {
499
500 _deleteJdbc(companyId);
501 }
502 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
503 _deleteRam(companyId);
504 }
505 else {
506 throw new RuntimeException(
507 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
508 }
509 }
510
511 private void _deleteFile(long companyId) {
512 String path = _getPath(companyId);
513
514 try {
515 Directory directory = getDirectory(path, false);
516
517 directory.close();
518 }
519 catch (Exception e) {
520 if (_log.isWarnEnabled()) {
521 _log.warn("Could not close directory " + path);
522 }
523 }
524
525 FileUtil.deltree(path);
526 }
527
528 private void _deleteJdbc(long companyId) {
529 String tableName = _getTableName(companyId);
530
531 try {
532 Directory directory = _jdbcDirectories.remove(tableName);
533
534 if (directory != null) {
535 directory.close();
536 }
537 }
538 catch (Exception e) {
539 if (_log.isWarnEnabled()) {
540 _log.warn("Could not close directory " + tableName);
541 }
542 }
543
544 Connection con = null;
545 Statement s = null;
546
547 try {
548 con = DataAccess.getConnection();
549
550 s = con.createStatement();
551
552 s.executeUpdate("DELETE FROM " + tableName);
553 }
554 catch (Exception e) {
555 if (_log.isWarnEnabled()) {
556 _log.warn("Could not truncate " + tableName);
557 }
558 }
559 finally {
560 DataAccess.cleanUp(con, s);
561 }
562 }
563
564 private void _deleteRam(long companyId) {
565 }
566
567 private Analyzer _getAnalyzer() {
568 try {
569 return (Analyzer)_analyzerClass.newInstance();
570 }
571 catch (Exception e) {
572 throw new RuntimeException(e);
573 }
574 }
575
576 private Directory _getLuceneDir(long companyId) {
577 if (_log.isDebugEnabled()) {
578 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
579 }
580
581 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
582 return _getLuceneDirFile(companyId);
583 }
584 else if (PropsValues.LUCENE_STORE_TYPE.equals(
585 _LUCENE_STORE_TYPE_JDBC)) {
586
587 return _getLuceneDirJdbc(companyId);
588 }
589 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
590 return _getLuceneDirRam(companyId);
591 }
592 else {
593 throw new RuntimeException(
594 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
595 }
596 }
597
598 private Directory _getLuceneDirFile(long companyId) {
599 Directory directory = null;
600
601 String path = _getPath(companyId);
602
603 try {
604 directory = getDirectory(path, false);
605 }
606 catch (IOException ioe1) {
607 try {
608 if (directory != null) {
609 directory.close();
610 }
611
612 directory = getDirectory(path, true);
613 }
614 catch (IOException ioe2) {
615 throw new RuntimeException(ioe2);
616 }
617 }
618
619 return directory;
620 }
621
622 private Directory _getLuceneDirJdbc(long companyId) {
623 JdbcDirectory directory = null;
624
625 Thread currentThread = Thread.currentThread();
626
627 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
628
629 try {
630 currentThread.setContextClassLoader(
631 PortalClassLoaderUtil.getClassLoader());
632
633 String tableName = _getTableName(companyId);
634
635 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
636
637 if (directory != null) {
638 return directory;
639 }
640
641 try {
642 DataSource ds = InfrastructureUtil.getDataSource();
643
644 directory = new JdbcDirectory(ds, _dialect, tableName);
645
646 _jdbcDirectories.put(tableName, directory);
647
648 if (!directory.tableExists()) {
649 directory.create();
650 }
651 }
652 catch (IOException ioe) {
653 throw new RuntimeException(ioe);
654 }
655 catch (UnsupportedOperationException uoe) {
656 if (_log.isWarnEnabled()) {
657 _log.warn(
658 "Database doesn't support the ability to check " +
659 "whether a table exists");
660 }
661
662 _manuallyCreateJdbcDirectory(directory, tableName);
663 }
664 }
665 finally {
666 currentThread.setContextClassLoader(contextClassLoader);
667 }
668
669 return directory;
670 }
671
672 private Directory _getLuceneDirRam(long companyId) {
673 String path = _getPath(companyId);
674
675 Directory directory = _ramDirectories.get(path);
676
677 if (directory == null) {
678 directory = new RAMDirectory();
679
680 _ramDirectories.put(path, directory);
681 }
682
683 return directory;
684 }
685
686 private String _getPath(long companyId) {
687 StringBuilder sb = new StringBuilder();
688
689 sb.append(PropsValues.LUCENE_DIR);
690 sb.append(companyId);
691 sb.append(StringPool.SLASH);
692
693 return sb.toString();
694 }
695
696 private String _getTableName(long companyId) {
697 return _LUCENE_TABLE_PREFIX + companyId;
698 }
699
700 private void _manuallyCreateJdbcDirectory(
701 JdbcDirectory directory, String tableName) {
702
703
705 Connection con = null;
706 ResultSet rs = null;
707
708 try {
709 con = DataAccess.getConnection();
710
711
713 DatabaseMetaData metaData = con.getMetaData();
714
715 rs = metaData.getTables(null, null, tableName, null);
716
717 if (!rs.next()) {
718 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
719
720 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
721
722 Class<?> lockClass = directory.getSettings().getLockClass();
723
724 JdbcLock jdbcLock = null;
725
726 try {
727 jdbcLock = (JdbcLock)lockClass.newInstance();
728 }
729 catch (Exception e) {
730 throw new JdbcStoreException(
731 "Failed to create lock class " + lockClass);
732 }
733
734 jdbcLock.initializeDatabase(directory);
735 }
736 }
737 catch (Exception e) {
738 if (_log.isWarnEnabled()) {
739 _log.warn("Could not create " + tableName);
740 }
741 }
742 finally {
743 DataAccess.cleanUp(con, null, rs);
744 }
745 }
746
747 private static final String _LUCENE_STORE_TYPE_FILE = "file";
748
749 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
750
751 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
752
753 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
754
755 private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
756
757 private static LuceneUtil _instance = new LuceneUtil();
758
759 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
760 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
761 private Dialect _dialect;
762 private Map<String, Directory> _jdbcDirectories =
763 new ConcurrentHashMap<String, Directory>();
764 private Map<String, Directory> _ramDirectories =
765 new ConcurrentHashMap<String, Directory>();
766
767 }