1
22
23 package com.liferay.portal.lucene;
24
25 import com.liferay.portal.SystemException;
26 import com.liferay.portal.kernel.search.Hits;
27 import com.liferay.portal.kernel.util.GetterUtil;
28 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
29 import com.liferay.portal.kernel.util.StringMaker;
30 import com.liferay.portal.kernel.util.StringPool;
31 import com.liferay.portal.kernel.util.Validator;
32 import com.liferay.portal.spring.hibernate.HibernateUtil;
33 import com.liferay.portal.util.PropsUtil;
34 import com.liferay.portal.util.PropsValues;
35 import com.liferay.util.FileUtil;
36 import com.liferay.util.dao.DataAccess;
37 import com.liferay.util.lucene.HitsImpl;
38 import com.liferay.util.lucene.KeywordsUtil;
39
40 import java.io.IOException;
41
42 import java.sql.Connection;
43 import java.sql.DatabaseMetaData;
44 import java.sql.ResultSet;
45 import java.sql.Statement;
46
47 import java.util.Date;
48 import java.util.Map;
49 import java.util.concurrent.ConcurrentHashMap;
50 import java.util.regex.Pattern;
51
52 import javax.sql.DataSource;
53
54 import org.apache.commons.logging.Log;
55 import org.apache.commons.logging.LogFactory;
56 import org.apache.lucene.analysis.Analyzer;
57 import org.apache.lucene.analysis.WhitespaceAnalyzer;
58 import org.apache.lucene.document.Document;
59 import org.apache.lucene.index.IndexReader;
60 import org.apache.lucene.index.IndexWriter;
61 import org.apache.lucene.index.Term;
62 import org.apache.lucene.queryParser.ParseException;
63 import org.apache.lucene.queryParser.QueryParser;
64 import org.apache.lucene.search.BooleanClause;
65 import org.apache.lucene.search.BooleanQuery;
66 import org.apache.lucene.search.IndexSearcher;
67 import org.apache.lucene.search.Query;
68 import org.apache.lucene.search.Searcher;
69 import org.apache.lucene.search.TermQuery;
70 import org.apache.lucene.store.Directory;
71 import org.apache.lucene.store.FSDirectory;
72 import org.apache.lucene.store.RAMDirectory;
73 import org.apache.lucene.store.jdbc.JdbcDirectory;
74 import org.apache.lucene.store.jdbc.JdbcStoreException;
75 import org.apache.lucene.store.jdbc.dialect.Dialect;
76 import org.apache.lucene.store.jdbc.lock.JdbcLock;
77 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
78
79
86 public class LuceneUtil {
87
88 public static boolean INDEX_READ_ONLY = GetterUtil.getBoolean(
89 PropsUtil.get(PropsUtil.INDEX_READ_ONLY));
90
91 public static final Pattern TERM_END_PATTERN =
92 Pattern.compile("(\\w{4,}?)\\b");
93
94 public static void acquireLock(long companyId) {
95 try {
96 _instance._sharedWriter.acquireLock(companyId, true);
97 }
98 catch (InterruptedException ie) {
99 _log.error(ie);
100 }
101 }
102
103 public static void addDate(Document doc, String field, Date value) {
104 doc.add(LuceneFields.getDate(field, value));
105 }
106
107 public static void addExactTerm(
108 BooleanQuery booleanQuery, String field, long value)
109 throws ParseException {
110
111 addExactTerm(booleanQuery, field, String.valueOf(value));
112 }
113
114 public static void addExactTerm(
115 BooleanQuery booleanQuery, String field, String text)
116 throws ParseException {
117
118
120 Query query = new TermQuery(new Term(field, text));
121
122 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
123 }
124
125 public static void addKeyword(Document doc, String field, double value) {
126 addKeyword(doc, field, String.valueOf(value));
127 }
128
129 public static void addKeyword(Document doc, String field, long value) {
130 addKeyword(doc, field, String.valueOf(value));
131 }
132
133 public static void addKeyword(Document doc, String field, String value) {
134 if (Validator.isNotNull(value)) {
135 doc.add(LuceneFields.getKeyword(field, value));
136 }
137 }
138
139 public static void addKeyword(Document doc, String field, String[] values) {
140 if (values == null) {
141 return;
142 }
143
144 for (int i = 0; i < values.length; i++) {
145 addKeyword(doc, field, values[i]);
146 }
147 }
148
149 public static void addRequiredTerm(
150 BooleanQuery booleanQuery, String field, long value) {
151
152 addRequiredTerm(booleanQuery, field, String.valueOf(value));
153 }
154
155 public static void addRequiredTerm(
156 BooleanQuery booleanQuery, String field, String text) {
157
158
160 Term term = new Term(field, text);
161 TermQuery termQuery = new TermQuery(term);
162
163 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
164 }
165
166 public static void addModifiedDate(Document doc) {
167 doc.add(LuceneFields.getDate(LuceneFields.MODIFIED));
168 }
169
170 public static void addTerm(
171 BooleanQuery booleanQuery, String field, long value)
172 throws ParseException {
173
174 addTerm(booleanQuery, field, String.valueOf(value));
175 }
176
177 public static void addTerm(
178 BooleanQuery booleanQuery, String field, String text)
179 throws ParseException {
180
181 if (Validator.isNotNull(text)) {
182
184
186
188
190
192
194
199 QueryParser queryParser = new QueryParser(
200 field, LuceneUtil.getAnalyzer());
201
202 try {
203 Query query = queryParser.parse(text);
204
205 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
206 }
207 catch(ParseException pe) {
208 if (_log.isDebugEnabled()) {
209 _log.debug(
210 "ParseException thrown, reverting to literal search",
211 pe);
212 }
213
214 text = KeywordsUtil.escape(text);
215
216 Query query = queryParser.parse(text);
217
218 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
219 }
220 }
221 }
222
223 public static void addText(Document doc, String field, long value) {
224 addText(doc, field, String.valueOf(value));
225 }
226
227 public static void addText(Document doc, String field, String value) {
228 if (Validator.isNotNull(value)) {
229 doc.add(LuceneFields.getText(field, value));
230 }
231 }
232
233 public static void checkLuceneDir(long companyId) {
234 if (INDEX_READ_ONLY) {
235 return;
236 }
237
238 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
239
240 IndexWriter writer = null;
241
242
245 try {
246 if (luceneDir.fileExists("segments.gen")) {
247 writer = new IndexWriter(
248 luceneDir, LuceneUtil.getAnalyzer(), false);
249 }
250 else {
251 writer = new IndexWriter(
252 luceneDir, LuceneUtil.getAnalyzer(), true);
253 }
254 }
255 catch (IOException ioe) {
256 _log.error(ioe);
257 }
258 finally {
259 if (writer != null) {
260 try {
261 writer.close();
262 }
263 catch (IOException ioe) {
264 _log.error(ioe);
265 }
266 }
267 }
268 }
269
270 public static void closeSearcher(Searcher searcher) {
271 try {
272 if (searcher != null){
273 searcher.close();
274 }
275 }
276 catch (Exception e) {
277 }
278 }
279
280 public static Hits closeSearcher(
281 Searcher searcher, String keywords, Exception e)
282 throws SystemException {
283
284 closeSearcher(searcher);
285
286 if (e instanceof BooleanQuery.TooManyClauses ||
287 e instanceof ParseException) {
288
289 _log.error("Parsing keywords " + keywords, e);
290
291 return new HitsImpl();
292 }
293 else {
294 throw new SystemException(e);
295 }
296 }
297
298 public static void delete(long companyId) {
299 _instance._delete(companyId);
300 }
301
302 public static void deleteDocuments(long companyId, Term term)
303 throws IOException {
304
305 try {
306 _instance._sharedWriter.deleteDocuments(companyId, term);
307 }
308 catch (InterruptedException ie) {
309 _log.error(ie);
310 }
311 }
312
313 public static Analyzer getAnalyzer() {
314 return _instance._getAnalyzer();
315 }
316
317 public static Directory getLuceneDir(long companyId) {
318 return _instance._getLuceneDir(companyId);
319 }
320
321 public static IndexReader getReader(long companyId) throws IOException {
322 return IndexReader.open(getLuceneDir(companyId));
323 }
324
325 public static IndexSearcher getSearcher(long companyId)
326 throws IOException {
327
328 return new IndexSearcher(getLuceneDir(companyId));
329 }
330
331 public static IndexWriter getWriter(long companyId) throws IOException {
332 return getWriter(companyId, false);
333 }
334
335 public static IndexWriter getWriter(long companyId, boolean create)
336 throws IOException {
337
338 return _instance._sharedWriter.getWriter(companyId, create);
339 }
340
341 public static void releaseLock(long companyId) {
342 _instance._sharedWriter.releaseLock(companyId);
343 }
344
345 public static void write(long companyId) throws IOException {
346 _instance._sharedWriter.write(companyId);
347 }
348
349 public static void write(IndexWriter writer) throws IOException {
350 _instance._sharedWriter.write(writer);
351 }
352
353 private LuceneUtil() {
354 String analyzerName = PropsUtil.get(PropsUtil.LUCENE_ANALYZER);
355
356 if (Validator.isNotNull(analyzerName)) {
357 try {
358 _analyzerClass = Class.forName(analyzerName);
359 }
360 catch (Exception e) {
361 _log.error(e);
362 }
363 }
364
365
367 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
368 Connection con = null;
369
370 try {
371 con = HibernateUtil.getConnection();
372
373 String url = con.getMetaData().getURL();
374
375 int x = url.indexOf(":");
376 int y = url.indexOf(":", x + 1);
377
378 String urlPrefix = url.substring(x + 1, y);
379
380 String dialectClass = PropsUtil.get(
381 PropsUtil.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
382
383 if (dialectClass != null) {
384 if (_log.isDebugEnabled()) {
385 _log.debug("JDBC class implementation " + dialectClass);
386 }
387 }
388 else {
389 if (_log.isDebugEnabled()) {
390 _log.debug("JDBC class implementation is null");
391 }
392 }
393
394 if (dialectClass != null) {
395 _dialect =
396 (Dialect)Class.forName(dialectClass).newInstance();
397 }
398 }
399 catch (Exception e) {
400 _log.error(e);
401 }
402 finally{
403 DataAccess.cleanUp(con);
404 }
405
406 if (_dialect == null) {
407 _log.error("No JDBC dialect found");
408 }
409 }
410 }
411
412 public void _delete(long companyId) {
413 if (LuceneUtil.INDEX_READ_ONLY) {
414 return;
415 }
416
417 if (_log.isDebugEnabled()) {
418 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
419 }
420
421 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
422 _deleteFile(companyId);
423 }
424 else if (PropsValues.LUCENE_STORE_TYPE.equals(
425 _LUCENE_STORE_TYPE_JDBC)) {
426
427 _deleteJdbc(companyId);
428 }
429 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
430 _deleteRam(companyId);
431 }
432 else {
433 throw new RuntimeException(
434 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
435 }
436 }
437
438 private void _deleteFile(long companyId) {
439 String path = _getPath(companyId);
440
441 try {
442 Directory directory = FSDirectory.getDirectory(path, false);
443
444 directory.close();
445 }
446 catch (Exception e) {
447 if (_log.isWarnEnabled()) {
448 _log.warn("Could not close directory " + path);
449 }
450 }
451
452 FileUtil.deltree(path);
453 }
454
455 private void _deleteJdbc(long companyId) {
456 String tableName = _getTableName(companyId);
457
458 try {
459 Directory directory = _jdbcDirectories.remove(tableName);
460
461 if (directory != null) {
462 directory.close();
463 }
464 }
465 catch (Exception e) {
466 if (_log.isWarnEnabled()) {
467 _log.warn("Could not close directory " + tableName);
468 }
469 }
470
471 Connection con = null;
472 Statement s = null;
473
474 try {
475 con = HibernateUtil.getConnection();
476
477 s = con.createStatement();
478
479 s.executeUpdate("DELETE FROM " + tableName);
480 }
481 catch (Exception e) {
482 if (_log.isWarnEnabled()) {
483 _log.warn("Could not truncate " + tableName);
484 }
485 }
486 finally {
487 DataAccess.cleanUp(con, s);
488 }
489 }
490
491 private void _deleteRam(long companyId) {
492 }
493
494 private Analyzer _getAnalyzer() {
495 try {
496 return (Analyzer)_analyzerClass.newInstance();
497 }
498 catch (Exception e) {
499 throw new RuntimeException(e);
500 }
501 }
502
503 private Directory _getLuceneDir(long companyId) {
504 if (_log.isDebugEnabled()) {
505 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
506 }
507
508 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
509 return _getLuceneDirFile(companyId);
510 }
511 else if (PropsValues.LUCENE_STORE_TYPE.equals(
512 _LUCENE_STORE_TYPE_JDBC)) {
513
514 return _getLuceneDirJdbc(companyId);
515 }
516 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
517 return _getLuceneDirRam(companyId);
518 }
519 else {
520 throw new RuntimeException(
521 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
522 }
523 }
524
525 private Directory _getLuceneDirFile(long companyId) {
526 Directory directory = null;
527
528 String path = _getPath(companyId);
529
530 try {
531 directory = FSDirectory.getDirectory(path, false);
532 }
533 catch (IOException ioe1) {
534 try {
535 if (directory != null) {
536 directory.close();
537 }
538
539 directory = FSDirectory.getDirectory(path, true);
540 }
541 catch (IOException ioe2) {
542 throw new RuntimeException(ioe2);
543 }
544 }
545
546 return directory;
547 }
548
549 private Directory _getLuceneDirJdbc(long companyId) {
550 JdbcDirectory directory = null;
551
552 ClassLoader contextClassLoader =
553 Thread.currentThread().getContextClassLoader();
554
555 try {
556 Thread.currentThread().setContextClassLoader(
557 PortalClassLoaderUtil.getClassLoader());
558
559 String tableName = _getTableName(companyId);
560
561 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
562
563 if (directory != null) {
564 return directory;
565 }
566
567 try {
568 DataSource ds = HibernateUtil.getDataSource();
569
570 directory = new JdbcDirectory(ds, _dialect, tableName);
571
572 _jdbcDirectories.put(tableName, directory);
573
574 if (!directory.tableExists()) {
575 directory.create();
576 }
577 }
578 catch (IOException ioe) {
579 throw new RuntimeException(ioe);
580 }
581 catch (UnsupportedOperationException uoe) {
582 if (_log.isWarnEnabled()) {
583 _log.warn(
584 "Database doesn't support the ability to check " +
585 "whether a table exists");
586 }
587
588 _manuallyCreateJdbcDirectory(directory, tableName);
589 }
590 }
591 finally {
592 Thread.currentThread().setContextClassLoader(contextClassLoader);
593 }
594
595 return directory;
596 }
597
598 private Directory _getLuceneDirRam(long companyId) {
599 String path = _getPath(companyId);
600
601 Directory directory = _ramDirectories.get(path);
602
603 if (directory == null) {
604 directory = new RAMDirectory();
605
606 _ramDirectories.put(path, directory);
607 }
608
609 return directory;
610 }
611
612 private String _getPath(long companyId) {
613 StringMaker sm = new StringMaker();
614
615 sm.append(PropsValues.LUCENE_DIR);
616 sm.append(companyId);
617 sm.append(StringPool.SLASH);
618
619 return sm.toString();
620 }
621
622 private String _getTableName(long companyId) {
623 return _LUCENE_TABLE_PREFIX + companyId;
624 }
625
626 private void _manuallyCreateJdbcDirectory(
627 JdbcDirectory directory, String tableName) {
628
629
631 Connection con = null;
632 ResultSet rs = null;
633
634 try {
635 con = HibernateUtil.getConnection();
636
637
639 DatabaseMetaData metaData = con.getMetaData();
640
641 rs = metaData.getTables(null, null, tableName, null);
642
643 if (!rs.next()) {
644 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
645
646 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
647
648 Class<?> lockClass = directory.getSettings().getLockClass();
649
650 JdbcLock jdbcLock = null;
651
652 try {
653 jdbcLock = (JdbcLock)lockClass.newInstance();
654 }
655 catch (Exception e) {
656 throw new JdbcStoreException(
657 "Failed to create lock class " + lockClass);
658 }
659
660 jdbcLock.initializeDatabase(directory);
661 }
662 }
663 catch (Exception e) {
664 if (_log.isWarnEnabled()) {
665 _log.warn("Could not create " + tableName);
666 }
667 }
668 finally {
669 DataAccess.cleanUp(con, null, rs);
670 }
671 }
672
673 private static final String _LUCENE_STORE_TYPE_FILE = "file";
674
675 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
676
677 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
678
679 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
680
681 private static Log _log = LogFactory.getLog(LuceneUtil.class);
682
683 private static LuceneUtil _instance = new LuceneUtil();
684
685 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
686 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
687 private Dialect _dialect;
688 private Map<String, Directory> _jdbcDirectories =
689 new ConcurrentHashMap<String, Directory>();
690 private Map<String, Directory> _ramDirectories =
691 new ConcurrentHashMap<String, Directory>();
692
693 }