1
22
23 package com.liferay.portal.search.lucene;
24
25 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26 import com.liferay.portal.kernel.jndi.PortalJNDIUtil;
27 import com.liferay.portal.kernel.search.Field;
28 import com.liferay.portal.kernel.util.FileUtil;
29 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
30 import com.liferay.portal.kernel.util.StringPool;
31 import com.liferay.portal.kernel.util.Validator;
32 import com.liferay.portal.util.PropsKeys;
33 import com.liferay.portal.util.PropsUtil;
34 import com.liferay.portal.util.PropsValues;
35 import com.liferay.util.lucene.KeywordsUtil;
36
37 import java.io.IOException;
38
39 import java.sql.Connection;
40 import java.sql.DatabaseMetaData;
41 import java.sql.ResultSet;
42 import java.sql.Statement;
43
44 import java.util.Date;
45 import java.util.Map;
46 import java.util.concurrent.ConcurrentHashMap;
47
48 import javax.naming.NamingException;
49
50 import javax.sql.DataSource;
51
52 import org.apache.commons.logging.Log;
53 import org.apache.commons.logging.LogFactory;
54 import org.apache.lucene.analysis.Analyzer;
55 import org.apache.lucene.analysis.WhitespaceAnalyzer;
56 import org.apache.lucene.document.Document;
57 import org.apache.lucene.index.IndexReader;
58 import org.apache.lucene.index.IndexWriter;
59 import org.apache.lucene.index.Term;
60 import org.apache.lucene.queryParser.ParseException;
61 import org.apache.lucene.queryParser.QueryParser;
62 import org.apache.lucene.search.BooleanClause;
63 import org.apache.lucene.search.BooleanQuery;
64 import org.apache.lucene.search.IndexSearcher;
65 import org.apache.lucene.search.Query;
66 import org.apache.lucene.search.TermQuery;
67 import org.apache.lucene.store.Directory;
68 import org.apache.lucene.store.FSDirectory;
69 import org.apache.lucene.store.RAMDirectory;
70 import org.apache.lucene.store.jdbc.JdbcDirectory;
71 import org.apache.lucene.store.jdbc.JdbcStoreException;
72 import org.apache.lucene.store.jdbc.dialect.Dialect;
73 import org.apache.lucene.store.jdbc.lock.JdbcLock;
74 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75
76
83 public class LuceneUtil {
84
85 public static void acquireLock(long companyId) {
86 try {
87 _instance._sharedWriter.acquireLock(companyId, true);
88 }
89 catch (InterruptedException ie) {
90 _log.error(ie);
91 }
92 }
93
94 public static void addDate(Document doc, String field, Date value) {
95 doc.add(LuceneFields.getDate(field, value));
96 }
97
98 public static void addExactTerm(
99 BooleanQuery booleanQuery, String field, long value) {
100
101 addExactTerm(booleanQuery, field, String.valueOf(value));
102 }
103
104 public static void addExactTerm(
105 BooleanQuery booleanQuery, String field, String value) {
106
107
109 Query query = new TermQuery(new Term(field, value));
110
111 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
112 }
113
114 public static void addKeyword(Document doc, String field, double value) {
115 addKeyword(doc, field, String.valueOf(value));
116 }
117
118 public static void addKeyword(Document doc, String field, long value) {
119 addKeyword(doc, field, String.valueOf(value));
120 }
121
122 public static void addKeyword(Document doc, String field, String value) {
123 if (Validator.isNotNull(value)) {
124 doc.add(LuceneFields.getKeyword(field, value));
125 }
126 }
127
128 public static void addKeyword(Document doc, String field, String[] values) {
129 if (values == null) {
130 return;
131 }
132
133 for (int i = 0; i < values.length; i++) {
134 addKeyword(doc, field, values[i]);
135 }
136 }
137
138 public static void addRequiredTerm(
139 BooleanQuery booleanQuery, String field, long value) {
140
141 addRequiredTerm(booleanQuery, field, String.valueOf(value));
142 }
143
144 public static void addRequiredTerm(
145 BooleanQuery booleanQuery, String field, String value) {
146
147
149 Term term = new Term(field, value);
150 TermQuery termQuery = new TermQuery(term);
151
152 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
153 }
154
155 public static void addModifiedDate(Document doc) {
156 doc.add(LuceneFields.getDate(Field.MODIFIED));
157 }
158
159 public static void addTerm(
160 BooleanQuery booleanQuery, String field, long value)
161 throws ParseException {
162
163 addTerm(booleanQuery, field, String.valueOf(value));
164 }
165
166 public static void addTerm(
167 BooleanQuery booleanQuery, String field, String value)
168 throws ParseException {
169
170 if (Validator.isNotNull(value)) {
171 QueryParser queryParser = new QueryParser(
172 field, LuceneUtil.getAnalyzer());
173
174 try {
175 Query query = queryParser.parse(value);
176
177 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
178 }
179 catch (ParseException pe) {
180 if (_log.isDebugEnabled()) {
181 _log.debug(
182 "ParseException thrown, reverting to literal search",
183 pe);
184 }
185
186 value = KeywordsUtil.escape(value);
187
188 Query query = queryParser.parse(value);
189
190 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
191 }
192 }
193 }
194
195 public static void addText(Document doc, String field, long value) {
196 addText(doc, field, String.valueOf(value));
197 }
198
199 public static void addText(Document doc, String field, String value) {
200 if (Validator.isNotNull(value)) {
201 doc.add(LuceneFields.getText(field, value));
202 }
203 }
204
205 public static void checkLuceneDir(long companyId) {
206 if (PropsValues.INDEX_READ_ONLY) {
207 return;
208 }
209
210 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
211
212 try {
213
214
216 if (luceneDir.fileExists("write.lock")) {
217 luceneDir.deleteFile("write.lock");
218 }
219 }
220 catch (IOException ioe) {
221 _log.error("Unable to clear write lock", ioe);
222 }
223
224 IndexWriter writer = null;
225
226
229 try {
230 if (luceneDir.fileExists("segments.gen")) {
231 writer = new IndexWriter(
232 luceneDir, LuceneUtil.getAnalyzer(), false);
233 }
234 else {
235 writer = new IndexWriter(
236 luceneDir, LuceneUtil.getAnalyzer(), true);
237 }
238 }
239 catch (IOException ioe) {
240 _log.error("Check Lucene directory failed for " + companyId, ioe);
241 }
242 finally {
243 if (writer != null) {
244 try {
245 writer.close();
246 }
247 catch (IOException ioe) {
248 _log.error(ioe);
249 }
250 }
251 }
252 }
253
254 public static void delete(long companyId) {
255 _instance._delete(companyId);
256 }
257
258 public static void deleteDocuments(long companyId, Term term)
259 throws IOException {
260
261 try {
262 _instance._sharedWriter.deleteDocuments(companyId, term);
263 }
264 catch (InterruptedException ie) {
265 _log.error(ie);
266 }
267 }
268
269 public static Analyzer getAnalyzer() {
270 return _instance._getAnalyzer();
271 }
272
273 public static Directory getLuceneDir(long companyId) {
274 return _instance._getLuceneDir(companyId);
275 }
276
277 public static IndexReader getReader(long companyId) throws IOException {
278 return IndexReader.open(getLuceneDir(companyId));
279 }
280
281 public static IndexSearcher getSearcher(long companyId)
282 throws IOException {
283
284 return new IndexSearcher(getLuceneDir(companyId));
285 }
286
287 public static IndexWriter getWriter(long companyId) throws IOException {
288 return getWriter(companyId, false);
289 }
290
291 public static IndexWriter getWriter(long companyId, boolean create)
292 throws IOException {
293
294 return _instance._sharedWriter.getWriter(companyId, create);
295 }
296
297 public static void releaseLock(long companyId) {
298 _instance._sharedWriter.releaseLock(companyId);
299 }
300
301 public static void write(long companyId) {
302 _instance._sharedWriter.write(companyId);
303 }
304
305 public static void write(IndexWriter writer) throws IOException {
306 _instance._sharedWriter.write(writer);
307 }
308
309 private LuceneUtil() {
310 String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
311
312 if (Validator.isNotNull(analyzerName)) {
313 try {
314 _analyzerClass = Class.forName(analyzerName);
315 }
316 catch (Exception e) {
317 _log.error(e);
318 }
319 }
320
321
323 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
324 Connection con = null;
325
326 try {
327 con = DataAccess.getConnection();
328
329 String url = con.getMetaData().getURL();
330
331 int x = url.indexOf(":");
332 int y = url.indexOf(":", x + 1);
333
334 String urlPrefix = url.substring(x + 1, y);
335
336 String dialectClass = PropsUtil.get(
337 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
338
339 if (dialectClass != null) {
340 if (_log.isDebugEnabled()) {
341 _log.debug("JDBC class implementation " + dialectClass);
342 }
343 }
344 else {
345 if (_log.isDebugEnabled()) {
346 _log.debug("JDBC class implementation is null");
347 }
348 }
349
350 if (dialectClass != null) {
351 _dialect =
352 (Dialect)Class.forName(dialectClass).newInstance();
353 }
354 }
355 catch (Exception e) {
356 _log.error(e);
357 }
358 finally{
359 DataAccess.cleanUp(con);
360 }
361
362 if (_dialect == null) {
363 _log.error("No JDBC dialect found");
364 }
365 }
366 }
367
368 public void _delete(long companyId) {
369 if (PropsValues.INDEX_READ_ONLY) {
370 return;
371 }
372
373 if (_log.isDebugEnabled()) {
374 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
375 }
376
377 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
378 _deleteFile(companyId);
379 }
380 else if (PropsValues.LUCENE_STORE_TYPE.equals(
381 _LUCENE_STORE_TYPE_JDBC)) {
382
383 _deleteJdbc(companyId);
384 }
385 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
386 _deleteRam(companyId);
387 }
388 else {
389 throw new RuntimeException(
390 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
391 }
392 }
393
394 private void _deleteFile(long companyId) {
395 String path = _getPath(companyId);
396
397 try {
398 Directory directory = FSDirectory.getDirectory(path, false);
399
400 directory.close();
401 }
402 catch (Exception e) {
403 if (_log.isWarnEnabled()) {
404 _log.warn("Could not close directory " + path);
405 }
406 }
407
408 FileUtil.deltree(path);
409 }
410
411 private void _deleteJdbc(long companyId) {
412 String tableName = _getTableName(companyId);
413
414 try {
415 Directory directory = _jdbcDirectories.remove(tableName);
416
417 if (directory != null) {
418 directory.close();
419 }
420 }
421 catch (Exception e) {
422 if (_log.isWarnEnabled()) {
423 _log.warn("Could not close directory " + tableName);
424 }
425 }
426
427 Connection con = null;
428 Statement s = null;
429
430 try {
431 con = DataAccess.getConnection();
432
433 s = con.createStatement();
434
435 s.executeUpdate("DELETE FROM " + tableName);
436 }
437 catch (Exception e) {
438 if (_log.isWarnEnabled()) {
439 _log.warn("Could not truncate " + tableName);
440 }
441 }
442 finally {
443 DataAccess.cleanUp(con, s);
444 }
445 }
446
447 private void _deleteRam(long companyId) {
448 }
449
450 private Analyzer _getAnalyzer() {
451 try {
452 return (Analyzer)_analyzerClass.newInstance();
453 }
454 catch (Exception e) {
455 throw new RuntimeException(e);
456 }
457 }
458
459 private Directory _getLuceneDir(long companyId) {
460 if (_log.isDebugEnabled()) {
461 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
462 }
463
464 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
465 return _getLuceneDirFile(companyId);
466 }
467 else if (PropsValues.LUCENE_STORE_TYPE.equals(
468 _LUCENE_STORE_TYPE_JDBC)) {
469
470 return _getLuceneDirJdbc(companyId);
471 }
472 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
473 return _getLuceneDirRam(companyId);
474 }
475 else {
476 throw new RuntimeException(
477 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
478 }
479 }
480
481 private Directory _getLuceneDirFile(long companyId) {
482 Directory directory = null;
483
484 String path = _getPath(companyId);
485
486 try {
487 directory = FSDirectory.getDirectory(path, false);
488 }
489 catch (IOException ioe1) {
490 try {
491 if (directory != null) {
492 directory.close();
493 }
494
495 directory = FSDirectory.getDirectory(path, true);
496 }
497 catch (IOException ioe2) {
498 throw new RuntimeException(ioe2);
499 }
500 }
501
502 return directory;
503 }
504
505 private Directory _getLuceneDirJdbc(long companyId) {
506 JdbcDirectory directory = null;
507
508 ClassLoader contextClassLoader =
509 Thread.currentThread().getContextClassLoader();
510
511 try {
512 Thread.currentThread().setContextClassLoader(
513 PortalClassLoaderUtil.getClassLoader());
514
515 String tableName = _getTableName(companyId);
516
517 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
518
519 if (directory != null) {
520 return directory;
521 }
522
523 try {
524 DataSource ds = PortalJNDIUtil.getDataSource();
525
526 directory = new JdbcDirectory(ds, _dialect, tableName);
527
528 _jdbcDirectories.put(tableName, directory);
529
530 if (!directory.tableExists()) {
531 directory.create();
532 }
533 }
534 catch (IOException ioe) {
535 throw new RuntimeException(ioe);
536 }
537 catch (NamingException ne) {
538 throw new RuntimeException(ne);
539 }
540 catch (UnsupportedOperationException uoe) {
541 if (_log.isWarnEnabled()) {
542 _log.warn(
543 "Database doesn't support the ability to check " +
544 "whether a table exists");
545 }
546
547 _manuallyCreateJdbcDirectory(directory, tableName);
548 }
549 }
550 finally {
551 Thread.currentThread().setContextClassLoader(contextClassLoader);
552 }
553
554 return directory;
555 }
556
557 private Directory _getLuceneDirRam(long companyId) {
558 String path = _getPath(companyId);
559
560 Directory directory = _ramDirectories.get(path);
561
562 if (directory == null) {
563 directory = new RAMDirectory();
564
565 _ramDirectories.put(path, directory);
566 }
567
568 return directory;
569 }
570
571 private String _getPath(long companyId) {
572 StringBuilder sb = new StringBuilder();
573
574 sb.append(PropsValues.LUCENE_DIR);
575 sb.append(companyId);
576 sb.append(StringPool.SLASH);
577
578 return sb.toString();
579 }
580
581 private String _getTableName(long companyId) {
582 return _LUCENE_TABLE_PREFIX + companyId;
583 }
584
585 private void _manuallyCreateJdbcDirectory(
586 JdbcDirectory directory, String tableName) {
587
588
590 Connection con = null;
591 ResultSet rs = null;
592
593 try {
594 con = DataAccess.getConnection();
595
596
598 DatabaseMetaData metaData = con.getMetaData();
599
600 rs = metaData.getTables(null, null, tableName, null);
601
602 if (!rs.next()) {
603 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
604
605 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
606
607 Class<?> lockClass = directory.getSettings().getLockClass();
608
609 JdbcLock jdbcLock = null;
610
611 try {
612 jdbcLock = (JdbcLock)lockClass.newInstance();
613 }
614 catch (Exception e) {
615 throw new JdbcStoreException(
616 "Failed to create lock class " + lockClass);
617 }
618
619 jdbcLock.initializeDatabase(directory);
620 }
621 }
622 catch (Exception e) {
623 if (_log.isWarnEnabled()) {
624 _log.warn("Could not create " + tableName);
625 }
626 }
627 finally {
628 DataAccess.cleanUp(con, null, rs);
629 }
630 }
631
632 private static final String _LUCENE_STORE_TYPE_FILE = "file";
633
634 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
635
636 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
637
638 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
639
640 private static Log _log = LogFactory.getLog(LuceneUtil.class);
641
642 private static LuceneUtil _instance = new LuceneUtil();
643
644 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
645 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
646 private Dialect _dialect;
647 private Map<String, Directory> _jdbcDirectories =
648 new ConcurrentHashMap<String, Directory>();
649 private Map<String, Directory> _ramDirectories =
650 new ConcurrentHashMap<String, Directory>();
651
652 }