1
22
23 package com.liferay.portal.lucene;
24
25 import com.liferay.portal.SystemException;
26 import com.liferay.portal.kernel.search.Hits;
27 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
28 import com.liferay.portal.kernel.util.StringMaker;
29 import com.liferay.portal.kernel.util.StringPool;
30 import com.liferay.portal.kernel.util.Validator;
31 import com.liferay.portal.spring.hibernate.HibernateUtil;
32 import com.liferay.portal.util.PropsUtil;
33 import com.liferay.util.CollectionFactory;
34 import com.liferay.util.FileUtil;
35 import com.liferay.util.dao.DataAccess;
36 import com.liferay.util.lucene.HitsImpl;
37 import com.liferay.util.lucene.KeywordsUtil;
38
39 import java.io.IOException;
40
41 import java.sql.Connection;
42 import java.sql.DatabaseMetaData;
43 import java.sql.ResultSet;
44 import java.sql.Statement;
45
46 import java.util.Map;
47 import java.util.regex.Pattern;
48
49 import javax.sql.DataSource;
50
51 import org.apache.commons.logging.Log;
52 import org.apache.commons.logging.LogFactory;
53 import org.apache.lucene.analysis.Analyzer;
54 import org.apache.lucene.analysis.WhitespaceAnalyzer;
55 import org.apache.lucene.index.IndexReader;
56 import org.apache.lucene.index.IndexWriter;
57 import org.apache.lucene.index.Term;
58 import org.apache.lucene.queryParser.ParseException;
59 import org.apache.lucene.queryParser.QueryParser;
60 import org.apache.lucene.search.BooleanClause;
61 import org.apache.lucene.search.BooleanQuery;
62 import org.apache.lucene.search.IndexSearcher;
63 import org.apache.lucene.search.Query;
64 import org.apache.lucene.search.Searcher;
65 import org.apache.lucene.search.TermQuery;
66 import org.apache.lucene.store.Directory;
67 import org.apache.lucene.store.FSDirectory;
68 import org.apache.lucene.store.RAMDirectory;
69 import org.apache.lucene.store.jdbc.JdbcDirectory;
70 import org.apache.lucene.store.jdbc.JdbcStoreException;
71 import org.apache.lucene.store.jdbc.dialect.Dialect;
72 import org.apache.lucene.store.jdbc.lock.JdbcLock;
73 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
74
75
82 public class LuceneUtil {
83
84 public static final Pattern TERM_END_PATTERN =
85 Pattern.compile("(\\w{4,}?)\\b");
86
87 public static void acquireLock(long companyId) {
88 try {
89 _instance._sharedWriter.acquireLock(companyId, true);
90 }
91 catch (InterruptedException ie) {
92 _log.error(ie);
93 }
94 }
95
96 public static void addExactTerm(
97 BooleanQuery booleanQuery, String field, long number)
98 throws ParseException {
99
100 addExactTerm(booleanQuery, field, String.valueOf(number));
101 }
102
103 public static void addExactTerm(
104 BooleanQuery booleanQuery, String field, String text)
105 throws ParseException {
106
107
109 Query query = new TermQuery(new Term(field, text));
110
111 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
112 }
113
114 public static void addTerm(
115 BooleanQuery booleanQuery, String field, long number)
116 throws ParseException {
117
118 addTerm(booleanQuery, field, String.valueOf(number));
119 }
120
121 public static void addTerm(
122 BooleanQuery booleanQuery, String field, String text)
123 throws ParseException {
124
125 if (Validator.isNotNull(text)) {
126
128
130
132
134
136
138
143 QueryParser queryParser = new QueryParser(
144 field, LuceneUtil.getAnalyzer());
145
146 try {
147 Query query = queryParser.parse(text);
148
149 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
150 }
151 catch(ParseException pe) {
152 if (_log.isDebugEnabled()) {
153 _log.debug(
154 "ParseException thrown, reverting to literal search",
155 pe);
156 }
157
158 text = KeywordsUtil.escape(text);
159
160 Query query = queryParser.parse(text);
161
162 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
163 }
164 }
165 }
166
167 public static void addRequiredTerm(
168 BooleanQuery booleanQuery, String field, long number) {
169
170 addRequiredTerm(booleanQuery, field, String.valueOf(number));
171 }
172
173 public static void addRequiredTerm(
174 BooleanQuery booleanQuery, String field, String text) {
175
176
178 Term term = new Term(field, text);
179 TermQuery termQuery = new TermQuery(term);
180
181 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
182 }
183
184 public static void checkLuceneDir(long companyId) {
185 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
186
187 IndexWriter writer = null;
188
189
192 try {
193 if (luceneDir.fileExists("segments.gen")) {
194 writer = new IndexWriter(
195 luceneDir, LuceneUtil.getAnalyzer(), false);
196 }
197 else {
198 writer = new IndexWriter(
199 luceneDir, LuceneUtil.getAnalyzer(), true);
200 }
201 }
202 catch (IOException ioe) {
203 _log.error(ioe);
204 }
205 finally {
206 if (writer != null) {
207 try {
208 writer.close();
209 }
210 catch (IOException ioe) {
211 _log.error(ioe);
212 }
213 }
214 }
215 }
216
217 public static void closeSearcher(Searcher searcher) {
218 try {
219 if (searcher != null){
220 searcher.close();
221 }
222 }
223 catch (Exception e) {
224 }
225 }
226
227 public static Hits closeSearcher(
228 Searcher searcher, String keywords, Exception e)
229 throws SystemException {
230
231 closeSearcher(searcher);
232
233 if (e instanceof BooleanQuery.TooManyClauses ||
234 e instanceof ParseException) {
235
236 _log.error("Parsing keywords " + keywords, e);
237
238 return new HitsImpl();
239 }
240 else {
241 throw new SystemException(e);
242 }
243 }
244
245 public static void delete(long companyId) {
246 _instance._delete(companyId);
247 }
248
249 public static void deleteDocuments(long companyId, Term term)
250 throws IOException {
251
252 try {
253 _instance._sharedWriter.deleteDocuments(companyId, term);
254 }
255 catch (InterruptedException ie) {
256 _log.error(ie);
257 }
258 }
259
260 public static Analyzer getAnalyzer() {
261 return _instance._getAnalyzer();
262 }
263
264 public static Directory getLuceneDir(long companyId) {
265 return _instance._getLuceneDir(companyId);
266 }
267
268 public static IndexReader getReader(long companyId) throws IOException {
269 return IndexReader.open(getLuceneDir(companyId));
270 }
271
272 public static IndexSearcher getSearcher(long companyId)
273 throws IOException {
274
275 return new IndexSearcher(getLuceneDir(companyId));
276 }
277
278 public static IndexWriter getWriter(long companyId) throws IOException {
279 return getWriter(companyId, false);
280 }
281
282 public static IndexWriter getWriter(long companyId, boolean create)
283 throws IOException {
284
285 return _instance._sharedWriter.getWriter(companyId, create);
286 }
287
288 public static void releaseLock(long companyId) {
289 _instance._sharedWriter.releaseLock(companyId);
290 }
291
292 public static void write(long companyId) throws IOException {
293 _instance._sharedWriter.write(companyId);
294 }
295
296 public static void write(IndexWriter writer) throws IOException {
297 _instance._sharedWriter.write(writer);
298 }
299
300 private LuceneUtil() {
301 String analyzerName = PropsUtil.get(PropsUtil.LUCENE_ANALYZER);
302
303 if (Validator.isNotNull(analyzerName)) {
304 try {
305 _analyzerClass = Class.forName(analyzerName);
306 }
307 catch (Exception e) {
308 _log.error(e);
309 }
310 }
311
312
314 if (PropsUtil.get(PropsUtil.LUCENE_STORE_TYPE).equals(
315 _LUCENE_STORE_TYPE_JDBC)) {
316
317 Connection con = null;
318
319 try {
320 con = HibernateUtil.getConnection();
321
322 String url = con.getMetaData().getURL();
323
324 int x = url.indexOf(":");
325 int y = url.indexOf(":", x + 1);
326
327 String urlPrefix = url.substring(x + 1, y);
328
329 String dialectClass = PropsUtil.get(
330 PropsUtil.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
331
332 if (dialectClass != null) {
333 if (_log.isDebugEnabled()) {
334 _log.debug("JDBC class implementation " + dialectClass);
335 }
336 }
337 else {
338 if (_log.isDebugEnabled()) {
339 _log.debug("JDBC class implementation is null");
340 }
341 }
342
343 if (dialectClass != null) {
344 _dialect =
345 (Dialect)Class.forName(dialectClass).newInstance();
346 }
347 }
348 catch (Exception e) {
349 _log.error(e);
350 }
351 finally{
352 DataAccess.cleanUp(con);
353 }
354
355 if (_dialect == null) {
356 _log.error("No JDBC dialect found");
357 }
358 }
359 }
360
361 public void _delete(long companyId) {
362 String storeType = PropsUtil.get(PropsUtil.LUCENE_STORE_TYPE);
363
364 if (_log.isDebugEnabled()) {
365 _log.debug("Lucene store type " + storeType);
366 }
367
368 if (storeType.equals(_LUCENE_STORE_TYPE_FILE)) {
369 _deleteFile(companyId);
370 }
371 else if (storeType.equals(_LUCENE_STORE_TYPE_JDBC)) {
372 _deleteJdbc(companyId);
373 }
374 else if (storeType.equals(_LUCENE_STORE_TYPE_RAM)) {
375 _deleteRam(companyId);
376 }
377 else {
378 throw new RuntimeException("Invalid store type " + storeType);
379 }
380 }
381
382 private void _deleteFile(long companyId) {
383 String path = _getPath(companyId);
384
385 try {
386 Directory directory = FSDirectory.getDirectory(path, false);
387
388 directory.close();
389 }
390 catch (Exception e) {
391 if (_log.isWarnEnabled()) {
392 _log.warn("Could not close directory " + path);
393 }
394 }
395
396 FileUtil.deltree(path);
397 }
398
399 private void _deleteJdbc(long companyId) {
400 String tableName = _getTableName(companyId);
401
402 try {
403 Directory directory = (Directory)_jdbcDirectories.remove(tableName);
404
405 if (directory != null) {
406 directory.close();
407 }
408 }
409 catch (Exception e) {
410 if (_log.isWarnEnabled()) {
411 _log.warn("Could not close directory " + tableName);
412 }
413 }
414
415 Connection con = null;
416 Statement s = null;
417
418 try {
419 con = HibernateUtil.getConnection();
420
421 s = con.createStatement();
422
423 s.executeUpdate("DELETE FROM " + tableName);
424 }
425 catch (Exception e) {
426 if (_log.isWarnEnabled()) {
427 _log.warn("Could not truncate " + tableName);
428 }
429 }
430 finally {
431 DataAccess.cleanUp(con, s);
432 }
433 }
434
435 private void _deleteRam(long companyId) {
436 }
437
438 private Analyzer _getAnalyzer() {
439 try {
440 return (Analyzer)_analyzerClass.newInstance();
441 }
442 catch (Exception e) {
443 throw new RuntimeException(e);
444 }
445 }
446
447 private Directory _getLuceneDir(long companyId) {
448 String storeType = PropsUtil.get(PropsUtil.LUCENE_STORE_TYPE);
449
450 if (_log.isDebugEnabled()) {
451 _log.debug("Lucene store type " + storeType);
452 }
453
454 if (storeType.equals(_LUCENE_STORE_TYPE_FILE)) {
455 return _getLuceneDirFile(companyId);
456 }
457 else if (storeType.equals(_LUCENE_STORE_TYPE_JDBC)) {
458 return _getLuceneDirJdbc(companyId);
459 }
460 else if (storeType.equals(_LUCENE_STORE_TYPE_RAM)) {
461 return _getLuceneDirRam(companyId);
462 }
463 else {
464 throw new RuntimeException("Invalid store type " + storeType);
465 }
466 }
467
468 private Directory _getLuceneDirFile(long companyId) {
469 Directory directory = null;
470
471 String path = _getPath(companyId);
472
473 try {
474 directory = FSDirectory.getDirectory(path, false);
475 }
476 catch (IOException ioe1) {
477 try {
478 if (directory != null) {
479 directory.close();
480 }
481
482 directory = FSDirectory.getDirectory(path, true);
483 }
484 catch (IOException ioe2) {
485 throw new RuntimeException(ioe2);
486 }
487 }
488
489 return directory;
490 }
491
492 private Directory _getLuceneDirJdbc(long companyId) {
493 JdbcDirectory directory = null;
494
495 ClassLoader contextClassLoader =
496 Thread.currentThread().getContextClassLoader();
497
498 try {
499 Thread.currentThread().setContextClassLoader(
500 PortalClassLoaderUtil.getClassLoader());
501
502 String tableName = _getTableName(companyId);
503
504 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
505
506 if (directory != null) {
507 return directory;
508 }
509
510 try {
511 DataSource ds = HibernateUtil.getDataSource();
512
513 directory = new JdbcDirectory(ds, _dialect, tableName);
514
515 _jdbcDirectories.put(tableName, directory);
516
517 if (!directory.tableExists()) {
518 directory.create();
519 }
520 }
521 catch (IOException ioe) {
522 throw new RuntimeException(ioe);
523 }
524 catch (UnsupportedOperationException uoe) {
525 if (_log.isWarnEnabled()) {
526 _log.warn(
527 "Database doesn't support the ability to check " +
528 "whether a table exists");
529 }
530
531 _manuallyCreateJdbcDirectory(directory, tableName);
532 }
533 }
534 finally {
535 Thread.currentThread().setContextClassLoader(contextClassLoader);
536 }
537
538 return directory;
539 }
540
541 private Directory _getLuceneDirRam(long companyId) {
542 String path = _getPath(companyId);
543
544 Directory directory = (Directory)_ramDirectories.get(path);
545
546 if (directory == null) {
547 directory = new RAMDirectory();
548
549 _ramDirectories.put(path, directory);
550 }
551
552 return directory;
553 }
554
555 private String _getPath(long companyId) {
556 StringMaker sm = new StringMaker();
557
558 sm.append(PropsUtil.get(PropsUtil.LUCENE_DIR));
559 sm.append(companyId);
560 sm.append(StringPool.SLASH);
561
562 return sm.toString();
563 }
564
565 private String _getTableName(long companyId) {
566 return _LUCENE_TABLE_PREFIX + companyId;
567 }
568
569 private void _manuallyCreateJdbcDirectory(
570 JdbcDirectory directory, String tableName) {
571
572
574 Connection con = null;
575 ResultSet rs = null;
576
577 try {
578 con = HibernateUtil.getConnection();
579
580
582 DatabaseMetaData metaData = con.getMetaData();
583
584 rs = metaData.getTables(null, null, tableName, null);
585
586 if (!rs.next()) {
587 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
588
589 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
590
591 Class lockClass = directory.getSettings().getLockClass();
592
593 JdbcLock jdbcLock = null;
594
595 try {
596 jdbcLock = (JdbcLock)lockClass.newInstance();
597 }
598 catch (Exception e) {
599 throw new JdbcStoreException(
600 "Failed to create lock class " + lockClass);
601 }
602
603 jdbcLock.initializeDatabase(directory);
604 }
605 }
606 catch (Exception e) {
607 if (_log.isWarnEnabled()) {
608 _log.warn("Could not create " + tableName);
609 }
610 }
611 finally {
612 DataAccess.cleanUp(con, null, rs);
613 }
614 }
615
616 private static final String _LUCENE_STORE_TYPE_FILE = "file";
617
618 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
619
620 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
621
622 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
623
624 private static Log _log = LogFactory.getLog(LuceneUtil.class);
625
626 private static LuceneUtil _instance = new LuceneUtil();
627
628 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
629 private Class _analyzerClass = WhitespaceAnalyzer.class;
630 private Dialect _dialect;
631 private Map _jdbcDirectories = CollectionFactory.getSyncHashMap();
632 private Map _ramDirectories = CollectionFactory.getSyncHashMap();
633
634 }