001
014
015 package com.liferay.portal.search.lucene;
016
017 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
018 import com.liferay.portal.kernel.log.Log;
019 import com.liferay.portal.kernel.log.LogFactoryUtil;
020 import com.liferay.portal.kernel.search.SearchEngineUtil;
021 import com.liferay.portal.kernel.util.CharPool;
022 import com.liferay.portal.kernel.util.FileUtil;
023 import com.liferay.portal.kernel.util.InfrastructureUtil;
024 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
025 import com.liferay.portal.kernel.util.PropsKeys;
026 import com.liferay.portal.kernel.util.StringPool;
027 import com.liferay.portal.util.PropsUtil;
028 import com.liferay.portal.util.PropsValues;
029
030 import java.io.File;
031 import java.io.IOException;
032
033 import java.sql.Connection;
034 import java.sql.DatabaseMetaData;
035 import java.sql.ResultSet;
036 import java.sql.Statement;
037
038 import java.util.Map;
039 import java.util.concurrent.ConcurrentHashMap;
040 import java.util.concurrent.Executors;
041 import java.util.concurrent.ScheduledExecutorService;
042 import java.util.concurrent.TimeUnit;
043
044 import javax.sql.DataSource;
045
046 import org.apache.lucene.document.Document;
047 import org.apache.lucene.index.IndexWriter;
048 import org.apache.lucene.index.Term;
049 import org.apache.lucene.store.Directory;
050 import org.apache.lucene.store.FSDirectory;
051 import org.apache.lucene.store.RAMDirectory;
052 import org.apache.lucene.store.jdbc.JdbcDirectory;
053 import org.apache.lucene.store.jdbc.JdbcStoreException;
054 import org.apache.lucene.store.jdbc.dialect.Dialect;
055 import org.apache.lucene.store.jdbc.lock.JdbcLock;
056 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
057
058
063 public class IndexAccessorImpl implements IndexAccessor {
064
065 public IndexAccessorImpl(long companyId) {
066 _companyId = companyId;
067
068 _initDialect();
069 _checkLuceneDir();
070 _initIndexWriter();
071 _initCommitScheduler();
072 }
073
074 public void addDocument(Document document) throws IOException {
075 if (SearchEngineUtil.isIndexReadOnly()) {
076 return;
077 }
078
079 _write(null, document);
080 }
081
082 public void close() {
083 try {
084 _indexWriter.close();
085 }
086 catch(Exception e) {
087 _log.error(
088 "Closing Lucene writer failed for " + _companyId, e);
089 }
090 }
091
092 public void delete() {
093 if (SearchEngineUtil.isIndexReadOnly()) {
094 return;
095 }
096
097 close();
098
099 if (_log.isDebugEnabled()) {
100 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
101 }
102
103 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
104 _deleteFile();
105 }
106 else if (PropsValues.LUCENE_STORE_TYPE.equals(
107 _LUCENE_STORE_TYPE_JDBC)) {
108
109 _deleteJdbc();
110 }
111 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
112 _deleteRam();
113 }
114 else {
115 throw new RuntimeException(
116 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
117 }
118
119 _initIndexWriter();
120 }
121
122 public void deleteDocuments(Term term) throws IOException {
123 if (SearchEngineUtil.isIndexReadOnly()) {
124 return;
125 }
126
127 try {
128 _indexWriter.deleteDocuments(term);
129
130 _batchCount++;
131 }
132 finally {
133 _commit();
134 }
135 }
136
137 public long getCompanyId() {
138 return _companyId;
139 }
140
141 public Directory getLuceneDir() {
142 if (_log.isDebugEnabled()) {
143 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
144 }
145
146 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
147 return _getLuceneDirFile();
148 }
149 else if (PropsValues.LUCENE_STORE_TYPE.equals(
150 _LUCENE_STORE_TYPE_JDBC)) {
151
152 return _getLuceneDirJdbc();
153 }
154 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
155 return _getLuceneDirRam();
156 }
157 else {
158 throw new RuntimeException(
159 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
160 }
161 }
162
163 public void updateDocument(Term term, Document document)
164 throws IOException {
165
166 if (SearchEngineUtil.isIndexReadOnly()) {
167 return;
168 }
169
170 _write(term, document);
171 }
172
173 private void _checkLuceneDir() {
174 if (SearchEngineUtil.isIndexReadOnly()) {
175 return;
176 }
177
178 try {
179 Directory directory = getLuceneDir();
180
181 if (IndexWriter.isLocked(directory)) {
182 IndexWriter.unlock(directory);
183 }
184 }
185 catch (Exception e) {
186 _log.error("Check Lucene directory failed for " + _companyId, e);
187 }
188 }
189
190 private void _commit() throws IOException {
191 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
192 (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
193
194 _doCommit();
195 }
196 }
197
198 private void _deleteFile() {
199 String path = _getPath();
200
201 try {
202 Directory directory = _getDirectory(path);
203
204 directory.close();
205 }
206 catch (Exception e) {
207 if (_log.isWarnEnabled()) {
208 _log.warn("Could not close directory " + path);
209 }
210 }
211
212 FileUtil.deltree(path);
213 }
214
215 private void _deleteJdbc() {
216 String tableName = _getTableName();
217
218 try {
219 Directory directory = _jdbcDirectories.remove(tableName);
220
221 if (directory != null) {
222 directory.close();
223 }
224 }
225 catch (Exception e) {
226 if (_log.isWarnEnabled()) {
227 _log.warn("Could not close directory " + tableName);
228 }
229 }
230
231 Connection con = null;
232 Statement s = null;
233
234 try {
235 con = DataAccess.getConnection();
236
237 s = con.createStatement();
238
239 s.executeUpdate("DELETE FROM " + tableName);
240 }
241 catch (Exception e) {
242 if (_log.isWarnEnabled()) {
243 _log.warn("Could not truncate " + tableName);
244 }
245 }
246 finally {
247 DataAccess.cleanUp(con, s);
248 }
249 }
250
251 private void _deleteRam() {
252 }
253
254 private void _doCommit() throws IOException {
255 if (_indexWriter != null) {
256 _indexWriter.commit();
257 }
258
259 _batchCount = 0;
260 }
261
262 private FSDirectory _getDirectory(String path) throws IOException {
263 return FSDirectory.open(new File(path));
264 }
265
266 private Directory _getLuceneDirFile() {
267 Directory directory = null;
268
269 String path = _getPath();
270
271 try {
272 directory = _getDirectory(path);
273 }
274 catch (IOException ioe1) {
275 if (directory != null) {
276 try {
277 directory.close();
278 }
279 catch (Exception e) {
280 }
281 }
282 }
283
284 return directory;
285 }
286
287 private Directory _getLuceneDirJdbc() {
288 JdbcDirectory jdbcDirectory = null;
289
290 Thread currentThread = Thread.currentThread();
291
292 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
293
294 try {
295 currentThread.setContextClassLoader(
296 PortalClassLoaderUtil.getClassLoader());
297
298 String tableName = _getTableName();
299
300 jdbcDirectory = (JdbcDirectory)_jdbcDirectories.get(tableName);
301
302 if (jdbcDirectory != null) {
303 return jdbcDirectory;
304 }
305
306 try {
307 DataSource dataSource = InfrastructureUtil.getDataSource();
308
309 jdbcDirectory = new JdbcDirectory(
310 dataSource, _dialect, tableName);
311
312 _jdbcDirectories.put(tableName, jdbcDirectory);
313
314 if (!jdbcDirectory.tableExists()) {
315 jdbcDirectory.create();
316 }
317 }
318 catch (IOException ioe) {
319 throw new RuntimeException(ioe);
320 }
321 catch (UnsupportedOperationException uoe) {
322 if (_log.isWarnEnabled()) {
323 _log.warn(
324 "Database doesn't support the ability to check " +
325 "whether a table exists");
326 }
327
328 _manuallyCreateJdbcDirectory(jdbcDirectory, tableName);
329 }
330 }
331 finally {
332 currentThread.setContextClassLoader(contextClassLoader);
333 }
334
335 return jdbcDirectory;
336 }
337
338 private Directory _getLuceneDirRam() {
339 String path = _getPath();
340
341 Directory directory = _ramDirectories.get(path);
342
343 if (directory == null) {
344 directory = new RAMDirectory();
345
346 _ramDirectories.put(path, directory);
347 }
348
349 return directory;
350 }
351
352 private String _getPath() {
353 return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
354 StringPool.SLASH);
355 }
356
357 private String _getTableName() {
358 return _LUCENE_TABLE_PREFIX + _companyId;
359 }
360
361 private void _initCommitScheduler() {
362 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
363 (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
364
365 return;
366 }
367
368 ScheduledExecutorService scheduledExecutorService =
369 Executors.newSingleThreadScheduledExecutor();
370
371 Runnable runnable = new Runnable() {
372
373 public void run() {
374 try {
375 _doCommit();
376 }
377 catch (IOException ioe) {
378 _log.error("Could not run scheduled commit", ioe);
379 }
380 }
381
382 };
383
384 scheduledExecutorService.scheduleWithFixedDelay(
385 runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
386 TimeUnit.MILLISECONDS);
387 }
388
389 private void _initDialect() {
390 if (!PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
391 return;
392 }
393
394 Connection con = null;
395
396 try {
397 con = DataAccess.getConnection();
398
399 String url = con.getMetaData().getURL();
400
401 int x = url.indexOf(CharPool.COLON);
402 int y = url.indexOf(CharPool.COLON, x + 1);
403
404 String urlPrefix = url.substring(x + 1, y);
405
406 String dialectClass = PropsUtil.get(
407 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
408
409 if (dialectClass != null) {
410 if (_log.isDebugEnabled()) {
411 _log.debug("JDBC class implementation " + dialectClass);
412 }
413 }
414 else {
415 if (_log.isDebugEnabled()) {
416 _log.debug("JDBC class implementation is null");
417 }
418 }
419
420 if (dialectClass != null) {
421 _dialect = (Dialect)Class.forName(dialectClass).newInstance();
422 }
423 }
424 catch (Exception e) {
425 _log.error(e);
426 }
427 finally{
428 DataAccess.cleanUp(con);
429 }
430
431 if (_dialect == null) {
432 _log.error("No JDBC dialect found");
433 }
434 }
435
436 private void _initIndexWriter() {
437 try {
438 _indexWriter = new IndexWriter(
439 getLuceneDir(), LuceneHelperUtil.getAnalyzer(),
440 IndexWriter.MaxFieldLength.LIMITED);
441
442 _indexWriter.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
443 _indexWriter.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE);
444 }
445 catch (Exception e) {
446 _log.error(
447 "Initializing Lucene writer failed for " + _companyId, e);
448 }
449 }
450
451 private void _manuallyCreateJdbcDirectory(
452 JdbcDirectory jdbcDirectory, String tableName) {
453
454
455
456 Connection con = null;
457 ResultSet rs = null;
458
459 try {
460 con = DataAccess.getConnection();
461
462
463
464 DatabaseMetaData metaData = con.getMetaData();
465
466 rs = metaData.getTables(null, null, tableName, null);
467
468 if (!rs.next()) {
469 JdbcTemplate jdbcTemplate = jdbcDirectory.getJdbcTemplate();
470
471 jdbcTemplate.executeUpdate(
472 jdbcDirectory.getTable().sqlCreate());
473
474 Class<?> lockClass = jdbcDirectory.getSettings().getLockClass();
475
476 JdbcLock jdbcLock = null;
477
478 try {
479 jdbcLock = (JdbcLock)lockClass.newInstance();
480 }
481 catch (Exception e) {
482 throw new JdbcStoreException(
483 "Could not create lock class " + lockClass);
484 }
485
486 jdbcLock.initializeDatabase(jdbcDirectory);
487 }
488 }
489 catch (Exception e) {
490 if (_log.isWarnEnabled()) {
491 _log.warn("Could not create " + tableName);
492 }
493 }
494 finally {
495 DataAccess.cleanUp(con, null, rs);
496 }
497 }
498
499 private void _write(Term term, Document document) throws IOException {
500 try {
501 if (term != null) {
502 _indexWriter.updateDocument(term, document);
503 }
504 else {
505 _indexWriter.addDocument(document);
506 }
507
508 _optimizeCount++;
509
510 if ((PropsValues.LUCENE_OPTIMIZE_INTERVAL == 0) ||
511 (_optimizeCount >= PropsValues.LUCENE_OPTIMIZE_INTERVAL)) {
512
513 _indexWriter.optimize();
514
515 _optimizeCount = 0;
516 }
517
518 _batchCount++;
519 }
520 finally {
521 _commit();
522 }
523 }
524
525 private static final String _LUCENE_STORE_TYPE_FILE = "file";
526
527 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
528
529 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
530
531 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
532
533 private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
534
535 private int _batchCount;
536 private long _companyId;
537 private Dialect _dialect;
538 private IndexWriter _indexWriter;
539 private Map<String, Directory> _jdbcDirectories =
540 new ConcurrentHashMap<String, Directory>();
541 private int _optimizeCount;
542 private Map<String, Directory> _ramDirectories =
543 new ConcurrentHashMap<String, Directory>();
544
545 }