001    /**
002     * Copyright (c) 2000-2010 Liferay, Inc. All rights reserved.
003     *
004     * The contents of this file are subject to the terms of the Liferay Enterprise
005     * Subscription License ("License"). You may not use this file except in
006     * compliance with the License. You can obtain a copy of the License by
007     * contacting Liferay, Inc. See the License for the specific language governing
008     * permissions and limitations under the License, including but not limited to
009     * distribution rights of the Software.
010     *
011     *
012     *
013     */
014    
015    package com.liferay.portal.search.lucene;
016    
017    import com.liferay.portal.kernel.dao.jdbc.DataAccess;
018    import com.liferay.portal.kernel.log.Log;
019    import com.liferay.portal.kernel.log.LogFactoryUtil;
020    import com.liferay.portal.kernel.search.SearchEngineUtil;
021    import com.liferay.portal.kernel.util.CharPool;
022    import com.liferay.portal.kernel.util.FileUtil;
023    import com.liferay.portal.kernel.util.InfrastructureUtil;
024    import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
025    import com.liferay.portal.kernel.util.PropsKeys;
026    import com.liferay.portal.kernel.util.StringPool;
027    import com.liferay.portal.util.PropsUtil;
028    import com.liferay.portal.util.PropsValues;
029    
030    import java.io.File;
031    import java.io.IOException;
032    
033    import java.sql.Connection;
034    import java.sql.DatabaseMetaData;
035    import java.sql.ResultSet;
036    import java.sql.Statement;
037    
038    import java.util.Map;
039    import java.util.concurrent.ConcurrentHashMap;
040    import java.util.concurrent.Executors;
041    import java.util.concurrent.ScheduledExecutorService;
042    import java.util.concurrent.TimeUnit;
043    
044    import javax.sql.DataSource;
045    
046    import org.apache.lucene.document.Document;
047    import org.apache.lucene.index.IndexWriter;
048    import org.apache.lucene.index.Term;
049    import org.apache.lucene.store.Directory;
050    import org.apache.lucene.store.FSDirectory;
051    import org.apache.lucene.store.RAMDirectory;
052    import org.apache.lucene.store.jdbc.JdbcDirectory;
053    import org.apache.lucene.store.jdbc.JdbcStoreException;
054    import org.apache.lucene.store.jdbc.dialect.Dialect;
055    import org.apache.lucene.store.jdbc.lock.JdbcLock;
056    import org.apache.lucene.store.jdbc.support.JdbcTemplate;
057    
058    /**
059     * @author Harry Mark
060     * @author Brian Wing Shun Chan
061     * @author Bruno Farache
062     */
063    public class IndexAccessorImpl implements IndexAccessor {
064    
065            public IndexAccessorImpl(long companyId) {
066                    _companyId = companyId;
067    
068                    _initDialect();
069                    _checkLuceneDir();
070                    _initIndexWriter();
071                    _initCommitScheduler();
072            }
073    
074            public void addDocument(Document document) throws IOException {
075                    if (SearchEngineUtil.isIndexReadOnly()) {
076                            return;
077                    }
078    
079                    _write(null, document);
080            }
081    
082            public void close() {
083                    try {
084                            _indexWriter.close();
085                    }
086                    catch(Exception e) {
087                            _log.error(
088                                    "Closing Lucene writer failed for " + _companyId, e);
089                    }
090            }
091    
092            public void delete() {
093                    if (SearchEngineUtil.isIndexReadOnly()) {
094                            return;
095                    }
096    
097                    close();
098    
099                    if (_log.isDebugEnabled()) {
100                            _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
101                    }
102    
103                    if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
104                            _deleteFile();
105                    }
106                    else if (PropsValues.LUCENE_STORE_TYPE.equals(
107                                            _LUCENE_STORE_TYPE_JDBC)) {
108    
109                            _deleteJdbc();
110                    }
111                    else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
112                            _deleteRam();
113                    }
114                    else {
115                            throw new RuntimeException(
116                                    "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
117                    }
118    
119                    _initIndexWriter();
120            }
121    
122            public void deleteDocuments(Term term) throws IOException {
123                    if (SearchEngineUtil.isIndexReadOnly()) {
124                            return;
125                    }
126    
127                    try {
128                            _indexWriter.deleteDocuments(term);
129    
130                            _batchCount++;
131                    }
132                    finally {
133                            _commit();
134                    }
135            }
136    
137            public long getCompanyId() {
138                    return _companyId;
139            }
140    
141            public Directory getLuceneDir() {
142                    if (_log.isDebugEnabled()) {
143                            _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
144                    }
145    
146                    if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
147                            return _getLuceneDirFile();
148                    }
149                    else if (PropsValues.LUCENE_STORE_TYPE.equals(
150                                            _LUCENE_STORE_TYPE_JDBC)) {
151    
152                            return _getLuceneDirJdbc();
153                    }
154                    else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
155                            return _getLuceneDirRam();
156                    }
157                    else {
158                            throw new RuntimeException(
159                                    "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
160                    }
161            }
162    
163            public void updateDocument(Term term, Document document)
164                    throws IOException {
165    
166                    if (SearchEngineUtil.isIndexReadOnly()) {
167                            return;
168                    }
169    
170                    _write(term, document);
171            }
172    
173            private void _checkLuceneDir() {
174                    if (SearchEngineUtil.isIndexReadOnly()) {
175                            return;
176                    }
177    
178                    try {
179                            Directory directory = getLuceneDir();
180    
181                            if (IndexWriter.isLocked(directory)) {
182                                    IndexWriter.unlock(directory);
183                            }
184                    }
185                    catch (Exception e) {
186                            _log.error("Check Lucene directory failed for " + _companyId, e);
187                    }
188            }
189    
190            private void _commit() throws IOException {
191                    if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
192                            (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
193    
194                            _doCommit();
195                    }
196            }
197    
198            private void _deleteFile() {
199                    String path = _getPath();
200    
201                    try {
202                            Directory directory = _getDirectory(path);
203    
204                            directory.close();
205                    }
206                    catch (Exception e) {
207                            if (_log.isWarnEnabled()) {
208                                    _log.warn("Could not close directory " + path);
209                            }
210                    }
211    
212                    FileUtil.deltree(path);
213            }
214    
215            private void _deleteJdbc() {
216                    String tableName = _getTableName();
217    
218                    try {
219                            Directory directory = _jdbcDirectories.remove(tableName);
220    
221                            if (directory != null) {
222                                    directory.close();
223                            }
224                    }
225                    catch (Exception e) {
226                            if (_log.isWarnEnabled()) {
227                                    _log.warn("Could not close directory " + tableName);
228                            }
229                    }
230    
231                    Connection con = null;
232                    Statement s = null;
233    
234                    try {
235                            con = DataAccess.getConnection();
236    
237                            s = con.createStatement();
238    
239                            s.executeUpdate("DELETE FROM " + tableName);
240                    }
241                    catch (Exception e) {
242                            if (_log.isWarnEnabled()) {
243                                    _log.warn("Could not truncate " + tableName);
244                            }
245                    }
246                    finally {
247                            DataAccess.cleanUp(con, s);
248                    }
249            }
250    
251            private void _deleteRam() {
252            }
253    
254            private void _doCommit() throws IOException {
255                    if (_indexWriter != null) {
256                            _indexWriter.commit();
257                    }
258    
259                    _batchCount = 0;
260            }
261    
262            private FSDirectory _getDirectory(String path) throws IOException {
263                    return FSDirectory.open(new File(path));
264            }
265    
266            private Directory _getLuceneDirFile() {
267                    Directory directory = null;
268    
269                    String path = _getPath();
270    
271                    try {
272                            directory = _getDirectory(path);
273                    }
274                    catch (IOException ioe1) {
275                            if (directory != null) {
276                                    try {
277                                            directory.close();
278                                    }
279                                    catch (Exception e) {
280                                    }
281                            }
282                    }
283    
284                    return directory;
285            }
286    
287            private Directory _getLuceneDirJdbc() {
288                    JdbcDirectory jdbcDirectory = null;
289    
290                    Thread currentThread = Thread.currentThread();
291    
292                    ClassLoader contextClassLoader = currentThread.getContextClassLoader();
293    
294                    try {
295                            currentThread.setContextClassLoader(
296                                    PortalClassLoaderUtil.getClassLoader());
297    
298                            String tableName = _getTableName();
299    
300                            jdbcDirectory = (JdbcDirectory)_jdbcDirectories.get(tableName);
301    
302                            if (jdbcDirectory != null) {
303                                    return jdbcDirectory;
304                            }
305    
306                            try {
307                                    DataSource dataSource = InfrastructureUtil.getDataSource();
308    
309                                    jdbcDirectory = new JdbcDirectory(
310                                            dataSource, _dialect, tableName);
311    
312                                    _jdbcDirectories.put(tableName, jdbcDirectory);
313    
314                                    if (!jdbcDirectory.tableExists()) {
315                                            jdbcDirectory.create();
316                                    }
317                            }
318                            catch (IOException ioe) {
319                                    throw new RuntimeException(ioe);
320                            }
321                            catch (UnsupportedOperationException uoe) {
322                                    if (_log.isWarnEnabled()) {
323                                            _log.warn(
324                                                    "Database doesn't support the ability to check " +
325                                                            "whether a table exists");
326                                    }
327    
328                                    _manuallyCreateJdbcDirectory(jdbcDirectory, tableName);
329                            }
330                    }
331                    finally {
332                            currentThread.setContextClassLoader(contextClassLoader);
333                    }
334    
335                    return jdbcDirectory;
336            }
337    
338            private Directory _getLuceneDirRam() {
339                    String path = _getPath();
340    
341                    Directory directory = _ramDirectories.get(path);
342    
343                    if (directory == null) {
344                            directory = new RAMDirectory();
345    
346                            _ramDirectories.put(path, directory);
347                    }
348    
349                    return directory;
350            }
351    
352            private String _getPath() {
353                    return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
354                            StringPool.SLASH);
355            }
356    
357            private String _getTableName() {
358                    return _LUCENE_TABLE_PREFIX + _companyId;
359            }
360    
361            private void _initCommitScheduler() {
362                    if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
363                            (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
364    
365                            return;
366                    }
367    
368                    ScheduledExecutorService scheduledExecutorService =
369                            Executors.newSingleThreadScheduledExecutor();
370    
371                    Runnable runnable = new Runnable() {
372    
373                            public void run() {
374                                    try {
375                                            _doCommit();
376                                    }
377                                    catch (IOException ioe) {
378                                            _log.error("Could not run scheduled commit", ioe);
379                                    }
380                            }
381    
382                    };
383    
384                    scheduledExecutorService.scheduleWithFixedDelay(
385                            runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
386                            TimeUnit.MILLISECONDS);
387            }
388    
389            private void _initDialect() {
390                    if (!PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
391                            return;
392                    }
393    
394                    Connection con = null;
395    
396                    try {
397                            con = DataAccess.getConnection();
398    
399                            String url = con.getMetaData().getURL();
400    
401                            int x = url.indexOf(CharPool.COLON);
402                            int y = url.indexOf(CharPool.COLON, x + 1);
403    
404                            String urlPrefix = url.substring(x + 1, y);
405    
406                            String dialectClass = PropsUtil.get(
407                                    PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
408    
409                            if (dialectClass != null) {
410                                    if (_log.isDebugEnabled()) {
411                                            _log.debug("JDBC class implementation " + dialectClass);
412                                    }
413                            }
414                            else {
415                                    if (_log.isDebugEnabled()) {
416                                            _log.debug("JDBC class implementation is null");
417                                    }
418                            }
419    
420                            if (dialectClass != null) {
421                                    _dialect = (Dialect)Class.forName(dialectClass).newInstance();
422                            }
423                    }
424                    catch (Exception e) {
425                            _log.error(e);
426                    }
427                    finally{
428                            DataAccess.cleanUp(con);
429                    }
430    
431                    if (_dialect == null) {
432                            _log.error("No JDBC dialect found");
433                    }
434            }
435    
436            private void _initIndexWriter() {
437                    try {
438                            _indexWriter = new IndexWriter(
439                                    getLuceneDir(), LuceneHelperUtil.getAnalyzer(),
440                                    IndexWriter.MaxFieldLength.LIMITED);
441    
442                            _indexWriter.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
443                            _indexWriter.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE);
444                    }
445                    catch (Exception e) {
446                            _log.error(
447                                    "Initializing Lucene writer failed for " + _companyId, e);
448                    }
449            }
450    
451            private void _manuallyCreateJdbcDirectory(
452                    JdbcDirectory jdbcDirectory, String tableName) {
453    
454                    // LEP-2181
455    
456                    Connection con = null;
457                    ResultSet rs = null;
458    
459                    try {
460                            con = DataAccess.getConnection();
461    
462                            // Check if table exists
463    
464                            DatabaseMetaData metaData = con.getMetaData();
465    
466                            rs = metaData.getTables(null, null, tableName, null);
467    
468                            if (!rs.next()) {
469                                    JdbcTemplate jdbcTemplate = jdbcDirectory.getJdbcTemplate();
470    
471                                    jdbcTemplate.executeUpdate(
472                                            jdbcDirectory.getTable().sqlCreate());
473    
474                                    Class<?> lockClass = jdbcDirectory.getSettings().getLockClass();
475    
476                                    JdbcLock jdbcLock = null;
477    
478                                    try {
479                                            jdbcLock = (JdbcLock)lockClass.newInstance();
480                                    }
481                                    catch (Exception e) {
482                                            throw new JdbcStoreException(
483                                                    "Could not create lock class " + lockClass);
484                                    }
485    
486                                    jdbcLock.initializeDatabase(jdbcDirectory);
487                            }
488                    }
489                    catch (Exception e) {
490                            if (_log.isWarnEnabled()) {
491                                    _log.warn("Could not create " + tableName);
492                            }
493                    }
494                    finally {
495                            DataAccess.cleanUp(con, null, rs);
496                    }
497            }
498    
499            private void _write(Term term, Document document) throws IOException {
500                    try {
501                            if (term != null) {
502                                    _indexWriter.updateDocument(term, document);
503                            }
504                            else {
505                                    _indexWriter.addDocument(document);
506                            }
507    
508                            _optimizeCount++;
509    
510                            if ((PropsValues.LUCENE_OPTIMIZE_INTERVAL == 0) ||
511                                    (_optimizeCount >= PropsValues.LUCENE_OPTIMIZE_INTERVAL)) {
512    
513                                    _indexWriter.optimize();
514    
515                                    _optimizeCount = 0;
516                            }
517    
518                            _batchCount++;
519                    }
520                    finally {
521                            _commit();
522                    }
523            }
524    
525            private static final String _LUCENE_STORE_TYPE_FILE = "file";
526    
527            private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
528    
529            private static final String _LUCENE_STORE_TYPE_RAM = "ram";
530    
531            private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
532    
533            private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
534    
535            private int _batchCount;
536            private long _companyId;
537            private Dialect _dialect;
538            private IndexWriter _indexWriter;
539            private Map<String, Directory> _jdbcDirectories =
540                    new ConcurrentHashMap<String, Directory>();
541            private int _optimizeCount;
542            private Map<String, Directory> _ramDirectories =
543                    new ConcurrentHashMap<String, Directory>();
544    
545    }