1   /**
2    * Copyright (c) 2000-2009 Liferay, Inc. All rights reserved.
3    *
4    * The contents of this file are subject to the terms of the Liferay Enterprise
5    * Subscription License ("License"). You may not use this file except in
6    * compliance with the License. You can obtain a copy of the License by
7    * contacting Liferay, Inc. See the License for the specific language governing
8    * permissions and limitations under the License, including but not limited to
9    * distribution rights of the Software.
10   *
11   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
12   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
13   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
14   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
15   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
16   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
17   * SOFTWARE.
18   */
19  
20  package com.liferay.portal.search.lucene;
21  
22  import com.liferay.portal.SystemException;
23  import com.liferay.portal.kernel.log.Log;
24  import com.liferay.portal.kernel.log.LogFactoryUtil;
25  import com.liferay.portal.kernel.search.SearchEngineUtil;
26  import com.liferay.portal.kernel.util.GetterUtil;
27  import com.liferay.portal.model.Company;
28  import com.liferay.portal.model.CompanyConstants;
29  import com.liferay.portal.service.CompanyLocalServiceUtil;
30  import com.liferay.portal.util.PropsKeys;
31  import com.liferay.portal.util.PropsUtil;
32  import com.liferay.util.SystemProperties;
33  
34  import java.io.File;
35  import java.io.IOException;
36  
37  import java.util.HashMap;
38  import java.util.List;
39  import java.util.Map;
40  import java.util.concurrent.Semaphore;
41  
42  import org.apache.lucene.analysis.SimpleAnalyzer;
43  import org.apache.lucene.index.IndexReader;
44  import org.apache.lucene.index.IndexWriter;
45  import org.apache.lucene.index.Term;
46  import org.apache.lucene.store.Directory;
47  import org.apache.lucene.store.FSDirectory;
48  
49  /**
50   * <a href="IndexWriterFactory.java.html"><b><i>View Source</i></b></a>
51   *
52   * <p>
53   * Lucene only allows one IndexWriter to be open at a time. However, multiple
54   * threads can use this single IndexWriter. This class manages a global
55   * IndexWriter and uses reference counting to determine when it can be closed.
56   * </p>
57   *
58   * <p>
59   * To delete documents, IndexReaders are used but cannot delete while another
60   * IndexWriter or IndexReader has the write lock. A semaphore is used to
61   * serialize delete and add operations. If the shared IndexWriter is open,
62   * concurrent add operations are permitted.
63   * </p>
64   *
65   * @author Harry Mark
66   * @author Brian Wing Shun Chan
67   *
68   */
69  public class IndexWriterFactory {
70  
71      public IndexWriterFactory() {
72          if (SearchEngineUtil.isIndexReadOnly()) {
73              return;
74          }
75  
76          // Create semaphores for all companies
77  
78          try {
79              List<Company> companies = CompanyLocalServiceUtil.getCompanies(
80                  false);
81  
82              for (Company company : companies) {
83                  _lockLookup.put(company.getCompanyId(), new Semaphore(1));
84              }
85  
86              _lockLookup.put(CompanyConstants.SYSTEM, new Semaphore(1));
87          }
88          catch (SystemException se) {
89              _log.error(se);
90          }
91      }
92  
93      public void acquireLock(long companyId, boolean needExclusive)
94          throws InterruptedException {
95  
96          if (SearchEngineUtil.isIndexReadOnly()) {
97              return;
98          }
99  
100         Semaphore lock = _lockLookup.get(companyId);
101 
102         if (lock != null) {
103 
104             // Exclusive checking is used to prevent greedy IndexWriter sharing.
105             // This registers a need for exclusive lock, and causes IndexWriters
106             // to wait in FIFO order.
107 
108             if (needExclusive) {
109                 synchronized (_lockLookup) {
110                     _needExclusiveLock++;
111                 }
112             }
113 
114             try {
115                 lock.acquire();
116             }
117             finally {
118                 if (needExclusive) {
119                     synchronized (_lockLookup) {
120                         _needExclusiveLock--;
121                     }
122                 }
123             }
124         }
125         else {
126             if (_log.isWarnEnabled()) {
127                 _log.warn("IndexWriterFactory lock not found for " + companyId);
128             }
129         }
130     }
131 
132     public void deleteDocuments(long companyId, Term term)
133         throws InterruptedException, IOException {
134 
135         if (SearchEngineUtil.isIndexReadOnly()) {
136             return;
137         }
138 
139         try {
140             acquireLock(companyId, true);
141 
142             IndexReader reader = null;
143 
144             try {
145                 reader = IndexReader.open(LuceneUtil.getLuceneDir(companyId));
146 
147                 reader.deleteDocuments(term);
148             }
149             finally {
150                 if (reader != null) {
151                     reader.close();
152                 }
153             }
154         }
155         finally {
156             releaseLock(companyId);
157         }
158     }
159 
160     public IndexWriter getWriter(long companyId, boolean create)
161         throws IOException {
162 
163         if (SearchEngineUtil.isIndexReadOnly()) {
164             return getReadOnlyIndexWriter();
165         }
166 
167         boolean hasError = false;
168         boolean newWriter = false;
169 
170         try {
171 
172             // If others need an exclusive lock, then wait to acquire lock
173             // before proceeding. This prevents starvation.
174 
175             if (_needExclusiveLock > 0) {
176                 acquireLock(companyId, false);
177                 releaseLock(companyId);
178             }
179 
180             synchronized (this) {
181                 IndexWriterData writerData = _writerLookup.get(companyId);
182 
183                 if (writerData == null) {
184                     newWriter = true;
185 
186                     acquireLock(companyId, false);
187 
188                     IndexWriter writer = new IndexWriter(
189                         LuceneUtil.getLuceneDir(companyId),
190                         LuceneUtil.getAnalyzer(), create);
191 
192                     writer.setMergeFactor(_MERGE_FACTOR);
193 
194                     writerData = new IndexWriterData(companyId, writer, 0);
195 
196                     _writerLookup.put(companyId, writerData);
197                 }
198 
199                 writerData.setCount(writerData.getCount() + 1);
200 
201                 return writerData.getWriter();
202             }
203         }
204         catch (Exception e) {
205             hasError = true;
206 
207             _log.error("Unable to create a new writer", e);
208 
209             throw new IOException("Unable to create a new writer");
210         }
211         finally {
212             if (hasError && newWriter) {
213                 try {
214                     releaseLock(companyId);
215                 }
216                 catch (Exception e) {
217                 }
218             }
219         }
220     }
221 
222     public void releaseLock(long companyId) {
223         if (SearchEngineUtil.isIndexReadOnly()) {
224             return;
225         }
226 
227         Semaphore lock = _lockLookup.get(companyId);
228 
229         if (lock != null) {
230             lock.release();
231         }
232     }
233 
234     public void write(long companyId) {
235         if (SearchEngineUtil.isIndexReadOnly()) {
236             return;
237         }
238 
239         IndexWriterData writerData = _writerLookup.get(companyId);
240 
241         if (writerData != null) {
242             decrement(writerData);
243         }
244         else {
245             if (_log.isWarnEnabled()) {
246                 _log.warn("IndexWriterData not found for " + companyId);
247             }
248         }
249     }
250 
251     public void write(IndexWriter writer) throws IOException {
252         if (SearchEngineUtil.isIndexReadOnly()) {
253             return;
254         }
255 
256         boolean writerFound = false;
257 
258         synchronized (this) {
259             if (!_writerLookup.isEmpty()) {
260                 for (IndexWriterData writerData : _writerLookup.values()) {
261                     if (writerData.getWriter() == writer) {
262                         writerFound = true;
263 
264                         decrement(writerData);
265 
266                         break;
267                     }
268                 }
269             }
270         }
271 
272         if (!writerFound) {
273             try {
274                 _optimizeCount++;
275 
276                 if ((_OPTIMIZE_INTERVAL == 0) ||
277                     (_optimizeCount >= _OPTIMIZE_INTERVAL)) {
278 
279                     writer.optimize();
280 
281                     _optimizeCount = 0;
282                 }
283             }
284             finally {
285                 writer.close();
286             }
287         }
288     }
289 
290     protected void decrement(IndexWriterData writerData) {
291         if (writerData.getCount() > 0) {
292             writerData.setCount(writerData.getCount() - 1);
293 
294             if (writerData.getCount() == 0) {
295                 _writerLookup.remove(writerData.getCompanyId());
296 
297                 try {
298                     IndexWriter writer = writerData.getWriter();
299 
300                     try {
301                         _optimizeCount++;
302 
303                         if ((_OPTIMIZE_INTERVAL == 0) ||
304                             (_optimizeCount >= _OPTIMIZE_INTERVAL)) {
305 
306                             writer.optimize();
307 
308                             _optimizeCount = 0;
309                         }
310                     }
311                     finally {
312                         writer.close();
313                     }
314                 }
315                 catch (Exception e) {
316                     _log.error(e, e);
317                 }
318                 finally {
319                     releaseLock(writerData.getCompanyId());
320                 }
321             }
322         }
323     }
324 
325     protected IndexWriter getReadOnlyIndexWriter() {
326         if (_readOnlyIndexWriter == null) {
327             try {
328                 if (_log.isInfoEnabled()) {
329                     _log.info("Disabling writing to index for this process");
330                 }
331 
332                 _readOnlyIndexWriter = new ReadOnlyIndexWriter(
333                     getReadOnlyLuceneDir(), new SimpleAnalyzer(), true);
334             }
335             catch (IOException ioe) {
336                 throw new RuntimeException(ioe);
337             }
338         }
339 
340         return _readOnlyIndexWriter;
341     }
342 
343     protected Directory getReadOnlyLuceneDir() throws IOException {
344         if (_readOnlyLuceneDir == null) {
345             String tmpDir = SystemProperties.get(SystemProperties.TMP_DIR);
346 
347             File dir = new File(tmpDir + "/liferay/lucene/empty");
348 
349             dir.mkdir();
350 
351             _readOnlyLuceneDir = LuceneUtil.getDirectory(dir.getPath(), false);
352         }
353 
354         return _readOnlyLuceneDir;
355     }
356 
357     private static final int _MERGE_FACTOR = GetterUtil.getInteger(
358         PropsUtil.get(PropsKeys.LUCENE_MERGE_FACTOR));
359 
360     private static final int _OPTIMIZE_INTERVAL = GetterUtil.getInteger(
361         PropsUtil.get(PropsKeys.LUCENE_OPTIMIZE_INTERVAL));
362 
363     private static Log _log = LogFactoryUtil.getLog(IndexWriterFactory.class);
364 
365     private FSDirectory _readOnlyLuceneDir = null;
366     private IndexWriter _readOnlyIndexWriter = null;
367     private Map<Long, Semaphore> _lockLookup = new HashMap<Long, Semaphore>();
368     private Map<Long, IndexWriterData> _writerLookup =
369         new HashMap<Long, IndexWriterData>();
370     private int _needExclusiveLock = 0;
371     private int _optimizeCount = 0;
372 
373 }