1
22
23 package com.liferay.documentlibrary.util;
24
25 import com.liferay.documentlibrary.NoSuchFileException;
26 import com.liferay.portal.PortalException;
27 import com.liferay.portal.SystemException;
28 import com.liferay.portal.kernel.search.SearchException;
29 import com.liferay.portal.kernel.util.GetterUtil;
30 import com.liferay.portal.kernel.util.StringMaker;
31 import com.liferay.portal.kernel.util.StringPool;
32 import com.liferay.portal.kernel.util.Validator;
33 import com.liferay.portal.lucene.LuceneUtil;
34 import com.liferay.portal.util.PropsUtil;
35 import com.liferay.util.FileUtil;
36 import com.liferay.util.SystemProperties;
37 import com.liferay.util.servlet.ServletResponseUtil;
38
39 import java.io.File;
40 import java.io.FileInputStream;
41 import java.io.IOException;
42 import java.io.InputStream;
43
44 import java.util.ArrayList;
45 import java.util.Arrays;
46 import java.util.HashSet;
47 import java.util.Iterator;
48 import java.util.List;
49 import java.util.Set;
50
51 import org.apache.commons.id.uuid.UUID;
52 import org.apache.commons.logging.Log;
53 import org.apache.commons.logging.LogFactory;
54 import org.apache.lucene.document.Document;
55 import org.apache.lucene.index.IndexWriter;
56
57 import org.jets3t.service.S3Service;
58 import org.jets3t.service.S3ServiceException;
59 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
60 import org.jets3t.service.model.S3Bucket;
61 import org.jets3t.service.model.S3Object;
62 import org.jets3t.service.security.AWSCredentials;
63
64
71 public class S3Hook extends BaseHook {
72
73 public S3Hook() {
74 try {
75 _s3Service = getS3Service();
76 _s3Bucket = getS3Bucket();
77 }
78 catch (S3ServiceException s3se) {
79 _log.error(s3se.getMessage());
80 }
81 }
82
83 public void addDirectory(long companyId, long repositoryId, String dirName)
84 throws PortalException, SystemException {
85 }
86
87 public void addFile(
88 long companyId, String portletId, long groupId, long repositoryId,
89 String fileName, String properties, InputStream is)
90 throws PortalException, SystemException {
91
92 try {
93 S3Object s3Object = new S3Object(
94 _s3Bucket,
95 getKey(companyId, repositoryId, fileName, DEFAULT_VERSION));
96
97 s3Object.setDataInputStream(is);
98
99 _s3Service.putObject(_s3Bucket, s3Object);
100
101 Indexer.addFile(
102 companyId, portletId, groupId, repositoryId, fileName,
103 properties);
104 }
105 catch (IOException ioe) {
106 throw new SystemException(ioe);
107 }
108 catch (S3ServiceException s3se) {
109 throw new SystemException(s3se);
110 }
111 }
112
113 public void checkRoot(long companyId) throws SystemException {
114 }
115
116 public void deleteDirectory(
117 long companyId, String portletId, long repositoryId, String dirName)
118 throws PortalException, SystemException {
119
120 try {
121 S3Object[] s3Objects = _s3Service.listObjects(
122 _s3Bucket, getKey(companyId, repositoryId, dirName), null);
123
124 for (int i = 0; i < s3Objects.length; i++) {
125 S3Object s3Object = s3Objects[i];
126
127 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
128 }
129 }
130 catch (S3ServiceException s3se) {
131 throw new SystemException(s3se);
132 }
133 }
134
135 public void deleteFile(
136 long companyId, String portletId, long repositoryId,
137 String fileName)
138 throws PortalException, SystemException {
139
140 try {
141 S3Object[] s3Objects = _s3Service.listObjects(
142 _s3Bucket, getKey(companyId, repositoryId, fileName), null);
143
144 for (int i = 0; i < s3Objects.length; i++) {
145 S3Object s3Object = s3Objects[i];
146
147 _s3Service.deleteObject(_s3Bucket, s3Object.getKey());
148 }
149
150 Indexer.deleteFile(companyId, portletId, repositoryId, fileName);
151 }
152 catch (IOException ioe) {
153 throw new SystemException(ioe);
154 }
155 catch (S3ServiceException s3se) {
156 throw new SystemException(s3se);
157 }
158 }
159
160 public void deleteFile(
161 long companyId, String portletId, long repositoryId,
162 String fileName, double versionNumber)
163 throws PortalException, SystemException {
164
165 try {
166 _s3Service.deleteObject(
167 _s3Bucket,
168 getKey(companyId, repositoryId, fileName, versionNumber));
169 }
170 catch (S3ServiceException s3se) {
171 throw new SystemException(s3se);
172 }
173 }
174
175 public InputStream getFileAsStream(
176 long companyId, long repositoryId, String fileName,
177 double versionNumber)
178 throws PortalException, SystemException {
179
180 try {
181 if (versionNumber == 0) {
182 versionNumber = getHeadVersionNumber(
183 companyId, repositoryId, fileName);
184 }
185
186 S3Object s3Object = _s3Service.getObject(
187 _s3Bucket,
188 getKey(companyId, repositoryId, fileName, versionNumber));
189
190 return s3Object.getDataInputStream();
191 }
192 catch (S3ServiceException s3se) {
193 throw new SystemException(s3se);
194 }
195 }
196
197 public String[] getFileNames(
198 long companyId, long repositoryId, String dirName)
199 throws PortalException, SystemException {
200
201 try {
202 List list = new ArrayList();
203
204 S3Object[] s3Objects = _s3Service.listObjects(
205 _s3Bucket, getKey(companyId, repositoryId, dirName), null);
206
207 for (int i = 0; i < s3Objects.length; i++) {
208 S3Object s3Object = s3Objects[i];
209
210
213 String key = s3Object.getKey();
214
215 int x = key.indexOf(StringPool.SLASH);
216
217 x = key.indexOf(StringPool.SLASH, x + 1);
218
219 int y = key.lastIndexOf(StringPool.SLASH);
220
221 list.add(key.substring(x, y));
222 }
223
224 return (String[])list.toArray(new String[0]);
225 }
226 catch (S3ServiceException s3se) {
227 throw new SystemException(s3se);
228 }
229 }
230
231 public long getFileSize(
232 long companyId, long repositoryId, String fileName)
233 throws PortalException, SystemException {
234
235 try {
236 double versionNumber = getHeadVersionNumber(
237 companyId, repositoryId, fileName);
238
239 S3Object objectDetails = _s3Service.getObjectDetails(
240 _s3Bucket,
241 getKey(companyId, repositoryId, fileName, versionNumber));
242
243 return objectDetails.getContentLength();
244 }
245 catch (S3ServiceException s3se) {
246 throw new SystemException(s3se);
247 }
248 }
249
250 public boolean hasFile(
251 long companyId, long repositoryId, String fileName,
252 double versionNumber)
253 throws PortalException, SystemException {
254
255 try {
256 S3Object[] s3Objects = _s3Service.listObjects(
257 _s3Bucket,
258 getKey(companyId, repositoryId, fileName, versionNumber), null);
259
260 if (s3Objects.length == 0) {
261 return false;
262 }
263 else {
264 return true;
265 }
266 }
267 catch (S3ServiceException s3se) {
268 throw new SystemException(s3se);
269 }
270 }
271
272 public void move(String srcDir, String destDir) throws SystemException {
273 }
274
275 public void reIndex(String[] ids) throws SearchException {
276 long companyId = GetterUtil.getLong(ids[0]);
277 String portletId = ids[1];
278 long groupId = GetterUtil.getLong(ids[2]);
279 long repositoryId = GetterUtil.getLong(ids[3]);
280
281 IndexWriter writer = null;
282
283 try {
284 writer = LuceneUtil.getWriter(companyId);
285
286 S3Object[] searchObjects = _s3Service.listObjects(
287 _s3Bucket, getKey(companyId, repositoryId), null);
288
289 Set fileNameSet = new HashSet();
290
291 for (int i = 0; i < searchObjects.length; i++) {
292 S3Object currentObject = searchObjects[i];
293
294 String fileName = (String)getFileName(currentObject.getKey());
295
296 fileNameSet.add(fileName);
297 }
298
299 Iterator itr = fileNameSet.iterator();
300
301 while (itr.hasNext()) {
302 String fileName = (String)itr.next();
303
304 try {
305 Document doc = IndexerImpl.getAddFileDocument(
306 companyId, portletId, groupId, repositoryId, fileName);
307
308 writer.addDocument(doc);
309 }
310 catch (Exception e) {
311 _log.error("Reindexing " + fileName, e);
312 }
313 }
314 }
315 catch (IOException ioe) {
316 throw new SearchException(ioe);
317 }
318 catch (S3ServiceException s3se) {
319 throw new SearchException(s3se);
320 }
321 finally {
322 try {
323 if (writer != null) {
324 LuceneUtil.write(companyId);
325 }
326 }
327 catch (Exception e) {
328 _log.error(e);
329 }
330 }
331 }
332
333 public void updateFile(
334 long companyId, String portletId, long groupId, long repositoryId,
335 String fileName, double versionNumber, String sourceFileName,
336 String properties, InputStream is)
337 throws PortalException, SystemException {
338
339 try {
340 S3Object s3Object = new S3Object(
341 _s3Bucket,
342 getKey(companyId, repositoryId, fileName, versionNumber));
343
344 s3Object.setDataInputStream(is);
345
346 _s3Service.putObject(_s3Bucket, s3Object);
347
348 Indexer.updateFile(
349 companyId, portletId, groupId, repositoryId, fileName,
350 properties);
351 }
352 catch (IOException ioe) {
353 throw new SystemException(ioe);
354 }
355 catch (S3ServiceException s3se) {
356 throw new SystemException(s3se);
357 }
358 }
359
360 public void updateFile(
361 long companyId, String portletId, long groupId, long repositoryId,
362 long newRepositoryId, String fileName)
363 throws PortalException, SystemException {
364
365 try {
366 S3Object[] s3Objects = _s3Service.listObjects(
367 _s3Bucket, getKey(companyId, repositoryId, fileName), null);
368
369 for (int i = 0; i < s3Objects.length; i++) {
370 S3Object oldS3Object = s3Objects[i];
371
372 String oldKey = oldS3Object.getKey();
373
374 oldS3Object = _s3Service.getObject(_s3Bucket, oldKey);
375
376 File tempFile = new File(
377 SystemProperties.get(SystemProperties.TMP_DIR) +
378 File.separator + UUID.timeUUID());
379
380 InputStream is = null;
381
382 try {
383 is = oldS3Object.getDataInputStream();
384
385 FileUtil.write(tempFile, is);
386 }
387 catch (Exception e) {
388 }
389 finally {
390 ServletResponseUtil.cleanUp(is);
391 }
392
393 is = new FileInputStream(tempFile);
394
395 String newPrefix = getKey(companyId, newRepositoryId);
396
397 int x = oldKey.indexOf(StringPool.SLASH);
398
399 x = oldKey.indexOf(StringPool.SLASH, x + 1);
400
401 String newKey =
402 newPrefix + oldKey.substring(x + 1, oldKey.length());
403
404 S3Object newS3Object = new S3Object(
405 _s3Bucket, newKey);
406
407 newS3Object.setDataInputStream(is);
408
409 _s3Service.putObject(_s3Bucket, newS3Object);
410 _s3Service.deleteObject(_s3Bucket, oldKey);
411
412 FileUtil.delete(tempFile);
413 }
414
415 Indexer.deleteFile(
416 companyId, portletId, repositoryId, fileName);
417
418 Indexer.addFile(
419 companyId, portletId, groupId, newRepositoryId, fileName);
420 }
421 catch (IOException ioe) {
422 throw new SystemException(ioe);
423 }
424 catch (S3ServiceException s3se) {
425 throw new SystemException(s3se);
426 }
427 }
428
429 protected AWSCredentials getAWSCredentials() throws S3ServiceException {
430 if (Validator.isNull(_ACCESS_KEY) || Validator.isNull(_SECRET_KEY)) {
431 throw new S3ServiceException(
432 "S3 access and secret keys are not set");
433 }
434 else {
435 return new AWSCredentials(_ACCESS_KEY, _SECRET_KEY);
436 }
437 }
438
439 protected Object getFileName(String key) {
440 int x = key.indexOf(StringPool.SLASH);
441
442 x = key.indexOf(StringPool.SLASH, x + 1);
443
444 int y = key.lastIndexOf(StringPool.SLASH);
445
446 return key.substring(x + 1, y);
447 }
448
449 protected double getHeadVersionNumber(
450 long companyId, long repositoryId, String fileName)
451 throws PortalException, S3ServiceException {
452
453 S3Object[] s3Objects = _s3Service.listObjects(
454 _s3Bucket, getKey(companyId, repositoryId, fileName), null);
455
456 String[] keys = new String[s3Objects.length];
457
458 for (int i = 0; i < s3Objects.length; i++) {
459 S3Object s3Object = s3Objects[i];
460
461 keys[i] = s3Object.getKey();
462 }
463
464 if (keys.length > 0) {
465 Arrays.sort(keys);
466
467 String headKey = keys[keys.length - 1];
468
469 int x = headKey.lastIndexOf(StringPool.SLASH);
470
471 return GetterUtil.getDouble(
472 headKey.substring(x + 1, headKey.length()));
473 }
474 else {
475 throw new NoSuchFileException(fileName);
476 }
477 }
478
479 protected String getKey(long companyId, long repositoryId) {
480 StringMaker sm = new StringMaker();
481
482 sm.append(companyId);
483 sm.append(StringPool.SLASH);
484 sm.append(repositoryId);
485 sm.append(StringPool.SLASH);
486
487 return sm.toString();
488 }
489
490 protected String getKey(
491 long companyId, long repositoryId, String fileName) {
492
493 StringMaker sm = new StringMaker();
494
495 sm.append(companyId);
496 sm.append(StringPool.SLASH);
497 sm.append(repositoryId);
498 sm.append(StringPool.SLASH);
499 sm.append(fileName);
500 sm.append(StringPool.SLASH);
501
502 return sm.toString();
503 }
504
505 protected String getKey(
506 long companyId, long repositoryId, String fileName,
507 double versionNumber) {
508
509 StringMaker sm = new StringMaker();
510
511 sm.append(companyId);
512 sm.append(StringPool.SLASH);
513 sm.append(repositoryId);
514 sm.append(StringPool.SLASH);
515 sm.append(fileName);
516 sm.append(StringPool.SLASH);
517 sm.append(versionNumber);
518
519 return sm.toString();
520 }
521
522 protected S3Bucket getS3Bucket() throws S3ServiceException {
523 if (Validator.isNull(_BUCKET_NAME)) {
524 throw new S3ServiceException("S3 bucket name is not set");
525 }
526 else {
527 return getS3Service().createBucket(_BUCKET_NAME);
528 }
529 }
530
531 protected S3Service getS3Service() throws S3ServiceException {
532 AWSCredentials credentials = getAWSCredentials();
533
534 return new RestS3Service(credentials);
535 }
536
537 private static final String _ACCESS_KEY = PropsUtil.get(
538 PropsUtil.DL_HOOK_S3_ACCESS_KEY);
539
540 private static final String _SECRET_KEY = PropsUtil.get(
541 PropsUtil.DL_HOOK_S3_SECRET_KEY);
542
543 private static final String _BUCKET_NAME = PropsUtil.get(
544 PropsUtil.DL_HOOK_S3_BUCKET_NAME);
545
546 private static Log _log = LogFactory.getLog(S3Hook.class);
547
548 private S3Bucket _s3Bucket;
549 private S3Service _s3Service;
550
551 }