nextcloud/apps/files_external/lib/Lib/Storage/AmazonS3.php

638 lines
15 KiB
PHP
Raw Normal View History

<?php
/**
2015-03-26 13:44:34 +03:00
* @author André Gaul <gaul@web-yard.de>
2016-05-26 20:56:05 +03:00
* @author Arthur Schiwon <blizzz@arthur-schiwon.de>
2015-03-26 13:44:34 +03:00
* @author Bart Visscher <bartv@thisnet.nl>
* @author Christian Berendt <berendt@b1-systems.de>
* @author Christopher T. Johnson <ctjctj@gmail.com>
* @author Johan Björk <johanimon@gmail.com>
* @author Jörn Friedrich Dreyer <jfd@butonic.de>
2015-10-05 21:54:56 +03:00
* @author Martin Mattel <martin.mattel@diemattels.at>
2015-03-26 13:44:34 +03:00
* @author Michael Gapczynski <GapczynskiM@gmail.com>
* @author Morris Jobke <hey@morrisjobke.de>
* @author Philipp Kapfer <philipp.kapfer@gmx.at>
* @author Robin Appelman <icewind@owncloud.com>
2016-01-12 17:02:16 +03:00
* @author Robin McCorkell <robin@mccorkell.me.uk>
2015-03-26 13:44:34 +03:00
* @author Thomas Müller <thomas.mueller@tmit.eu>
* @author Vincent Petry <pvince81@owncloud.com>
*
2016-01-12 17:02:16 +03:00
* @copyright Copyright (c) 2016, ownCloud, Inc.
2015-03-26 13:44:34 +03:00
* @license AGPL-3.0
*
2015-03-26 13:44:34 +03:00
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
2015-03-26 13:44:34 +03:00
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
2015-03-26 13:44:34 +03:00
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OCA\Files_External\Lib\Storage;
set_include_path(get_include_path() . PATH_SEPARATOR .
2014-05-16 00:47:28 +04:00
\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
require 'aws-autoloader.php';
use Aws\S3\S3Client;
use Aws\S3\Exception\S3Exception;
use Icewind\Streams\IteratorDirectory;
2012-09-07 20:30:48 +04:00
class AmazonS3 extends \OC\Files\Storage\Common {
2013-08-01 12:05:01 +04:00
/**
* @var \Aws\S3\S3Client
*/
private $connection;
2013-08-01 12:05:01 +04:00
/**
* @var string
*/
private $bucket;
2013-08-01 12:05:01 +04:00
/**
* @var array
*/
private static $tmpFiles = array();
2014-10-22 23:48:22 +04:00
/**
* @var array
*/
private $params;
2013-08-01 12:05:01 +04:00
/**
* @var bool
*/
private $test = false;
2013-08-01 12:05:01 +04:00
/**
* @var int
*/
private $timeout = 15;
/**
* @var int in seconds
*/
private $rescanDelay = 10;
2016-07-05 23:45:05 +03:00
/** @var string */
private $id;
/**
* @param string $path
2014-08-14 19:23:19 +04:00
* @return string correctly encoded path
*/
private function normalizePath($path) {
2013-07-08 13:14:47 +04:00
$path = trim($path, '/');
2013-07-10 18:12:48 +04:00
if (!$path) {
$path = '.';
}
return $path;
}
2014-10-06 13:08:21 +04:00
/**
* when running the tests wait to let the buckets catch up
*/
private function testTimeout() {
if ($this->test) {
sleep($this->timeout);
}
}
2014-08-01 16:20:00 +04:00
private function isRoot($path) {
return $path === '.';
}
private function cleanKey($path) {
if ($this->isRoot($path)) {
return '/';
}
return $path;
}
public function __construct($params) {
if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) {
2013-07-08 17:10:10 +04:00
throw new \Exception("Access Key, Secret and Bucket have to be configured.");
}
$this->id = 'amazon::' . $params['bucket'];
$this->updateLegacyId($params);
$this->bucket = $params['bucket'];
2013-08-17 14:58:10 +04:00
$this->test = isset($params['test']);
2014-08-01 16:20:00 +04:00
$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
$this->rescanDelay = (!isset($params['rescanDelay'])) ? 10 : $params['rescanDelay'];
$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
if (!isset($params['port']) || $params['port'] === '') {
$params['port'] = ($params['use_ssl'] === false) ? 80 : 443;
}
2014-10-22 23:48:22 +04:00
$this->params = $params;
}
/**
* Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
* TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
*
* @param array $params
*/
public function updateLegacyId (array $params) {
$oldId = 'amazon::' . $params['key'] . md5($params['secret']);
// find by old id or bucket
$stmt = \OC::$server->getDatabaseConnection()->prepare(
'SELECT `numeric_id`, `id` FROM `*PREFIX*storages` WHERE `id` IN (?, ?)'
);
$stmt->execute(array($oldId, $this->id));
while ($row = $stmt->fetch()) {
$storages[$row['id']] = $row['numeric_id'];
}
if (isset($storages[$this->id]) && isset($storages[$oldId])) {
// if both ids exist, delete the old storage and corresponding filecache entries
\OC\Files\Cache\Storage::remove($oldId);
} else if (isset($storages[$oldId])) {
// if only the old id exists do an update
$stmt = \OC::$server->getDatabaseConnection()->prepare(
'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
);
$stmt->execute(array($this->id, $oldId));
}
// only the bucket based id may exist, do nothing
}
/**
* Remove a file or folder
*
* @param string $path
* @return bool
*/
protected function remove($path) {
// remember fileType to reduce http calls
$fileType = $this->filetype($path);
if ($fileType === 'dir') {
return $this->rmdir($path);
} else if ($fileType === 'file') {
return $this->unlink($path);
} else {
return false;
}
}
public function mkdir($path) {
$path = $this->normalizePath($path);
2013-07-10 18:12:48 +04:00
if ($this->is_dir($path)) {
return false;
}
try {
$this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
2014-08-01 16:20:00 +04:00
'Key' => $path . '/',
'Body' => '',
'ContentType' => 'httpd/unix-directory'
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
return true;
}
public function file_exists($path) {
return $this->filetype($path) !== false;
}
public function rmdir($path) {
$path = $this->normalizePath($path);
if ($this->isRoot($path)) {
2014-08-01 16:20:00 +04:00
return $this->clearBucket();
}
2013-07-10 18:12:48 +04:00
if (!$this->file_exists($path)) {
return false;
}
2014-10-06 13:08:21 +04:00
return $this->batchDelete($path);
}
protected function clearBucket() {
try {
$this->getConnection()->clearBucket($this->bucket);
2014-10-06 13:08:21 +04:00
return true;
// clearBucket() is not working with Ceph, so if it fails we try the slower approach
} catch (\Exception $e) {
return $this->batchDelete();
}
return false;
}
private function batchDelete ($path = null) {
$params = array(
'Bucket' => $this->bucket
);
if ($path !== null) {
$params['Prefix'] = $path . '/';
}
try {
// Since there are no real directories on S3, we need
// to delete all objects prefixed with the path.
do {
// instead of the iterator, manually loop over the list ...
$objects = $this->getConnection()->listObjects($params);
2014-10-06 13:08:21 +04:00
// ... so we can delete the files in batches
$this->getConnection()->deleteObjects(array(
'Bucket' => $this->bucket,
'Objects' => $objects['Contents']
));
$this->testTimeout();
2014-10-06 13:08:21 +04:00
// we reached the end when the list is no longer truncated
} while ($objects['IsTruncated']);
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
return true;
}
public function opendir($path) {
$path = $this->normalizePath($path);
if ($this->isRoot($path)) {
$path = '';
} else {
$path .= '/';
}
try {
$files = array();
$result = $this->getConnection()->getIterator('ListObjects', array(
'Bucket' => $this->bucket,
'Delimiter' => '/',
'Prefix' => $path
), array('return_prefixes' => true));
foreach ($result as $object) {
if (isset($object['Key']) && $object['Key'] === $path) {
// it's the directory itself, skip
continue;
}
$file = basename(
isset($object['Key']) ? $object['Key'] : $object['Prefix']
);
$files[] = $file;
}
return IteratorDirectory::wrap($files);
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
}
public function stat($path) {
$path = $this->normalizePath($path);
try {
$stat = array();
if ($this->is_dir($path)) {
//folders don't really exist
$stat['size'] = -1; //unknown
$stat['mtime'] = time() - $this->rescanDelay * 1000;
} else {
$result = $this->getConnection()->headObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
$stat['size'] = $result['ContentLength'] ? $result['ContentLength'] : 0;
if ($result['Metadata']['lastmodified']) {
$stat['mtime'] = strtotime($result['Metadata']['lastmodified']);
} else {
$stat['mtime'] = strtotime($result['LastModified']);
}
}
$stat['atime'] = time();
return $stat;
} catch(S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
}
public function filetype($path) {
$path = $this->normalizePath($path);
if ($this->isRoot($path)) {
return 'dir';
}
try {
if ($this->getConnection()->doesObjectExist($this->bucket, $path)) {
return 'file';
}
if ($this->getConnection()->doesObjectExist($this->bucket, $path.'/')) {
return 'dir';
}
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
return false;
}
public function unlink($path) {
$path = $this->normalizePath($path);
2014-08-01 16:20:00 +04:00
if ($this->is_dir($path)) {
return $this->rmdir($path);
}
try {
$this->getConnection()->deleteObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
return true;
}
public function fopen($path, $mode) {
$path = $this->normalizePath($path);
switch ($mode) {
case 'r':
case 'rb':
$tmpFile = \OCP\Files::tmpFile();
self::$tmpFiles[$tmpFile] = $path;
try {
$this->getConnection()->getObject(array(
'Bucket' => $this->bucket,
'Key' => $path,
'SaveAs' => $tmpFile
));
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
return fopen($tmpFile, 'r');
case 'w':
case 'wb':
case 'a':
case 'ab':
case 'r+':
case 'w+':
case 'wb+':
case 'a+':
case 'x':
case 'x+':
case 'c':
case 'c+':
if (strrpos($path, '.') !== false) {
$ext = substr($path, strrpos($path, '.'));
} else {
$ext = '';
}
$tmpFile = \OCP\Files::tmpFile($ext);
\OC\Files\Stream\Close::registerCallback($tmpFile, array($this, 'writeBack'));
if ($this->file_exists($path)) {
$source = $this->fopen($path, 'r');
file_put_contents($tmpFile, $source);
}
self::$tmpFiles[$tmpFile] = $path;
return fopen('close://' . $tmpFile, $mode);
}
return false;
}
public function touch($path, $mtime = null) {
$path = $this->normalizePath($path);
$metadata = array();
if (is_null($mtime)) {
$mtime = time();
}
$metadata = [
'lastmodified' => gmdate(\Aws\Common\Enum\DateFormat::RFC1123, $mtime)
];
$fileType = $this->filetype($path);
try {
if ($fileType !== false) {
if ($fileType === 'dir' && ! $this->isRoot($path)) {
$path .= '/';
}
$this->getConnection()->copyObject([
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path),
'Metadata' => $metadata,
'CopySource' => $this->bucket . '/' . $path,
'MetadataDirective' => 'REPLACE',
]);
$this->testTimeout();
} else {
$mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
$this->getConnection()->putObject([
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path),
'Metadata' => $metadata,
2014-10-21 14:58:26 +04:00
'Body' => '',
'ContentType' => $mimeType,
'MetadataDirective' => 'REPLACE',
]);
$this->testTimeout();
}
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
return true;
}
public function copy($path1, $path2) {
$path1 = $this->normalizePath($path1);
$path2 = $this->normalizePath($path2);
if ($this->is_file($path1)) {
try {
$this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path2),
2014-08-14 19:23:19 +04:00
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
} else {
$this->remove($path2);
try {
$this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $path2 . '/',
2014-08-14 19:23:19 +04:00
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
$dh = $this->opendir($path1);
2014-08-01 16:20:00 +04:00
if (is_resource($dh)) {
while (($file = readdir($dh)) !== false) {
if (\OC\Files\Filesystem::isIgnoredDir($file)) {
continue;
}
$source = $path1 . '/' . $file;
$target = $path2 . '/' . $file;
$this->copy($source, $target);
}
}
}
return true;
}
public function rename($path1, $path2) {
$path1 = $this->normalizePath($path1);
$path2 = $this->normalizePath($path2);
if ($this->is_file($path1)) {
2014-08-01 16:57:43 +04:00
2013-07-10 18:09:22 +04:00
if ($this->copy($path1, $path2) === false) {
return false;
}
2013-07-10 18:09:22 +04:00
if ($this->unlink($path1) === false) {
$this->unlink($path2);
return false;
}
} else {
2013-07-10 18:09:22 +04:00
if ($this->copy($path1, $path2) === false) {
return false;
}
2013-07-10 18:12:48 +04:00
if ($this->rmdir($path1) === false) {
$this->rmdir($path2);
return false;
}
}
return true;
}
public function test() {
$test = $this->getConnection()->getBucketAcl(array(
2013-12-02 21:32:32 +04:00
'Bucket' => $this->bucket,
));
if (isset($test) && !is_null($test->getPath('Owner/ID'))) {
return true;
}
return false;
}
public function getId() {
return $this->id;
}
/**
* Returns the connection
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
*/
public function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}
$scheme = ($this->params['use_ssl'] === false) ? 'http' : 'https';
2014-10-22 23:48:22 +04:00
$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
$this->connection = S3Client::factory(array(
2014-10-22 23:48:22 +04:00
'key' => $this->params['key'],
'secret' => $this->params['secret'],
'base_url' => $base_url,
2015-09-16 04:02:59 +03:00
'region' => $this->params['region'],
S3Client::COMMAND_PARAMS => [
'PathStyle' => $this->params['use_path_style'],
],
));
if (!$this->connection->isValidBucketName($this->bucket)) {
throw new \Exception("The configured bucket name is invalid.");
}
if (!$this->connection->doesBucketExist($this->bucket)) {
try {
$this->connection->createBucket(array(
'Bucket' => $this->bucket
));
$this->connection->waitUntilBucketExists(array(
'Bucket' => $this->bucket,
'waiter.interval' => 1,
'waiter.max_attempts' => 15
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
throw new \Exception('Creation of bucket failed. '.$e->getMessage());
}
}
return $this->connection;
}
public function writeBack($tmpFile) {
2013-07-10 18:12:48 +04:00
if (!isset(self::$tmpFiles[$tmpFile])) {
return false;
}
try {
$this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey(self::$tmpFiles[$tmpFile]),
'SourceFile' => $tmpFile,
'ContentType' => \OC::$server->getMimeTypeDetector()->detect($tmpFile),
'ContentLength' => filesize($tmpFile)
));
$this->testTimeout();
unlink($tmpFile);
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
return false;
}
}
/**
* check if curl is installed
*/
public static function checkDependencies() {
return true;
}
}