From 9b3847f49bafd4dfeea954dd011aabc0d5b6a307 Mon Sep 17 00:00:00 2001 From: Michael Gapczynski Date: Wed, 16 May 2012 21:51:45 -0400 Subject: [PATCH] Initial support for Amazon S3 storage backend --- apps/files_external/lib/amazons3.php | 236 +++++++++++++++++++++++++ apps/files_external/tests/amazons3.php | 50 ++++++ apps/files_external/tests/config.php | 6 + 3 files changed, 292 insertions(+) create mode 100644 apps/files_external/lib/amazons3.php create mode 100644 apps/files_external/tests/amazons3.php diff --git a/apps/files_external/lib/amazons3.php b/apps/files_external/lib/amazons3.php new file mode 100644 index 0000000000..e847ef143c --- /dev/null +++ b/apps/files_external/lib/amazons3.php @@ -0,0 +1,236 @@ +. +*/ + +require_once 'aws-sdk-1.5.5/sdk.class.php'; + +class OC_Filestorage_AmazonS3 extends OC_Filestorage_Common { + + private $s3; + private $bucket; + private $objects = array(); + + private static $tempFiles = array(); + + // TODO options: storage class, encryption server side, encrypt before upload? + + public function __construct($params) { + $this->s3 = new AmazonS3(array('key' => $params['key'], 'secret' => $params['secret'])); + $this->bucket = $params['bucket']; + } + + private function getObject($path) { + if (array_key_exists($path, $this->objects)) { + return $this->objects[$path]; + } else { + $response = $this->s3->get_object_metadata($this->bucket, $path); + if ($response) { + $this->objects[$path] = $response; + return $response; + // This object could be a folder, a '/' must be at the end of the path + } else if (substr($path, -1) != '/') { + $response = $this->s3->get_object_metadata($this->bucket, $path.'/'); + if ($response) { + $this->objects[$path] = $response; + return $response; + } + } + } + return false; + } + + public function mkdir($path) { + // Folders in Amazon S3 are 0 byte objects with a '/' at the end of the name + if (substr($path, -1) != '/') { + $path .= '/'; + } + $response = $this->s3->create_object($this->bucket, $path, array('body' => '')); + return $response->isOK(); + } + + public function rmdir($path) { + if (substr($path, -1) != '/') { + $path .= '/'; + } + return $this->unlink($path); + } + + public function opendir($path) { + if ($path == '' || $path == '/') { + // Use the '/' delimiter to only fetch objects inside the folder + $opt = array('delimiter' => '/'); + } else { + if (substr($path, -1) != '/') { + $path .= '/'; + } + $opt = array('delimiter' => '/', 'prefix' => $path); + } + $response = $this->s3->list_objects($this->bucket, $opt); + if ($response->isOK()) { + $files = array(); + foreach ($response->body->Contents as $object) { + // The folder being opened also shows up in the list of objects, don't add it to the files + if ($object->Key != $path) { + $files[] = basename($object->Key); + } + } + // Sub folders show up as CommonPrefixes + foreach ($response->body->CommonPrefixes as $object) { + $files[] = basename($object->Prefix); + } + OC_FakeDirStream::$dirs['amazons3'] = $files; + return opendir('fakedir://amazons3'); + } + return false; + } + + public function stat($path) { + if ($path == '' || $path == '/') { + $stat['size'] = $this->s3->get_bucket_filesize($this->bucket); + $stat['atime'] = time(); + $stat['mtime'] = $stat['atime']; + $stat['ctime'] = $stat['atime']; + } else if ($object = $this->getObject($path)) { + $stat['size'] = $object['Size']; + $stat['atime'] = time(); + $stat['mtime'] = strtotime($object['LastModified']); + $stat['ctime'] = $stat['mtime']; + } + if (isset($stat)) { + return $stat; + } + return false; + } + + public function filetype($path) { + if ($path == '' || $path == '/') { + return 'dir'; + } else if ($object = $this->getObject($path)) { + // Amazon S3 doesn't have typical folders, this is an alternative method to detect a folder + if (substr($object['Key'], -1) == '/' && $object['Size'] == 0) { + return 'dir'; + } else { + return 'file'; + } + } + return false; + } + + public function is_readable($path) { + // TODO Check acl and determine who grantee is + return true; + } + + public function is_writable($path) { + // TODO Check acl and determine who grantee is + return true; + } + + public function file_exists($path) { + if ($this->filetype($path) == 'dir' && substr($path, -1) != '/') { + $path .= '/'; + } + return $this->s3->if_object_exists($this->bucket, $path); + } + + public function unlink($path) { + $response = $this->s3->delete_object($this->bucket, $path); + return $response->isOK(); + } + + public function fopen($path, $mode) { + switch ($mode) { + case 'r': + case 'rb': + $tmpFile = OC_Helper::tmpFile(); + $handle = fopen($tmpFile, 'w'); + $response = $this->s3->get_object($this->bucket, $path, array('fileDownload' => $handle)); + if ($response->isOK()) { + return fopen($tmpFile, 'r'); + } + break; + case 'w': + case 'wb': + case 'a': + case 'ab': + case 'r+': + case 'w+': + case 'wb+': + case 'a+': + case 'x': + case 'x+': + case 'c': + case 'c+': + if (strrpos($path, '.') !== false) { + $ext = substr($path, strrpos($path, '.')); + } else { + $ext = ''; + } + $tmpFile = OC_Helper::tmpFile($ext); + OC_CloseStreamWrapper::$callBacks[$tmpFile] = array($this, 'writeBack'); + if ($this->file_exists($path)) { + $source = $this->fopen($path, 'r'); + file_put_contents($tmpFile, $source); + } + self::$tempFiles[$tmpFile] = $path; + return fopen('close://'.$tmpFile, $mode); + } + return false; + } + + public function writeBack($tmpFile) { + if (isset(self::$tempFiles[$tmpFile])) { + $handle = fopen($tmpFile, 'r'); + $response = $this->s3->create_object($this->bucket, self::$tempFiles[$tmpFile], array('fileUpload' => $handle)); + if ($response->isOK()) { + unlink($tmpFile); + } + } + } + + public function getMimeType($path) { + if ($this->filetype($path) == 'dir') { + return 'httpd/unix-directory'; + } else if ($object = $this->getObject($path)) { + return $object['ContentType']; + } + return false; + } + + public function free_space($path) { + // Infinite? + return false; + } + + public function touch($path, $mtime = null) { + if (is_null($mtime)) { + $mtime = time(); + } + if ($this->filetype($path) == 'dir' && substr($path, -1) != '/') { + $path .= '/'; + } + $response = $this->s3->update_object($this->bucket, $path, array('meta' => array('LastModified' => $mtime))); + return $response->isOK(); + } + +} + +?> \ No newline at end of file diff --git a/apps/files_external/tests/amazons3.php b/apps/files_external/tests/amazons3.php new file mode 100644 index 0000000000..d0084c94af --- /dev/null +++ b/apps/files_external/tests/amazons3.php @@ -0,0 +1,50 @@ +. +*/ + +$config = include('apps/files_external/tests/config.php'); +if (!is_array($config) or !isset($config['amazons3']) or !$config['amazons3']['run']) { + abstract class Test_Filestorage_AmazonS3 extends Test_FileStorage{} + return; +} else { + class Test_Filestorage_AmazonS3 extends Test_FileStorage { + + private $config; + private $id; + + public function setUp() { + $id = uniqid(); + $this->config = include('apps/files_external/tests/config.php'); + $this->config['amazons3']['bucket'] = $id; // Make sure we have a new empty bucket to work in + $this->instance = new OC_Filestorage_AmazonS3($this->config['amazons3']); + } + + public function tearDown() { + $s3 = new AmazonS3(array('key' => $this->config['amazons3']['key'], 'secret' => $this->config['amazons3']['secret'])); + if ($s3->delete_all_objects($this->id)) { + $s3->delete_bucket($this->id); + } + } + } +} + +?> + diff --git a/apps/files_external/tests/config.php b/apps/files_external/tests/config.php index 5b6517755f..2f24824223 100644 --- a/apps/files_external/tests/config.php +++ b/apps/files_external/tests/config.php @@ -29,4 +29,10 @@ return array( 'host'=>'localhost:8080/auth', 'root'=>'/', ), + 'amazons3'=>array( + 'run'=>false, + 'key'=>'test', + 'secret'=>'test', + 'bucket'=>'bucket', + ) );