mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Lazy initialize external storages 

Fixed the following external storages to not connect in the constructor,
but do it on-demand when getConnection() is called.
- S3
- SWIFT
- SFTP
This commit is contained in:
Vincent Petry 2014-10-21 16:18:44 +02:00
parent a9aa7841c9
commit 075e8d8e86
6 changed files with 163 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
apps/files_external/lib/amazons3.php
View File

@ -111,34 +111,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$params['port'] = ($params['use_ssl'] === 'false') ? 80 : 443;
}
$base_url = $scheme . '://' . $params['hostname'] . ':' . $params['port'] . '/';
$this->connection = S3Client::factory(array(
'key' => $params['key'],
'secret' => $params['secret'],
'base_url' => $base_url,
'region' => $params['region']
));
if (!$this->connection->isValidBucketName($this->bucket)) {
throw new \Exception("The configured bucket name is invalid.");
}
if (!$this->connection->doesBucketExist($this->bucket)) {
try {
$this->connection->createBucket(array(
'Bucket' => $this->bucket
));
$this->connection->waitUntilBucketExists(array(
'Bucket' => $this->bucket,
'waiter.interval' => 1,
'waiter.max_attempts' => 15
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
throw new \Exception('Creation of bucket failed. '.$e->getMessage());
}
}
}
/**
@ -181,7 +153,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
$this->connection->putObject(array(
$this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
'Key' => $path . '/',
'ContentType' => 'httpd/unix-directory'
@ -216,7 +188,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
protected function clearBucket() {
try {
$this->connection->clearBucket($this->bucket);
$this->getConnection()->clearBucket($this->bucket);
return true;
// clearBucket() is not working with Ceph, so if it fails we try the slower approach
} catch (\Exception $e) {
@ -237,9 +209,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
// to delete all objects prefixed with the path.
do {
// instead of the iterator, manually loop over the list ...
$objects = $this->connection->listObjects($params);
$objects = $this->getConnection()->listObjects($params);
// ... so we can delete the files in batches
$this->connection->deleteObjects(array(
$this->getConnection()->deleteObjects(array(
'Bucket' => $this->bucket,
'Objects' => $objects['Contents']
));
@ -264,7 +236,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
try {
$files = array();
$result = $this->connection->getIterator('ListObjects', array(
$result = $this->getConnection()->getIterator('ListObjects', array(
'Bucket' => $this->bucket,
'Delimiter' => '/',
'Prefix' => $path
@ -299,7 +271,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$stat['size'] = -1; //unknown
$stat['mtime'] = time() - $this->rescanDelay * 1000;
} else {
$result = $this->connection->headObject(array(
$result = $this->getConnection()->headObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
@ -328,10 +300,10 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
if ($this->connection->doesObjectExist($this->bucket, $path)) {
if ($this->getConnection()->doesObjectExist($this->bucket, $path)) {
return 'file';
}
if ($this->connection->doesObjectExist($this->bucket, $path.'/')) {
if ($this->getConnection()->doesObjectExist($this->bucket, $path.'/')) {
return 'dir';
}
} catch (S3Exception $e) {
@ -350,7 +322,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
$this->connection->deleteObject(array(
$this->getConnection()->deleteObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
@ -373,7 +345,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
self::$tmpFiles[$tmpFile] = $path;
try {
$this->connection->getObject(array(
$this->getConnection()->getObject(array(
'Bucket' => $this->bucket,
'Key' => $path,
'SaveAs' => $tmpFile
@ -421,7 +393,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return 'httpd/unix-directory';
} else if ($this->file_exists($path)) {
try {
$result = $this->connection->headObject(array(
$result = $this->getConnection()->headObject(array(
'Bucket' => $this->bucket,
'Key' => $path
));
@ -449,7 +421,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
if ($fileType === 'dir' && ! $this->isRoot($path)) {
$path .= '/';
}
$this->connection->copyObject(array(
$this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path),
'Metadata' => $metadata,
@ -458,7 +430,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$this->testTimeout();
} else {
$mimeType = \OC_Helper::getMimetypeDetector()->detectPath($path);
$this->connection->putObject(array(
$this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path),
'Metadata' => $metadata,
@ -481,7 +453,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
if ($this->is_file($path1)) {
try {
$this->connection->copyObject(array(
$this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey($path2),
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
@ -495,7 +467,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
$this->remove($path2);
try {
$this->connection->copyObject(array(
$this->getConnection()->copyObject(array(
'Bucket' => $this->bucket,
'Key' => $path2 . '/',
'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
@ -553,7 +525,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
public function test() {
$test = $this->connection->getBucketAcl(array(
$test = $this->getConnection()->getBucketAcl(array(
'Bucket' => $this->bucket,
));
if (isset($test) && !is_null($test->getPath('Owner/ID'))) {
@ -566,7 +538,45 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $this->id;
}
/**
* Returns the connection
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
*/
public function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}
$this->connection = S3Client::factory(array(
'key' => $params['key'],
'secret' => $params['secret'],
'base_url' => $base_url,
'region' => $params['region']
));
if (!$this->connection->isValidBucketName($this->bucket)) {
throw new \Exception("The configured bucket name is invalid.");
}
if (!$this->connection->doesBucketExist($this->bucket)) {
try {
$this->connection->createBucket(array(
'Bucket' => $this->bucket
));
$this->connection->waitUntilBucketExists(array(
'Bucket' => $this->bucket,
'waiter.interval' => 1,
'waiter.max_attempts' => 15
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
throw new \Exception('Creation of bucket failed. '.$e->getMessage());
}
}
return $this->connection;
}
@ -576,7 +586,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
}
try {
$this->connection->putObject(array(
$this->getConnection()->putObject(array(
'Bucket' => $this->bucket,
'Key' => $this->cleanKey(self::$tmpFiles[$tmpFile]),
'SourceFile' => $tmpFile,

1
apps/files_external/lib/dropbox.php
View File

@ -44,6 +44,7 @@ class Dropbox extends \OC\Files\Storage\Common {
$this->id = 'dropbox::'.$params['app_key'] . $params['token']. '/' . $this->root;
$oauth = new \Dropbox_OAuth_Curl($params['app_key'], $params['app_secret']);
$oauth->setToken($params['token'], $params['token_secret']);
// note: Dropbox_API connection is lazy
$this->dropbox = new \Dropbox_API($oauth, 'auto');
} else {
throw new \Exception('Creating \OC\Files\Storage\Dropbox storage failed');

2
apps/files_external/lib/ftp.php
View File

@ -39,7 +39,7 @@ class FTP extends \OC\Files\Storage\StreamWrapper{
$this->root .= '/';
}
} else {
throw new \Exception();
throw new \Exception('Creating \OC\Files\Storage\FTP storage failed');
}
}

1
apps/files_external/lib/google.php
View File

@ -52,6 +52,7 @@ class Google extends \OC\Files\Storage\Common {
$client->setScopes(array('https://www.googleapis.com/auth/drive'));
$client->setUseObjects(true);
$client->setAccessToken($params['token']);
// note: API connection is lazy
$this->service = new \Google_DriveService($client);
$token = json_decode($params['token'], true);
$this->id = 'google::'.substr($params['client_id'], 0, 30).$token['created'];

39
apps/files_external/lib/sftp.php
View File

@ -53,6 +53,18 @@ class SFTP extends \OC\Files\Storage\Common {
if (substr($this->root, -1, 1) != '/') {
$this->root .= '/';
}
}
/**
* Returns the connection.
*
* @return \Net_SFTP connected client instance
* @throws \Exception when the connection failed
*/
public function getConnection() {
if (!is_null($this->client)) {
return $this->client;
}
$hostKeys = $this->readHostKeys();
$this->client = new \Net_SFTP($this->host);
@ -71,6 +83,7 @@ class SFTP extends \OC\Files\Storage\Common {
if (!$this->client->login($this->user, $this->password)) {
throw new \Exception('Login failed');
}
return $this->client;
}
public function test() {
@ -81,7 +94,7 @@ class SFTP extends \OC\Files\Storage\Common {
) {
return false;
}
return $this->client->nlist() !== false;
return $this->getConnection()->nlist() !== false;
}
public function getId(){
@ -149,7 +162,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function mkdir($path) {
try {
return $this->client->mkdir($this->absPath($path));
return $this->getConnection()->mkdir($this->absPath($path));
} catch (\Exception $e) {
return false;
}
@ -157,7 +170,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function rmdir($path) {
try {
return $this->client->delete($this->absPath($path), true);
return $this->getConnection()->delete($this->absPath($path), true);
} catch (\Exception $e) {
return false;
}
@ -165,7 +178,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function opendir($path) {
try {
$list = $this->client->nlist($this->absPath($path));
$list = $this->getConnection()->nlist($this->absPath($path));
if ($list === false) {
return false;
}
@ -186,7 +199,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function filetype($path) {
try {
$stat = $this->client->stat($this->absPath($path));
$stat = $this->getConnection()->stat($this->absPath($path));
if ($stat['type'] == NET_SFTP_TYPE_REGULAR) {
return 'file';
}
@ -202,7 +215,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function file_exists($path) {
try {
return $this->client->stat($this->absPath($path)) !== false;
return $this->getConnection()->stat($this->absPath($path)) !== false;
} catch (\Exception $e) {
return false;
}
@ -210,7 +223,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function unlink($path) {
try {
return $this->client->delete($this->absPath($path), true);
return $this->getConnection()->delete($this->absPath($path), true);
} catch (\Exception $e) {
return false;
}
@ -237,7 +250,7 @@ class SFTP extends \OC\Files\Storage\Common {
case 'x+':
case 'c':
case 'c+':
$context = stream_context_create(array('sftp' => array('session' => $this->client)));
$context = stream_context_create(array('sftp' => array('session' => $this->getConnection())));
return fopen($this->constructUrl($path), $mode, false, $context);
}
} catch (\Exception $e) {
@ -251,7 +264,7 @@ class SFTP extends \OC\Files\Storage\Common {
return false;
}
if (!$this->file_exists($path)) {
$this->client->put($this->absPath($path), '');
$this->getConnection()->put($this->absPath($path), '');
} else {
return false;
}
@ -262,11 +275,11 @@ class SFTP extends \OC\Files\Storage\Common {
}
public function getFile($path, $target) {
$this->client->get($path, $target);
$this->getConnection()->get($path, $target);
}
public function uploadFile($path, $target) {
$this->client->put($target, $path, NET_SFTP_LOCAL_FILE);
$this->getConnection()->put($target, $path, NET_SFTP_LOCAL_FILE);
}
public function rename($source, $target) {
@ -274,7 +287,7 @@ class SFTP extends \OC\Files\Storage\Common {
if (!$this->is_dir($target) && $this->file_exists($target)) {
$this->unlink($target);
}
return $this->client->rename(
return $this->getConnection()->rename(
$this->absPath($source),
$this->absPath($target)
);
@ -285,7 +298,7 @@ class SFTP extends \OC\Files\Storage\Common {
public function stat($path) {
try {
$stat = $this->client->stat($this->absPath($path));
$stat = $this->getConnection()->stat($this->absPath($path));
$mtime = $stat ? $stat['mtime'] : -1;
$size = $stat ? $stat['size'] : 0;

127
apps/files_external/lib/swift.php
View File

@ -48,6 +48,12 @@ class Swift extends \OC\Files\Storage\Common {
* @var string
*/
private $bucket;
/**
* Connection parameters
*
* @var array
*/
private $params;
/**
* @var array
*/
@ -86,7 +92,7 @@ class Swift extends \OC\Files\Storage\Common {
*/
private function doesObjectExist($path) {
try {
$this->container->getPartialObject($path);
$this->getContainer()->getPartialObject($path);
return true;
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@ -113,41 +119,7 @@ class Swift extends \OC\Files\Storage\Common {
$params['service_name'] = 'cloudFiles';
}
$settings = array(
'username' => $params['user'],
);
if (!empty($params['password'])) {
$settings['password'] = $params['password'];
} else if (!empty($params['key'])) {
$settings['apiKey'] = $params['key'];
}
if (!empty($params['tenant'])) {
$settings['tenantName'] = $params['tenant'];
}
if (!empty($params['timeout'])) {
$settings['timeout'] = $params['timeout'];
}
if (isset($settings['apiKey'])) {
$this->anchor = new Rackspace($params['url'], $settings);
} else {
$this->anchor = new OpenStack($params['url'], $settings);
}
$this->connection = $this->anchor->objectStoreService($params['service_name'], $params['region']);
try {
$this->container = $this->connection->getContainer($this->bucket);
} catch (ClientErrorResponseException $e) {
$this->container = $this->connection->createContainer($this->bucket);
}
if (!$this->file_exists('.')) {
$this->mkdir('.');
}
$this->params = $params;
}
public function mkdir($path) {
@ -165,7 +137,7 @@ class Swift extends \OC\Files\Storage\Common {
$customHeaders = array('content-type' => 'httpd/unix-directory');
$metadataHeaders = DataObject::stockHeaders(array());
$allHeaders = $customHeaders + $metadataHeaders;
$this->container->uploadObject($path, '', $allHeaders);
$this->getContainer()->uploadObject($path, '', $allHeaders);
} catch (Exceptions\CreateUpdateError $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@ -205,7 +177,7 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$this->container->dataObject()->setName($path . '/')->delete();
$this->getContainer()->dataObject()->setName($path . '/')->delete();
} catch (Exceptions\DeleteError $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@ -228,7 +200,7 @@ class Swift extends \OC\Files\Storage\Common {
try {
$files = array();
/** @var OpenCloud\Common\Collection $objects */
$objects = $this->container->objectList(array(
$objects = $this->getContainer()->objectList(array(
'prefix' => $path,
'delimiter' => '/'
));
@ -261,7 +233,7 @@ class Swift extends \OC\Files\Storage\Common {
try {
/** @var DataObject $object */
$object = $this->container->getPartialObject($path);
$object = $this->getContainer()->getPartialObject($path);
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@ -314,7 +286,7 @@ class Swift extends \OC\Files\Storage\Common {
}
try {
$this->container->dataObject()->setName($path)->delete();
$this->getContainer()->dataObject()->setName($path)->delete();
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@ -332,7 +304,7 @@ class Swift extends \OC\Files\Storage\Common {
$tmpFile = \OC_Helper::tmpFile();
self::$tmpFiles[$tmpFile] = $path;
try {
$object = $this->container->getObject($path);
$object = $this->getContainer()->getObject($path);
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
return false;
@ -385,7 +357,7 @@ class Swift extends \OC\Files\Storage\Common {
if ($this->is_dir($path)) {
return 'httpd/unix-directory';
} else if ($this->file_exists($path)) {
$object = $this->container->getPartialObject($path);
$object = $this->getContainer()->getPartialObject($path);
return $object->getContentType();
}
return false;
@ -402,7 +374,7 @@ class Swift extends \OC\Files\Storage\Common {
$path .= '/';
}
$object = $this->container->getPartialObject($path);
$object = $this->getContainer()->getPartialObject($path);
$object->saveMetadata($metadata);
return true;
} else {
@ -410,7 +382,7 @@ class Swift extends \OC\Files\Storage\Common {
$customHeaders = array('content-type' => $mimeType);
$metadataHeaders = DataObject::stockHeaders($metadata);
$allHeaders = $customHeaders + $metadataHeaders;
$this->container->uploadObject($path, '', $allHeaders);
$this->getContainer()->uploadObject($path, '', $allHeaders);
return true;
}
}
@ -426,7 +398,7 @@ class Swift extends \OC\Files\Storage\Common {
$this->unlink($path2);
try {
$source = $this->container->getPartialObject($path1);
$source = $this->getContainer()->getPartialObject($path1);
$source->copy($this->bucket . '/' . $path2);
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@ -439,7 +411,7 @@ class Swift extends \OC\Files\Storage\Common {
$this->unlink($path2);
try {
$source = $this->container->getPartialObject($path1 . '/');
$source = $this->getContainer()->getPartialObject($path1 . '/');
$source->copy($this->bucket . '/' . $path2 . '/');
} catch (ClientErrorResponseException $e) {
\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@ -497,16 +469,75 @@ class Swift extends \OC\Files\Storage\Common {
return $this->id;
}
/**
* Returns the connection
*
* @return OpenCloud\ObjectStore\Service connected client
* @throws \Exception if connection could not be made
*/
public function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}
$settings = array(
'username' => $this->params['user'],
);
if (!empty($this->params['password'])) {
$settings['password'] = $this->params['password'];
} else if (!empty($this->params['key'])) {
$settings['apiKey'] = $this->params['key'];
}
if (!empty($this->params['tenant'])) {
$settings['tenantName'] = $this->params['tenant'];
}
if (!empty($this->params['timeout'])) {
$settings['timeout'] = $this->params['timeout'];
}
if (isset($settings['apiKey'])) {
$this->anchor = new Rackspace($this->params['url'], $settings);
} else {
$this->anchor = new OpenStack($this->params['url'], $settings);
}
$this->connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']);
return $this->connection;
}
/**
* Returns the initialized object store container.
*
* @return OpenCloud\ObjectStore\Resource\Container
*/
public function getContainer() {
if (!is_null($this->container)) {
return $this->container;
}
try {
$this->container = $this->getConnection()->getContainer($this->bucket);
} catch (ClientErrorResponseException $e) {
$this->container = $this->getConnection()->createContainer($this->bucket);
}
if (!$this->file_exists('.')) {
$this->mkdir('.');
}
return $this->container;
}
public function writeBack($tmpFile) {
if (!isset(self::$tmpFiles[$tmpFile])) {
return false;
}
$fileData = fopen($tmpFile, 'r');
$this->container->uploadObject(self::$tmpFiles[$tmpFile], $fileData);
$this->getContainer()->uploadObject(self::$tmpFiles[$tmpFile], $fileData);
unlink($tmpFile);
}