Lazy initialize external storages

Fixed the following external storages to not connect in the constructor,
but do it on-demand when getConnection() is called.
- S3
- SWIFT
- SFTP

apps/files_external/lib/amazons3.php

@@ -111,34 +111,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 			$params['port'] = ($params['use_ssl'] === 'false') ? 80 : 443;
 		}
 		$base_url = $scheme . '://' . $params['hostname'] . ':' . $params['port'] . '/';
-
-		$this->connection = S3Client::factory(array(
-			'key' => $params['key'],
-			'secret' => $params['secret'],
-			'base_url' => $base_url,
-			'region' => $params['region']
-		));
-
-		if (!$this->connection->isValidBucketName($this->bucket)) {
-			throw new \Exception("The configured bucket name is invalid.");
-		}
-
-		if (!$this->connection->doesBucketExist($this->bucket)) {
-			try {
-				$this->connection->createBucket(array(
-					'Bucket' => $this->bucket
-				));
-				$this->connection->waitUntilBucketExists(array(
-					'Bucket' => $this->bucket,
-					'waiter.interval' => 1,
-					'waiter.max_attempts' => 15
-				));
-				$this->testTimeout();
-			} catch (S3Exception $e) {
-				\OCP\Util::logException('files_external', $e);
-				throw new \Exception('Creation of bucket failed. '.$e->getMessage());
-			}
-		}
 	}
 
 	/**
@@ -181,7 +153,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->connection->putObject(array(
+			$this->getConnection()->putObject(array(
 				'Bucket' => $this->bucket,
 				'Key' => $path . '/',
 				'ContentType' => 'httpd/unix-directory'
@@ -216,7 +188,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 
 	protected function clearBucket() {
 		try {
-			$this->connection->clearBucket($this->bucket);
+			$this->getConnection()->clearBucket($this->bucket);
 			return true;
 			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
 		} catch (\Exception $e) {
@@ -237,9 +209,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 			// to delete all objects prefixed with the path.
 			do {
 				// instead of the iterator, manually loop over the list ...
-				$objects = $this->connection->listObjects($params);
+				$objects = $this->getConnection()->listObjects($params);
 				// ... so we can delete the files in batches
-				$this->connection->deleteObjects(array(
+				$this->getConnection()->deleteObjects(array(
 					'Bucket' => $this->bucket,
 					'Objects' => $objects['Contents']
 				));
@@ -264,7 +236,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 
 		try {
 			$files = array();
-			$result = $this->connection->getIterator('ListObjects', array(
+			$result = $this->getConnection()->getIterator('ListObjects', array(
 				'Bucket' => $this->bucket,
 				'Delimiter' => '/',
 				'Prefix' => $path
@@ -299,7 +271,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 				$stat['size'] = -1; //unknown
 				$stat['mtime'] = time() - $this->rescanDelay * 1000;
 			} else {
-				$result = $this->connection->headObject(array(
+				$result = $this->getConnection()->headObject(array(
 					'Bucket' => $this->bucket,
 					'Key' => $path
 				));
@@ -328,10 +300,10 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			if ($this->connection->doesObjectExist($this->bucket, $path)) {
+			if ($this->getConnection()->doesObjectExist($this->bucket, $path)) {
 				return 'file';
 			}
-			if ($this->connection->doesObjectExist($this->bucket, $path.'/')) {
+			if ($this->getConnection()->doesObjectExist($this->bucket, $path.'/')) {
 				return 'dir';
 			}
 		} catch (S3Exception $e) {
@@ -350,7 +322,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->connection->deleteObject(array(
+			$this->getConnection()->deleteObject(array(
 				'Bucket' => $this->bucket,
 				'Key' => $path
 			));
@@ -373,7 +345,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 				self::$tmpFiles[$tmpFile] = $path;
 
 				try {
-					$this->connection->getObject(array(
+					$this->getConnection()->getObject(array(
 						'Bucket' => $this->bucket,
 						'Key' => $path,
 						'SaveAs' => $tmpFile
@@ -421,7 +393,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 			return 'httpd/unix-directory';
 		} else if ($this->file_exists($path)) {
 			try {
-				$result = $this->connection->headObject(array(
+				$result = $this->getConnection()->headObject(array(
 					'Bucket' => $this->bucket,
 					'Key' => $path
 				));
@@ -449,7 +421,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 				if ($fileType === 'dir' && ! $this->isRoot($path)) {
 					$path .= '/';
 				}
-				$this->connection->copyObject(array(
+				$this->getConnection()->copyObject(array(
 					'Bucket' => $this->bucket,
 					'Key' => $this->cleanKey($path),
 					'Metadata' => $metadata,
@@ -458,7 +430,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 				$this->testTimeout();
 			} else {
 				$mimeType = \OC_Helper::getMimetypeDetector()->detectPath($path);
-				$this->connection->putObject(array(
+				$this->getConnection()->putObject(array(
 					'Bucket' => $this->bucket,
 					'Key' => $this->cleanKey($path),
 					'Metadata' => $metadata,
@@ -481,7 +453,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 
 		if ($this->is_file($path1)) {
 			try {
-				$this->connection->copyObject(array(
+				$this->getConnection()->copyObject(array(
 					'Bucket' => $this->bucket,
 					'Key' => $this->cleanKey($path2),
 					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
@@ -495,7 +467,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 			$this->remove($path2);
 
 			try {
-				$this->connection->copyObject(array(
+				$this->getConnection()->copyObject(array(
 					'Bucket' => $this->bucket,
 					'Key' => $path2 . '/',
 					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
@@ -553,7 +525,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 	}
 
 	public function test() {
-		$test = $this->connection->getBucketAcl(array(
+		$test = $this->getConnection()->getBucketAcl(array(
 			'Bucket' => $this->bucket,
 		));
 		if (isset($test) && !is_null($test->getPath('Owner/ID'))) {
@@ -566,7 +538,45 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		return $this->id;
 	}
 
+	/**
+	 * Returns the connection
+	 *
+	 * @return S3Client connected client
+	 * @throws \Exception if connection could not be made
+	 */
 	public function getConnection() {
+		if (!is_null($this->connection)) {
+			return $this->connection;
+		}
+
+		$this->connection = S3Client::factory(array(
+			'key' => $params['key'],
+			'secret' => $params['secret'],
+			'base_url' => $base_url,
+			'region' => $params['region']
+		));
+
+		if (!$this->connection->isValidBucketName($this->bucket)) {
+			throw new \Exception("The configured bucket name is invalid.");
+		}
+
+		if (!$this->connection->doesBucketExist($this->bucket)) {
+			try {
+				$this->connection->createBucket(array(
+					'Bucket' => $this->bucket
+				));
+				$this->connection->waitUntilBucketExists(array(
+					'Bucket' => $this->bucket,
+					'waiter.interval' => 1,
+					'waiter.max_attempts' => 15
+				));
+				$this->testTimeout();
+			} catch (S3Exception $e) {
+				\OCP\Util::logException('files_external', $e);
+				throw new \Exception('Creation of bucket failed. '.$e->getMessage());
+			}
+		}
+
 		return $this->connection;
 	}
 
@@ -576,7 +586,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->connection->putObject(array(
+			$this->getConnection()->putObject(array(
 				'Bucket' => $this->bucket,
 				'Key' => $this->cleanKey(self::$tmpFiles[$tmpFile]),
 				'SourceFile' => $tmpFile,

apps/files_external/lib/dropbox.php

@@ -44,6 +44,7 @@ class Dropbox extends \OC\Files\Storage\Common {
 			$this->id = 'dropbox::'.$params['app_key'] . $params['token']. '/' . $this->root;
 			$oauth = new \Dropbox_OAuth_Curl($params['app_key'], $params['app_secret']);
 			$oauth->setToken($params['token'], $params['token_secret']);
+			// note: Dropbox_API connection is lazy
 			$this->dropbox = new \Dropbox_API($oauth, 'auto');
 		} else {
 			throw new \Exception('Creating \OC\Files\Storage\Dropbox storage failed');

apps/files_external/lib/ftp.php

@@ -39,7 +39,7 @@ class FTP extends \OC\Files\Storage\StreamWrapper{
 				$this->root .= '/';
 			}
 		} else {
-			throw new \Exception();
+			throw new \Exception('Creating \OC\Files\Storage\FTP storage failed');
 		}
 		
 	}

apps/files_external/lib/google.php

@@ -52,6 +52,7 @@ class Google extends \OC\Files\Storage\Common {
 			$client->setScopes(array('https://www.googleapis.com/auth/drive'));
 			$client->setUseObjects(true);
 			$client->setAccessToken($params['token']);
+			// note: API connection is lazy
 			$this->service = new \Google_DriveService($client);
 			$token = json_decode($params['token'], true);
 			$this->id = 'google::'.substr($params['client_id'], 0, 30).$token['created'];

apps/files_external/lib/sftp.php

@@ -53,6 +53,18 @@ class SFTP extends \OC\Files\Storage\Common {
 		if (substr($this->root, -1, 1) != '/') {
 			$this->root .= '/';
 		}
+	}
+
+	/**
+	 * Returns the connection.
+	 *
+	 * @return \Net_SFTP connected client instance
+	 * @throws \Exception when the connection failed
+	 */
+	public function getConnection() {
+		if (!is_null($this->client)) {
+			return $this->client;
+		}
 
 		$hostKeys = $this->readHostKeys();
 		$this->client = new \Net_SFTP($this->host);
@@ -71,6 +83,7 @@ class SFTP extends \OC\Files\Storage\Common {
 		if (!$this->client->login($this->user, $this->password)) {
 			throw new \Exception('Login failed');
 		}
+		return $this->client;
 	}
 
 	public function test() {
@@ -81,7 +94,7 @@ class SFTP extends \OC\Files\Storage\Common {
 		) {
 			return false;
 		}
-		return $this->client->nlist() !== false;
+		return $this->getConnection()->nlist() !== false;
 	}
 
 	public function getId(){
@@ -149,7 +162,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 	public function mkdir($path) {
 		try {
-			return $this->client->mkdir($this->absPath($path));
+			return $this->getConnection()->mkdir($this->absPath($path));
 		} catch (\Exception $e) {
 			return false;
 		}
@@ -157,7 +170,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 	public function rmdir($path) {
 		try {
-			return $this->client->delete($this->absPath($path), true);
+			return $this->getConnection()->delete($this->absPath($path), true);
 		} catch (\Exception $e) {
 			return false;
 		}
@@ -165,7 +178,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 	public function opendir($path) {
 		try {
-			$list = $this->client->nlist($this->absPath($path));
+			$list = $this->getConnection()->nlist($this->absPath($path));
 			if ($list === false) {
 				return false;
 			}
@@ -186,7 +199,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 	public function filetype($path) {
 		try {
-			$stat = $this->client->stat($this->absPath($path));
+			$stat = $this->getConnection()->stat($this->absPath($path));
 			if ($stat['type'] == NET_SFTP_TYPE_REGULAR) {
 				return 'file';
 			}
@@ -202,7 +215,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 	public function file_exists($path) {
 		try {
-			return $this->client->stat($this->absPath($path)) !== false;
+			return $this->getConnection()->stat($this->absPath($path)) !== false;
 		} catch (\Exception $e) {
 			return false;
 		}
@@ -210,7 +223,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 	public function unlink($path) {
 		try {
-			return $this->client->delete($this->absPath($path), true);
+			return $this->getConnection()->delete($this->absPath($path), true);
 		} catch (\Exception $e) {
 			return false;
 		}
@@ -237,7 +250,7 @@ class SFTP extends \OC\Files\Storage\Common {
 				case 'x+':
 				case 'c':
 				case 'c+':
-					$context = stream_context_create(array('sftp' => array('session' => $this->client)));
+					$context = stream_context_create(array('sftp' => array('session' => $this->getConnection())));
 					return fopen($this->constructUrl($path), $mode, false, $context);
 			}
 		} catch (\Exception $e) {
@@ -251,7 +264,7 @@ class SFTP extends \OC\Files\Storage\Common {
 				return false;
 			}
 			if (!$this->file_exists($path)) {
-				$this->client->put($this->absPath($path), '');
+				$this->getConnection()->put($this->absPath($path), '');
 			} else {
 				return false;
 			}
@@ -262,11 +275,11 @@ class SFTP extends \OC\Files\Storage\Common {
 	}
 
 	public function getFile($path, $target) {
-		$this->client->get($path, $target);
+		$this->getConnection()->get($path, $target);
 	}
 
 	public function uploadFile($path, $target) {
-		$this->client->put($target, $path, NET_SFTP_LOCAL_FILE);
+		$this->getConnection()->put($target, $path, NET_SFTP_LOCAL_FILE);
 	}
 
 	public function rename($source, $target) {
@@ -274,7 +287,7 @@ class SFTP extends \OC\Files\Storage\Common {
 			if (!$this->is_dir($target) && $this->file_exists($target)) {
 				$this->unlink($target);
 			}
-			return $this->client->rename(
+			return $this->getConnection()->rename(
 				$this->absPath($source),
 				$this->absPath($target)
 			);
@@ -285,7 +298,7 @@ class SFTP extends \OC\Files\Storage\Common {
 
 	public function stat($path) {
 		try {
-			$stat = $this->client->stat($this->absPath($path));
+			$stat = $this->getConnection()->stat($this->absPath($path));
 
 			$mtime = $stat ? $stat['mtime'] : -1;
 			$size = $stat ? $stat['size'] : 0;

apps/files_external/lib/swift.php

@@ -48,6 +48,12 @@ class Swift extends \OC\Files\Storage\Common {
 	 * @var string
 	 */
 	private $bucket;
+	/**
+	 * Connection parameters
+	 *
+	 * @var array
+	 */
+	private $params;
 	/**
 	 * @var array
 	 */
@@ -86,7 +92,7 @@ class Swift extends \OC\Files\Storage\Common {
 	 */
 	private function doesObjectExist($path) {
 		try {
-			$this->container->getPartialObject($path);
+			$this->getContainer()->getPartialObject($path);
 			return true;
 		} catch (ClientErrorResponseException $e) {
 			\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@@ -113,41 +119,7 @@ class Swift extends \OC\Files\Storage\Common {
 			$params['service_name'] = 'cloudFiles';
 		}
 
-		$settings = array(
+		$this->params = $params;
-			'username' => $params['user'],
-		);
-
-		if (!empty($params['password'])) {
-			$settings['password'] = $params['password'];
-		} else if (!empty($params['key'])) {
-			$settings['apiKey'] = $params['key'];
-		}
-
-		if (!empty($params['tenant'])) {
-			$settings['tenantName'] = $params['tenant'];
-		}
-
-		if (!empty($params['timeout'])) {
-			$settings['timeout'] = $params['timeout'];
-		}
-
-		if (isset($settings['apiKey'])) {
-			$this->anchor = new Rackspace($params['url'], $settings);
-		} else {
-			$this->anchor = new OpenStack($params['url'], $settings);
-		}
-
-		$this->connection = $this->anchor->objectStoreService($params['service_name'], $params['region']);
-
-		try {
-			$this->container = $this->connection->getContainer($this->bucket);
-		} catch (ClientErrorResponseException $e) {
-			$this->container = $this->connection->createContainer($this->bucket);
-		}
-
-		if (!$this->file_exists('.')) {
-			$this->mkdir('.');
-		}
 	}
 
 	public function mkdir($path) {
@@ -165,7 +137,7 @@ class Swift extends \OC\Files\Storage\Common {
 			$customHeaders = array('content-type' => 'httpd/unix-directory');
 			$metadataHeaders = DataObject::stockHeaders(array());
 			$allHeaders = $customHeaders + $metadataHeaders;
-			$this->container->uploadObject($path, '', $allHeaders);
+			$this->getContainer()->uploadObject($path, '', $allHeaders);
 		} catch (Exceptions\CreateUpdateError $e) {
 			\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
 			return false;
@@ -205,7 +177,7 @@ class Swift extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->container->dataObject()->setName($path . '/')->delete();
+			$this->getContainer()->dataObject()->setName($path . '/')->delete();
 		} catch (Exceptions\DeleteError $e) {
 			\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
 			return false;
@@ -228,7 +200,7 @@ class Swift extends \OC\Files\Storage\Common {
 		try {
 			$files = array();
 			/** @var OpenCloud\Common\Collection $objects */
-			$objects = $this->container->objectList(array(
+			$objects = $this->getContainer()->objectList(array(
 				'prefix' => $path,
 				'delimiter' => '/'
 			));
@@ -261,7 +233,7 @@ class Swift extends \OC\Files\Storage\Common {
 
 		try {
 			/** @var DataObject $object */
-			$object = $this->container->getPartialObject($path);
+			$object = $this->getContainer()->getPartialObject($path);
 		} catch (ClientErrorResponseException $e) {
 			\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
 			return false;
@@ -314,7 +286,7 @@ class Swift extends \OC\Files\Storage\Common {
 		}
 
 		try {
-			$this->container->dataObject()->setName($path)->delete();
+			$this->getContainer()->dataObject()->setName($path)->delete();
 		} catch (ClientErrorResponseException $e) {
 			\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
 			return false;
@@ -332,7 +304,7 @@ class Swift extends \OC\Files\Storage\Common {
 				$tmpFile = \OC_Helper::tmpFile();
 				self::$tmpFiles[$tmpFile] = $path;
 				try {
-					$object = $this->container->getObject($path);
+					$object = $this->getContainer()->getObject($path);
 				} catch (ClientErrorResponseException $e) {
 					\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
 					return false;
@@ -385,7 +357,7 @@ class Swift extends \OC\Files\Storage\Common {
 		if ($this->is_dir($path)) {
 			return 'httpd/unix-directory';
 		} else if ($this->file_exists($path)) {
-			$object = $this->container->getPartialObject($path);
+			$object = $this->getContainer()->getPartialObject($path);
 			return $object->getContentType();
 		}
 		return false;
@@ -402,7 +374,7 @@ class Swift extends \OC\Files\Storage\Common {
 				$path .= '/';
 			}
 
-			$object = $this->container->getPartialObject($path);
+			$object = $this->getContainer()->getPartialObject($path);
 			$object->saveMetadata($metadata);
 			return true;
 		} else {
@@ -410,7 +382,7 @@ class Swift extends \OC\Files\Storage\Common {
 			$customHeaders = array('content-type' => $mimeType);
 			$metadataHeaders = DataObject::stockHeaders($metadata);
 			$allHeaders = $customHeaders + $metadataHeaders;
-			$this->container->uploadObject($path, '', $allHeaders);
+			$this->getContainer()->uploadObject($path, '', $allHeaders);
 			return true;
 		}
 	}
@@ -426,7 +398,7 @@ class Swift extends \OC\Files\Storage\Common {
 			$this->unlink($path2);
 
 			try {
-				$source = $this->container->getPartialObject($path1);
+				$source = $this->getContainer()->getPartialObject($path1);
 				$source->copy($this->bucket . '/' . $path2);
 			} catch (ClientErrorResponseException $e) {
 				\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@@ -439,7 +411,7 @@ class Swift extends \OC\Files\Storage\Common {
 			$this->unlink($path2);
 
 			try {
-				$source = $this->container->getPartialObject($path1 . '/');
+				$source = $this->getContainer()->getPartialObject($path1 . '/');
 				$source->copy($this->bucket . '/' . $path2 . '/');
 			} catch (ClientErrorResponseException $e) {
 				\OCP\Util::writeLog('files_external', $e->getMessage(), \OCP\Util::ERROR);
@@ -497,16 +469,75 @@ class Swift extends \OC\Files\Storage\Common {
 		return $this->id;
 	}
 
+	/**
+	 * Returns the connection
+	 *
+	 * @return OpenCloud\ObjectStore\Service connected client
+	 * @throws \Exception if connection could not be made
+	 */
 	public function getConnection() {
+		if (!is_null($this->connection)) {
 			return $this->connection;
 		}
 
+		$settings = array(
+			'username' => $this->params['user'],
+		);
+
+		if (!empty($this->params['password'])) {
+			$settings['password'] = $this->params['password'];
+		} else if (!empty($this->params['key'])) {
+			$settings['apiKey'] = $this->params['key'];
+		}
+
+		if (!empty($this->params['tenant'])) {
+			$settings['tenantName'] = $this->params['tenant'];
+		}
+
+		if (!empty($this->params['timeout'])) {
+			$settings['timeout'] = $this->params['timeout'];
+		}
+
+		if (isset($settings['apiKey'])) {
+			$this->anchor = new Rackspace($this->params['url'], $settings);
+		} else {
+			$this->anchor = new OpenStack($this->params['url'], $settings);
+		}
+
+		$this->connection = $this->anchor->objectStoreService($this->params['service_name'], $this->params['region']);
+
+		return $this->connection;
+	}
+
+	/**
+	 * Returns the initialized object store container.
+	 *
+	 * @return OpenCloud\ObjectStore\Resource\Container
+	 */
+	public function getContainer() {
+		if (!is_null($this->container)) {
+			return $this->container;
+		}
+
+		try {
+			$this->container = $this->getConnection()->getContainer($this->bucket);
+		} catch (ClientErrorResponseException $e) {
+			$this->container = $this->getConnection()->createContainer($this->bucket);
+		}
+
+		if (!$this->file_exists('.')) {
+			$this->mkdir('.');
+		}
+
+		return $this->container;
+	}
+
 	public function writeBack($tmpFile) {
 		if (!isset(self::$tmpFiles[$tmpFile])) {
 			return false;
 		}
 		$fileData = fopen($tmpFile, 'r');
-		$this->container->uploadObject(self::$tmpFiles[$tmpFile], $fileData);
+		$this->getContainer()->uploadObject(self::$tmpFiles[$tmpFile], $fileData);
 		unlink($tmpFile);
 	}