mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #768 from nextcloud/s3-objectstore 

Add S3 objectstore backend
This commit is contained in:
Robin Appelman 2016-11-18 14:55:07 +01:00 committed by GitHub
parent 6c11c54434 4ac5fdcf11
commit 8b9ad46ba3
19 changed files with 827 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/files/js/filelist.js
View File

@ -2162,7 +2162,7 @@
self.filesClient.putFileContents(
targetPath,
'',
' ', // dont create empty files which fails on some storage backends
{
contentType: 'text/plain',
overwrite: true

108
apps/files_external/lib/Lib/Storage/AmazonS3.php
View File

@ -38,45 +38,30 @@ namespace OCA\Files_External\Lib\Storage;
set_include_path(get_include_path() . PATH_SEPARATOR .
\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
require 'aws-autoloader.php';
require_once 'aws-autoloader.php';
use Aws\S3\S3Client;
use Aws\S3\Exception\S3Exception;
use Icewind\Streams\IteratorDirectory;
use OC\Files\ObjectStore\S3ConnectionTrait;
class AmazonS3 extends \OC\Files\Storage\Common {
use S3ConnectionTrait;
/**
* @var \Aws\S3\S3Client
*/
private $connection;
/**
* @var string
*/
private $bucket;
/**
* @var array
*/
private static $tmpFiles = array();
/**
* @var array
*/
private $params;
/**
* @var bool
*/
private $test = false;
/**
* @var int
*/
private $timeout = 15;
/**
* @var int in seconds
*/
private $rescanDelay = 10;
/** @var string */
private $id;
public function __construct($parameters) {
parent::__construct($parameters);
$this->parseParams($parameters);
}
/**
* @param string $path
@ -92,15 +77,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $path;
}
/**
* when running the tests wait to let the buckets catch up
*/
private function testTimeout() {
if ($this->test) {
sleep($this->timeout);
}
}
private function isRoot($path) {
return $path === '.';
}
@ -112,26 +88,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $path;
}
public function __construct($params) {
if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) {
throw new \Exception("Access Key, Secret and Bucket have to be configured.");
}
$this->id = 'amazon::' . $params['bucket'];
$this->updateLegacyId($params);
$this->bucket = $params['bucket'];
$this->test = isset($params['test']);
$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
$this->rescanDelay = (!isset($params['rescanDelay'])) ? 10 : $params['rescanDelay'];
$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
if (!isset($params['port']) || $params['port'] === '') {
$params['port'] = ($params['use_ssl'] === false) ? 80 : 443;
}
$this->params = $params;
}
/**
* Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
* TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
@ -558,54 +514,6 @@ class AmazonS3 extends \OC\Files\Storage\Common {
return $this->id;
}
/**
* Returns the connection
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
*/
public function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}
$scheme = ($this->params['use_ssl'] === false) ? 'http' : 'https';
$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
$this->connection = S3Client::factory(array(
'key' => $this->params['key'],
'secret' => $this->params['secret'],
'base_url' => $base_url,
'region' => $this->params['region'],
S3Client::COMMAND_PARAMS => [
'PathStyle' => $this->params['use_path_style'],
],
));
if (!$this->connection->isValidBucketName($this->bucket)) {
throw new \Exception("The configured bucket name is invalid.");
}
if (!$this->connection->doesBucketExist($this->bucket)) {
try {
$this->connection->createBucket(array(
'Bucket' => $this->bucket
));
$this->connection->waitUntilBucketExists(array(
'Bucket' => $this->bucket,
'waiter.interval' => 1,
'waiter.max_attempts' => 15
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
throw new \Exception('Creation of bucket failed. '.$e->getMessage());
}
}
return $this->connection;
}
public function writeBack($tmpFile) {
if (!isset(self::$tmpFiles[$tmpFile])) {
return false;

27
apps/theming/lib/Controller/ThemingController.php
View File

@ -32,9 +32,12 @@ use OCP\AppFramework\Controller;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\DataDownloadResponse;
use OCP\AppFramework\Http\DataResponse;
use OCP\AppFramework\Http\NotFoundResponse;
use OCP\AppFramework\Http\StreamResponse;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\Files\File;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\IConfig;
use OCP\IL10N;
use OCP\IRequest;
@ -255,15 +258,17 @@ class ThemingController extends Controller {
* @PublicPage
* @NoCSRFRequired
*
* @return StreamResponse|DataResponse
* @return StreamResponse|NotFoundResponse
*/
public function getLogo() {
$pathToLogo = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/') . '/themedinstancelogo';
if(!file_exists($pathToLogo)) {
return new DataResponse();
try {
/** @var File $file */
$file = $this->rootFolder->get('themedinstancelogo');
} catch (NotFoundException $e) {
return new NotFoundResponse();
}
$response = new Http\StreamResponse($pathToLogo);
$response = new Http\StreamResponse($file->fopen('r'));
$response->cacheFor(3600);
$response->addHeader('Expires', date(\DateTime::RFC2822, $this->timeFactory->getTime()));
$response->addHeader('Content-Disposition', 'attachment');
@ -276,15 +281,17 @@ class ThemingController extends Controller {
* @PublicPage
* @NoCSRFRequired
*
* @return StreamResponse|DataResponse
* @return StreamResponse|NotFoundResponse
*/
public function getLoginBackground() {
$pathToLogo = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/') . '/themedbackgroundlogo';
if(!file_exists($pathToLogo)) {
return new DataResponse();
try {
/** @var File $file */
$file = $this->rootFolder->get('themedbackgroundlogo');
} catch (NotFoundException $e) {
return new NotFoundResponse();
}
$response = new StreamResponse($pathToLogo);
$response = new StreamResponse($file->fopen('r'));
$response->cacheFor(3600);
$response->addHeader('Expires', date(\DateTime::RFC2822, $this->timeFactory->getTime()));
$response->addHeader('Content-Disposition', 'attachment');

49
apps/theming/tests/Controller/ThemingControllerTest.php
View File

@ -28,7 +28,9 @@ use OCA\Theming\Controller\ThemingController;
use OCA\Theming\Util;
use OCP\AppFramework\Http;
use OCP\AppFramework\Http\DataResponse;
use OCP\Files\File;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\IConfig;
use OCP\IL10N;
use OCP\IRequest;
@ -338,26 +340,30 @@ class ThemingControllerTest extends TestCase {
}
public function testGetLogoNotExistent() {
$expected = new DataResponse();
$this->rootFolder->method('get')
->with($this->equalTo('themedinstancelogo'))
->willThrowException(new NotFoundException());
$expected = new Http\NotFoundResponse();
$this->assertEquals($expected, $this->themingController->getLogo());
}
public function testGetLogo() {
$dataFolder = \OC::$server->getTempManager()->getTemporaryFolder();
$tmpLogo = $dataFolder . '/themedinstancelogo';
touch($tmpLogo);
$this->config
->expects($this->once())
->method('getSystemValue')
->with('datadirectory', \OC::$SERVERROOT . '/data/')
->willReturn($dataFolder);
$file = $this->createMock(File::class);
$this->rootFolder->method('get')
->with('themedinstancelogo')
->willReturn($file);
$file->method('fopen')
->with('r')
->willReturn('mypath');
$this->config
->expects($this->once())
->method('getAppValue')
->with('theming', 'logoMime', '')
->willReturn('text/svg');
@$expected = new Http\StreamResponse($tmpLogo);
@$expected = new Http\StreamResponse('mypath');
$expected->cacheFor(3600);
$expected->addHeader('Expires', date(\DateTime::RFC2822, 123));
$expected->addHeader('Content-Disposition', 'attachment');
@ -368,26 +374,29 @@ class ThemingControllerTest extends TestCase {
public function testGetLoginBackgroundNotExistent() {
$expected = new DataResponse();
$this->rootFolder->method('get')
->with('themedbackgroundlogo')
->willThrowException(new NotFoundException());
$expected = new Http\NotFoundResponse();
$this->assertEquals($expected, $this->themingController->getLoginBackground());
}
public function testGetLoginBackground() {
$dataFolder = \OC::$server->getTempManager()->getTemporaryFolder();
$tmpLogo = $dataFolder . '/themedbackgroundlogo';
touch($tmpLogo);
$this->config
->expects($this->once())
->method('getSystemValue')
->with('datadirectory', \OC::$SERVERROOT . '/data/')
->willReturn($dataFolder);
$file = $this->createMock(File::class);
$this->rootFolder->method('get')
->with('themedbackgroundlogo')
->willReturn($file);
$file->method('fopen')
->with('r')
->willReturn('mypath');
$this->config
->expects($this->once())
->method('getAppValue')
->with('theming', 'backgroundMime', '')
->willReturn('image/png');
@$expected = new Http\StreamResponse($tmpLogo);
@$expected = new Http\StreamResponse('mypath');
$expected->cacheFor(3600);
$expected->addHeader('Expires', date(\DateTime::RFC2822, 123));
$expected->addHeader('Content-Disposition', 'attachment');

9
lib/private/AppFramework/Http/Output.php
View File

@ -48,12 +48,17 @@ class Output implements IOutput {
}
/**
* @param string $path
* @param string|resource $path or file handle
*
* @return bool false if an error occurred
*/
public function setReadfile($path) {
return @readfile($path);
if (is_resource($path)) {
$output = fopen('php://output', 'w');
return stream_copy_to_stream($path, $output) > 0;
} else {
return @readfile($path);
}
}
/**

7
lib/private/Files/FileInfo.php
View File

@ -33,6 +33,7 @@ namespace OC\Files;
use OCP\Files\Cache\ICacheEntry;
use OCP\Files\Mount\IMountPoint;
use OCP\Files\Storage\IStorage;
use OCP\Files\IHomeStorage;
use OCP\IUser;
class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
@ -305,7 +306,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
}
public function isMounted() {
$sid = $this->getStorage()->getId();
$storage = $this->getStorage();
if ($storage->instanceOfStorage('\OCP\Files\IHomeStorage')) {
return false;
}
$sid = $storage->getId();
if (!is_null($sid)) {
$sid = explode(':', $sid);
return ($sid[0] !== 'home' and $sid[0] !== 'shared');

2
lib/private/Files/Mount/ObjectHomeMountProvider.php
View File

@ -51,7 +51,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider {
*
* @param IUser $user
* @param IStorageFactory $loader
* @return \OCP\Files\Mount\IMountPoint[]
* @return \OCP\Files\Mount\IMountPoint
*/
public function getHomeMountForUser(IUser $user, IStorageFactory $loader) {

107
lib/private/Files/ObjectStore/S3.php Normal file
View File

@ -0,0 +1,107 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;
use OCP\Files\ObjectStore\IObjectStore;
// TODO: proper composer
set_include_path(get_include_path() . PATH_SEPARATOR .
\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
require_once 'aws-autoloader.php';
class S3 implements IObjectStore {
use S3ConnectionTrait;
public function __construct($parameters) {
$this->parseParams($parameters);
}
/**
* @return string the container or bucket name where objects are stored
* @since 7.0.0
*/
function getStorageId() {
return $this->id;
}
/**
* @param string $urn the unified resource name used to identify the object
* @return resource stream with the read data
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
function readObject($urn) {
// Create the command and serialize the request
$request = $this->getConnection()->getCommand('GetObject', [
'Bucket' => $this->bucket,
'Key' => $urn
])->prepare();
$request->dispatch('request.before_send', array(
'request' => $request
));
$headers = $request->getHeaderLines();
$headers[] = 'Connection: close';
$opts = [
'http' => [
'method' => "GET",
'header' => $headers
],
'ssl' => [
'verify_peer' => true
]
];
$context = stream_context_create($opts);
return fopen($request->getUrl(), 'r', false, $context);
}
/**
* @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
function writeObject($urn, $stream) {
$this->getConnection()->putObject([
'Bucket' => $this->bucket,
'Key' => $urn,
'Body' => $stream
]);
}
/**
* @param string $urn the unified resource name used to identify the object
* @return void
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
function deleteObject($urn) {
$this->getConnection()->deleteObject([
'Bucket' => $this->bucket,
'Key' => $urn
]);
}
}

124
lib/private/Files/ObjectStore/S3ConnectionTrait.php Normal file
View File

@ -0,0 +1,124 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;
use Aws\S3\Exception\S3Exception;
use Aws\S3\S3Client;
trait S3ConnectionTrait {
/** @var array */
protected $params;
/** @var S3Client */
protected $connection;
/** @var string */
protected $id;
/** @var string */
protected $bucket;
/** @var int */
protected $timeout;
protected $test;
protected function parseParams($params) {
if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) {
throw new \Exception("Access Key, Secret and Bucket have to be configured.");
}
$this->id = 'amazon::' . $params['bucket'];
$this->test = isset($params['test']);
$this->bucket = $params['bucket'];
$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
if (!isset($params['port']) || $params['port'] === '') {
$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
}
$this->params = $params;
}
/**
* Returns the connection
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
*/
protected function getConnection() {
if (!is_null($this->connection)) {
return $this->connection;
}
$scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https';
$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
$options = [
'key' => $this->params['key'],
'secret' => $this->params['secret'],
'base_url' => $base_url,
'region' => $this->params['region'],
S3Client::COMMAND_PARAMS => [
'PathStyle' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false,
]
];
if (isset($this->params['proxy'])) {
$options[S3Client::REQUEST_OPTIONS] = ['proxy' => $this->params['proxy']];
}
$this->connection = S3Client::factory($options);
if (!$this->connection->isValidBucketName($this->bucket)) {
throw new \Exception("The configured bucket name is invalid.");
}
if (!$this->connection->doesBucketExist($this->bucket)) {
try {
$this->connection->createBucket(array(
'Bucket' => $this->bucket
));
$this->connection->waitUntilBucketExists(array(
'Bucket' => $this->bucket,
'waiter.interval' => 1,
'waiter.max_attempts' => 15
));
$this->testTimeout();
} catch (S3Exception $e) {
\OCP\Util::logException('files_external', $e);
throw new \Exception('Creation of bucket failed. ' . $e->getMessage());
}
}
return $this->connection;
}
/**
* when running the tests wait to let the buckets catch up
*/
private function testTimeout() {
if ($this->test) {
sleep($this->timeout);
}
}
}

90
lib/private/Files/ObjectStore/StorageObjectStore.php Normal file
View File

@ -0,0 +1,90 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\Storage\IStorage;
/**
* Object store that wraps a storage backend, mostly for testing purposes
*/
class StorageObjectStore implements IObjectStore {
/** @var IStorage */
private $storage;
/**
* @param IStorage $storage
*/
public function __construct(IStorage $storage) {
$this->storage = $storage;
}
/**
* @return string the container or bucket name where objects are stored
* @since 7.0.0
*/
function getStorageId() {
$this->storage->getId();
}
/**
* @param string $urn the unified resource name used to identify the object
* @return resource stream with the read data
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
function readObject($urn) {
$handle = $this->storage->fopen($urn, 'r');
if ($handle) {
return $handle;
} else {
throw new \Exception();
}
}
/**
* @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
function writeObject($urn, $stream) {
$handle = $this->storage->fopen($urn, 'w');
if ($handle) {
stream_copy_to_stream($stream, $handle);
fclose($handle);
} else {
throw new \Exception();
}
}
/**
* @param string $urn the unified resource name used to identify the object
* @return void
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
function deleteObject($urn) {
$this->storage->unlink($urn);
}
}

2
lib/public/AppFramework/Http/IOutput.php
View File

@ -39,7 +39,7 @@ interface IOutput {
public function setOutput($out);
/**
* @param string $path
* @param string|resource $path or file handle
*
* @return bool false if an error occurred
* @since 8.1.0

4
lib/public/AppFramework/Http/StreamResponse.php
View File

@ -37,7 +37,7 @@ class StreamResponse extends Response implements ICallbackResponse {
private $filePath;
/**
* @param string $filePath the path to the file which should be streamed
* @param string|resource $filePath the path to the file or a file handle which should be streamed
* @since 8.1.0
*/
public function __construct ($filePath) {
@ -54,7 +54,7 @@ class StreamResponse extends Response implements ICallbackResponse {
public function callback (IOutput $output) {
// handle caching
if ($output->getHttpResponseCode() !== Http::STATUS_NOT_MODIFIED) {
if (!file_exists($this->filePath)) {
if (!(file_exists($this->filePath) || is_resource($this->filePath))) {
$output->setHttpResponseCode(Http::STATUS_NOT_FOUND);
} elseif ($output->setReadfile($this->filePath) === false) {
$output->setHttpResponseCode(Http::STATUS_BAD_REQUEST);

31
tests/lib/AppFramework/Http/OutputTest.php Normal file
View File

@ -0,0 +1,31 @@
<?php
/**
* Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test\AppFramework\Http;
use OC\AppFramework\Http\Output;
class OutputTest extends \Test\TestCase {
public function testSetOutput() {
$this->expectOutputString('foo');
$output = new Output('');
$output->setOutput('foo');
}
public function testSetReadfile() {
$this->expectOutputString(file_get_contents(__FILE__));
$output = new Output('');
$output->setReadfile(__FILE__);
}
public function testSetReadfileStream() {
$this->expectOutputString(file_get_contents(__FILE__));
$output = new Output('');
$output->setReadfile(fopen(__FILE__, 'r'));
}
}

46
tests/lib/Files/FileInfoTest.php Normal file
View File

@ -0,0 +1,46 @@
<?php
/**
* Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test\Files;
use OC\AllConfig;
use OC\Files\FileInfo;
use OC\Files\Storage\Home;
use OC\Files\Storage\Temporary;
use OC\User\User;
use OCP\IConfig;
use Test\TestCase;
use Test\Traits\UserTrait;
class FileInfoTest extends TestCase {
use UserTrait;
private $config;
public function setUp() {
parent::setUp();
$this->createUser('foo', 'foo');
$this->config = $this->getMockBuilder(IConfig::class)->getMock();
}
public function testIsMountedHomeStorage() {
$fileInfo = new FileInfo(
'',
new Home(['user' => new User('foo', $this->userBackend, null, $this->config)]),
'', [], null);
$this->assertFalse($fileInfo->isMounted());
}
public function testIsMountedNonHomeStorage() {
$fileInfo = new FileInfo(
'',
new Temporary(),
'', [], null);
$this->assertTrue($fileInfo->isMounted());
}
}

36
tests/lib/Files/ObjectStore/LocalTest.php Normal file
View File

@ -0,0 +1,36 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Files\ObjectStore;
use OC\Files\ObjectStore\StorageObjectStore;
use OC\Files\Storage\Temporary;
class LocalTest extends ObjectStoreTest {
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
protected function getInstance() {
$storage = new Temporary();
return new StorageObjectStore($storage);
}
}

169
tests/lib/Files/ObjectStore/ObjectStoreStorageTest.php Normal file
View File

@ -0,0 +1,169 @@
<?php
/**
* @author Jörn Friedrich Dreyer
* @copyright (c) 2014 Jörn Friedrich Dreyer <jfd@owncloud.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
* License as published by the Free Software Foundation; either
* version 3 of the License, or any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU Affero General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Files\ObjectStore;
use OC\Files\ObjectStore\ObjectStoreStorage;
use OC\Files\ObjectStore\StorageObjectStore;
use OC\Files\Storage\Temporary;
use OCP\Files\ObjectStore\IObjectStore;
use Test\Files\Storage\Storage;
/**
* @group DB
*/
class ObjectStoreStorageTest extends Storage {
/**
* @var IObjectStore
*/
private $objectStorage;
protected function setUp() {
parent::setUp();
$baseStorage = new Temporary();
$this->objectStorage = new StorageObjectStore($baseStorage);
$config['objectstore'] = $this->objectStorage;
$this->instance = new ObjectStoreStorage($config);
}
protected function tearDown() {
if (is_null($this->instance)) {
return;
}
$this->instance->getCache()->clear();
parent::tearDown();
}
public function testStat() {
$textFile = \OC::$SERVERROOT . '/tests/data/lorem.txt';
$ctimeStart = time();
$this->instance->file_put_contents('/lorem.txt', file_get_contents($textFile));
$this->assertTrue($this->instance->isReadable('/lorem.txt'));
$ctimeEnd = time();
$mTime = $this->instance->filemtime('/lorem.txt');
// check that ($ctimeStart - 5) <= $mTime <= ($ctimeEnd + 1)
$this->assertGreaterThanOrEqual(($ctimeStart - 5), $mTime);
$this->assertLessThanOrEqual(($ctimeEnd + 1), $mTime);
$this->assertEquals(filesize($textFile), $this->instance->filesize('/lorem.txt'));
$stat = $this->instance->stat('/lorem.txt');
//only size and mtime are required in the result
$this->assertEquals($stat['size'], $this->instance->filesize('/lorem.txt'));
$this->assertEquals($stat['mtime'], $mTime);
if ($this->instance->touch('/lorem.txt', 100) !== false) {
$mTime = $this->instance->filemtime('/lorem.txt');
$this->assertEquals($mTime, 100);
}
}
public function testCheckUpdate() {
$this->markTestSkipped('Detecting external changes is not supported on object storages');
}
/**
* @dataProvider copyAndMoveProvider
*/
public function testMove($source, $target) {
$this->initSourceAndTarget($source);
$sourceId = $this->instance->getCache()->getId(ltrim('/',$source));
$this->assertNotEquals(-1, $sourceId);
$this->instance->rename($source, $target);
$this->assertTrue($this->instance->file_exists($target), $target.' was not created');
$this->assertFalse($this->instance->file_exists($source), $source.' still exists');
$this->assertSameAsLorem($target);
$targetId = $this->instance->getCache()->getId(ltrim('/',$target));
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
}
public function testRenameDirectory() {
$this->instance->mkdir('source');
$this->instance->file_put_contents('source/test1.txt', 'foo');
$this->instance->file_put_contents('source/test2.txt', 'qwerty');
$this->instance->mkdir('source/subfolder');
$this->instance->file_put_contents('source/subfolder/test.txt', 'bar');
$sourceId = $this->instance->getCache()->getId('source');
$this->assertNotEquals(-1, $sourceId);
$this->instance->rename('source', 'target');
$this->assertFalse($this->instance->file_exists('source'));
$this->assertFalse($this->instance->file_exists('source/test1.txt'));
$this->assertFalse($this->instance->file_exists('source/test2.txt'));
$this->assertFalse($this->instance->file_exists('source/subfolder'));
$this->assertFalse($this->instance->file_exists('source/subfolder/test.txt'));
$this->assertTrue($this->instance->file_exists('target'));
$this->assertTrue($this->instance->file_exists('target/test1.txt'));
$this->assertTrue($this->instance->file_exists('target/test2.txt'));
$this->assertTrue($this->instance->file_exists('target/subfolder'));
$this->assertTrue($this->instance->file_exists('target/subfolder/test.txt'));
$this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt'));
$this->assertEquals('qwerty', $this->instance->file_get_contents('target/test2.txt'));
$this->assertEquals('bar', $this->instance->file_get_contents('target/subfolder/test.txt'));
$targetId = $this->instance->getCache()->getId('target');
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
}
public function testRenameOverWriteDirectory() {
$this->instance->mkdir('source');
$this->instance->file_put_contents('source/test1.txt', 'foo');
$sourceId = $this->instance->getCache()->getId('source');
$this->assertNotEquals(-1, $sourceId);
$this->instance->mkdir('target');
$this->instance->file_put_contents('target/test1.txt', 'bar');
$this->instance->file_put_contents('target/test2.txt', 'bar');
$this->instance->rename('source', 'target');
$this->assertFalse($this->instance->file_exists('source'));
$this->assertFalse($this->instance->file_exists('source/test1.txt'));
$this->assertFalse($this->instance->file_exists('target/test2.txt'));
$this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt'));
$targetId = $this->instance->getCache()->getId('target');
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
}
public function testRenameOverWriteDirectoryOverFile() {
$this->instance->mkdir('source');
$this->instance->file_put_contents('source/test1.txt', 'foo');
$sourceId = $this->instance->getCache()->getId('source');
$this->assertNotEquals(-1, $sourceId);
$this->instance->file_put_contents('target', 'bar');
$this->instance->rename('source', 'target');
$this->assertFalse($this->instance->file_exists('source'));
$this->assertFalse($this->instance->file_exists('source/test1.txt'));
$this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt'));
$targetId = $this->instance->getCache()->getId('target');
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
}
}

97
tests/lib/Files/ObjectStore/ObjectStoreTest.php Normal file
View File

@ -0,0 +1,97 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Files\ObjectStore;
use Test\TestCase;
abstract class ObjectStoreTest extends TestCase {
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
abstract protected function getInstance();
private function stringToStream($data) {
$stream = fopen('php://temp', 'w+');
fwrite($stream, $data);
rewind($stream);
return $stream;
}
public function testWriteRead() {
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance->writeObject('1', $stream);
$result = $instance->readObject('1');
$instance->deleteObject('1');
$this->assertEquals('foobar', stream_get_contents($result));
}
public function testDelete() {
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance->writeObject('2', $stream);
$instance->deleteObject('2');
try {
// to to read to verify that the object no longer exists
$instance->readObject('2');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testReadNonExisting() {
$instance = $this->getInstance();
try {
$instance->readObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testDeleteNonExisting() {
$instance = $this->getInstance();
try {
$instance->deleteObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
}

38
tests/lib/Files/ObjectStore/S3Test.php Normal file
View File

@ -0,0 +1,38 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Files\ObjectStore;
use OC\Files\ObjectStore\S3;
class S3Test extends ObjectStoreTest {
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
protected function getInstance() {
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
return new S3($config['arguments']);
}
}

194
tests/lib/Files/ObjectStore/SwiftTest.php
View File

@ -1,198 +1,38 @@
<?php
/**
* @author Jörn Friedrich Dreyer
* @copyright (c) 2014 Jörn Friedrich Dreyer <jfd@owncloud.com>
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
* License as published by the Free Software Foundation; either
* version 3 of the License, or any later version.
* @license GNU AGPL version 3 or any later version
*
* This library is distributed in the hope that it will be useful,
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Files\ObjectStore;
use OC\Files\ObjectStore\ObjectStoreStorage;
use OC\Files\ObjectStore\Swift;
/**
* Class SwiftTest
*
* @group DB
*
* @package Test\Files\Cache\ObjectStore
*/
class SwiftTest extends \Test\Files\Storage\Storage {
class SwiftTest extends ObjectStoreTest {
/**
* @var Swift
* @return \OCP\Files\ObjectStore\IObjectStore
*/
private $objectStorage;
protected function setUp() {
parent::setUp();
if (!getenv('RUN_OBJECTSTORE_TESTS')) {
$this->markTestSkipped('objectstore tests are unreliable in some environments');
}
// reset backend
\OC_User::clearBackends();
\OC_User::useBackend('database');
// create users
$users = array('test');
foreach($users as $userName) {
$user = \OC::$server->getUserManager()->get($userName);
if ($user !== null) { $user->delete(); }
\OC::$server->getUserManager()->createUser($userName, $userName);
}
// main test user
\OC_Util::tearDownFS();
\OC_User::setUserId('');
\OC\Files\Filesystem::tearDown();
\OC_User::setUserId('test');
protected function getInstance() {
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
$this->objectStorage = new Swift($config['arguments']);
$config['objectstore'] = $this->objectStorage;
$this->instance = new ObjectStoreStorage($config);
}
protected function tearDown() {
if (is_null($this->instance)) {
return;
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\Swift') {
$this->markTestSkipped('objectstore not configured for swift');
}
$this->objectStorage->deleteContainer(true);
$this->instance->getCache()->clear();
$users = array('test');
foreach($users as $userName) {
$user = \OC::$server->getUserManager()->get($userName);
if ($user !== null) { $user->delete(); }
}
parent::tearDown();
}
public function testStat() {
$textFile = \OC::$SERVERROOT . '/tests/data/lorem.txt';
$ctimeStart = time();
$this->instance->file_put_contents('/lorem.txt', file_get_contents($textFile));
$this->assertTrue($this->instance->isReadable('/lorem.txt'));
$ctimeEnd = time();
$mTime = $this->instance->filemtime('/lorem.txt');
// check that ($ctimeStart - 5) <= $mTime <= ($ctimeEnd + 1)
$this->assertGreaterThanOrEqual(($ctimeStart - 5), $mTime);
$this->assertLessThanOrEqual(($ctimeEnd + 1), $mTime);
$this->assertEquals(filesize($textFile), $this->instance->filesize('/lorem.txt'));
$stat = $this->instance->stat('/lorem.txt');
//only size and mtime are required in the result
$this->assertEquals($stat['size'], $this->instance->filesize('/lorem.txt'));
$this->assertEquals($stat['mtime'], $mTime);
if ($this->instance->touch('/lorem.txt', 100) !== false) {
$mTime = $this->instance->filemtime('/lorem.txt');
$this->assertEquals($mTime, 100);
}
}
public function testCheckUpdate() {
$this->markTestSkipped('Detecting external changes is not supported on object storages');
}
/**
* @dataProvider copyAndMoveProvider
*/
public function testMove($source, $target) {
$this->initSourceAndTarget($source);
$sourceId = $this->instance->getCache()->getId(ltrim('/',$source));
$this->assertNotEquals(-1, $sourceId);
$this->instance->rename($source, $target);
$this->assertTrue($this->instance->file_exists($target), $target.' was not created');
$this->assertFalse($this->instance->file_exists($source), $source.' still exists');
$this->assertSameAsLorem($target);
$targetId = $this->instance->getCache()->getId(ltrim('/',$target));
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
}
public function testRenameDirectory() {
$this->instance->mkdir('source');
$this->instance->file_put_contents('source/test1.txt', 'foo');
$this->instance->file_put_contents('source/test2.txt', 'qwerty');
$this->instance->mkdir('source/subfolder');
$this->instance->file_put_contents('source/subfolder/test.txt', 'bar');
$sourceId = $this->instance->getCache()->getId('source');
$this->assertNotEquals(-1, $sourceId);
$this->instance->rename('source', 'target');
$this->assertFalse($this->instance->file_exists('source'));
$this->assertFalse($this->instance->file_exists('source/test1.txt'));
$this->assertFalse($this->instance->file_exists('source/test2.txt'));
$this->assertFalse($this->instance->file_exists('source/subfolder'));
$this->assertFalse($this->instance->file_exists('source/subfolder/test.txt'));
$this->assertTrue($this->instance->file_exists('target'));
$this->assertTrue($this->instance->file_exists('target/test1.txt'));
$this->assertTrue($this->instance->file_exists('target/test2.txt'));
$this->assertTrue($this->instance->file_exists('target/subfolder'));
$this->assertTrue($this->instance->file_exists('target/subfolder/test.txt'));
$this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt'));
$this->assertEquals('qwerty', $this->instance->file_get_contents('target/test2.txt'));
$this->assertEquals('bar', $this->instance->file_get_contents('target/subfolder/test.txt'));
$targetId = $this->instance->getCache()->getId('target');
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
}
public function testRenameOverWriteDirectory() {
$this->instance->mkdir('source');
$this->instance->file_put_contents('source/test1.txt', 'foo');
$sourceId = $this->instance->getCache()->getId('source');
$this->assertNotEquals(-1, $sourceId);
$this->instance->mkdir('target');
$this->instance->file_put_contents('target/test1.txt', 'bar');
$this->instance->file_put_contents('target/test2.txt', 'bar');
$this->instance->rename('source', 'target');
$this->assertFalse($this->instance->file_exists('source'));
$this->assertFalse($this->instance->file_exists('source/test1.txt'));
$this->assertFalse($this->instance->file_exists('target/test2.txt'));
$this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt'));
$targetId = $this->instance->getCache()->getId('target');
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
}
public function testRenameOverWriteDirectoryOverFile() {
$this->instance->mkdir('source');
$this->instance->file_put_contents('source/test1.txt', 'foo');
$sourceId = $this->instance->getCache()->getId('source');
$this->assertNotEquals(-1, $sourceId);
$this->instance->file_put_contents('target', 'bar');
$this->instance->rename('source', 'target');
$this->assertFalse($this->instance->file_exists('source'));
$this->assertFalse($this->instance->file_exists('source/test1.txt'));
$this->assertEquals('foo', $this->instance->file_get_contents('target/test1.txt'));
$targetId = $this->instance->getCache()->getId('target');
$this->assertSame($sourceId, $targetId, 'fileid must be stable on move or shares will break');
return new Swift($config['arguments']);
}
}