From 5e2316d05d954ec9cb9a3d284007c3d329550395 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sun, 22 May 2016 20:44:06 +0200 Subject: [PATCH 1/5] Allow multibucket in objectstore --- .../Files/Mount/ObjectHomeMountProvider.php | 57 ++++++++++++++++++- lib/private/Files/ObjectStore/Mapper.php | 52 +++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 lib/private/Files/ObjectStore/Mapper.php diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index c910cf6bd4..aa6a443971 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -52,9 +52,27 @@ class ObjectHomeMountProvider implements IHomeMountProvider { * @return \OCP\Files\Mount\IMountPoint[] */ public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { + + $config = $this->multiBucketObjectStore($user); + if ($config === null) { + $config = $this->singleBucketObjectStore($user); + } + + if ($config === null) { + return null; + } + + return new MountPoint('\OC\Files\ObjectStore\HomeObjectStoreStorage', '/' . $user->getUID(), $config['arguments'], $loader); + } + + /** + * @param IUser $user + * @return array|null + */ + private function singleBucketObjectStore(IUser $user) { $config = $this->config->getSystemValue('objectstore'); if (!is_array($config)) { - return null; //fall back to local home provider + return null; } // sanity checks @@ -68,6 +86,41 @@ class ObjectHomeMountProvider implements IHomeMountProvider { // instantiate object store implementation $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - return new MountPoint('\OC\Files\ObjectStore\HomeObjectStoreStorage', '/' . $user->getUID(), $config['arguments'], $loader); + return $config; + } + + /** + * @param IUser $user + * @return array|null + */ + private function multiBucketObjectStore(IUser $user) { + $config = $this->config->getSystemValue('objectstore_multibucket'); + if (!is_array($config)) { + return null; + } + + // sanity checks + if (empty($config['class'])) { + \OCP\Util::writeLog('files', 'No class given for objectstore', \OCP\Util::ERROR); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + $config['arguments']['user'] = $user; + + /* + * Use any provided bucket argument as prefix + * and add the mapping from username => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $mapper = new \OC\Files\ObjectStore\Mapper($user); + $config['arguments']['bucket'] .= $mapper->getBucket(); + + // instantiate object store implementation + $config['arguments']['objectstore'] = new $config['class']($config['arguments']); + + return $config; } } diff --git a/lib/private/Files/ObjectStore/Mapper.php b/lib/private/Files/ObjectStore/Mapper.php new file mode 100644 index 0000000000..f0004f1f96 --- /dev/null +++ b/lib/private/Files/ObjectStore/Mapper.php @@ -0,0 +1,52 @@ + + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ +namespace OC\Files\ObjectStore; + +use OCP\IUser; + +/** + * Class Mapper + * + * @package OC\Files\ObjectStore + * + * Map a user to a bucket. + */ +class Mapper { + /** @var IUser */ + private $user; + + /** + * Mapper constructor. + * + * @param IUser $user + */ + public function __construct(IUser $user) { + $this->user = $user; + } + + /** + * @return string + */ + public function getBucket() { + $hash = md5($this->user->getUID()); + return substr($hash, 0, 3); + } +} \ No newline at end of file From 12b63258d55b3d8766983b12070469426e66942d Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sun, 22 May 2016 21:14:28 +0200 Subject: [PATCH 2/5] Add mapper unit tests --- tests/lib/Files/ObjectStore/MapperTest.php | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/lib/Files/ObjectStore/MapperTest.php diff --git a/tests/lib/Files/ObjectStore/MapperTest.php b/tests/lib/Files/ObjectStore/MapperTest.php new file mode 100644 index 0000000000..1ebb67a690 --- /dev/null +++ b/tests/lib/Files/ObjectStore/MapperTest.php @@ -0,0 +1,50 @@ + + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ +namespace Test\Files\ObjectStore; + + +use OC\Files\ObjectStore\Mapper; + +class MapperTest extends \Test\TestCase { + + public function dataGetBucket() { + return [ + ['user', substr(md5('user'), 0, 3)], + ['USER', substr(md5('USER'), 0, 3)], + ['bc0e8b52-a66c-1035-90c6-d9663bda9e3f', substr(md5('bc0e8b52-a66c-1035-90c6-d9663bda9e3f'), 0, 3)], + ]; + } + + /** + * @dataProvider dataGetBucket + * @param string $username + * @param string $expectedBucket + */ + public function testGetBucket($username, $expectedBucket) { + $user = $this->getMock('OCP\IUser'); + $user->method('getUID') + ->willReturn($username); + + $mapper = new Mapper($user); + + $this->assertSame($expectedBucket, $mapper->getBucket()); + } +} \ No newline at end of file From 7ef21b0b27206f2b32e7706abf463d277d8f151c Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 23 May 2016 10:59:10 +0200 Subject: [PATCH 3/5] Add unit tests for ObjectHomeMountProvider --- .../Mount/ObjectHomeMountProviderTest.php | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 tests/lib/Files/Mount/ObjectHomeMountProviderTest.php diff --git a/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php b/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php new file mode 100644 index 0000000000..f62dab093b --- /dev/null +++ b/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php @@ -0,0 +1,170 @@ +config = $this->getMock('OCP\IConfig'); + $this->user = $this->getMock('OCP\IUser'); + $this->loader = $this->getMock('OCP\Files\Storage\IStorageFactory'); + + $this->provider = new ObjectHomeMountProvider($this->config); + } + + public function testSingleBucket() { + $this->config->expects($this->once()) + ->method('getSystemValue') + ->with($this->equalTo('objectstore'), '') + ->willReturn([ + 'class' => 'Test\Files\Mount\FakeObjectStore', + ]); + + $this->user->expects($this->never())->method($this->anything()); + $this->loader->expects($this->never())->method($this->anything()); + + $config = $this->invokePrivate($this->provider, 'singleBucketObjectStore', [$this->user, $this->loader]); + + $this->assertArrayHasKey('class', $config); + $this->assertEquals($config['class'], 'Test\Files\Mount\FakeObjectStore'); + $this->assertArrayHasKey('arguments', $config); + $this->assertArrayHasKey('user', $config['arguments']); + $this->assertSame($this->user, $config['arguments']['user']); + $this->assertArrayHasKey('objectstore', $config['arguments']); + $this->assertInstanceOf('Test\Files\Mount\FakeObjectStore', $config['arguments']['objectstore']); + } + + public function testMultiBucket() { + $this->config->expects($this->once()) + ->method('getSystemValue') + ->with($this->equalTo('objectstore_multibucket'), '') + ->willReturn([ + 'class' => 'Test\Files\Mount\FakeObjectStore', + ]); + + $this->user->expects($this->once()) + ->method('getUID') + ->willReturn('uid'); + $this->loader->expects($this->never())->method($this->anything()); + + $config = $this->invokePrivate($this->provider, 'multiBucketObjectStore', [$this->user, $this->loader]); + + $this->assertArrayHasKey('class', $config); + $this->assertEquals($config['class'], 'Test\Files\Mount\FakeObjectStore'); + $this->assertArrayHasKey('arguments', $config); + $this->assertArrayHasKey('user', $config['arguments']); + $this->assertSame($this->user, $config['arguments']['user']); + $this->assertArrayHasKey('objectstore', $config['arguments']); + $this->assertInstanceOf('Test\Files\Mount\FakeObjectStore', $config['arguments']['objectstore']); + $this->assertArrayHasKey('bucket', $config['arguments']); + $this->assertEquals('987', $config['arguments']['bucket']); + } + + public function testMultiBucketWithPrefix() { + $this->config->expects($this->once()) + ->method('getSystemValue') + ->with($this->equalTo('objectstore_multibucket'), '') + ->willReturn([ + 'class' => 'Test\Files\Mount\FakeObjectStore', + 'arguments' => [ + 'bucket' => 'myBucketPrefix', + ], + ]); + + $this->user->expects($this->once()) + ->method('getUID') + ->willReturn('uid'); + $this->loader->expects($this->never())->method($this->anything()); + + $config = $this->invokePrivate($this->provider, 'multiBucketObjectStore', [$this->user, $this->loader]); + + $this->assertArrayHasKey('class', $config); + $this->assertEquals($config['class'], 'Test\Files\Mount\FakeObjectStore'); + $this->assertArrayHasKey('arguments', $config); + $this->assertArrayHasKey('user', $config['arguments']); + $this->assertSame($this->user, $config['arguments']['user']); + $this->assertArrayHasKey('objectstore', $config['arguments']); + $this->assertInstanceOf('Test\Files\Mount\FakeObjectStore', $config['arguments']['objectstore']); + $this->assertArrayHasKey('bucket', $config['arguments']); + $this->assertEquals('myBucketPrefix987', $config['arguments']['bucket']); + } + + public function testMultiBucketConfigFirst() { + $this->config->expects($this->once()) + ->method('getSystemValue') + ->with($this->equalTo('objectstore_multibucket')) + ->willReturn([ + 'class' => 'Test\Files\Mount\FakeObjectStore', + ]); + + $this->user->expects($this->exactly(2)) + ->method('getUID') + ->willReturn('uid'); + $this->loader->expects($this->never())->method($this->anything()); + + $mount = $this->provider->getHomeMountForUser($this->user, $this->loader); + $this->assertInstanceOf('OC\Files\Mount\MountPoint', $mount); + } + + public function testMultiBucketConfigFirstFallBackSingle() { + $this->config->expects($this->at(0)) + ->method('getSystemValue') + ->with($this->equalTo('objectstore_multibucket')) + ->willReturn(''); + + $this->config->expects($this->at(1)) + ->method('getSystemValue') + ->with($this->equalTo('objectstore')) + ->willReturn([ + 'class' => 'Test\Files\Mount\FakeObjectStore', + ]); + + $this->user->expects($this->once()) + ->method('getUID') + ->willReturn('uid'); + $this->loader->expects($this->never())->method($this->anything()); + + $mount = $this->provider->getHomeMountForUser($this->user, $this->loader); + $this->assertInstanceOf('OC\Files\Mount\MountPoint', $mount); + } + + public function testNoObjectStore() { + $this->config->expects($this->exactly(2)) + ->method('getSystemValue') + ->willReturn(''); + + $mount = $this->provider->getHomeMountForUser($this->user, $this->loader); + $this->assertNull($mount); + } +} + +class FakeObjectStore { + private $arguments; + + public function __construct(array $arguments) { + $this->arguments = $arguments; + } + + public function getArguments() { + return $this->arguments; + } +} \ No newline at end of file From e03e4921a01cd2b01e10db7b5d7bd39dd0224b0b Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 23 May 2016 20:41:51 +0200 Subject: [PATCH 4/5] Fix Name --- .htaccess | 4 ++++ lib/private/Files/Mount/ObjectHomeMountProvider.php | 8 ++++---- tests/lib/Files/Mount/ObjectHomeMountProviderTest.php | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.htaccess b/.htaccess index 5bf7b321f0..d19162fd71 100644 --- a/.htaccess +++ b/.htaccess @@ -72,3 +72,7 @@ Options -Indexes ModPagespeed Off +#### DO NOT CHANGE ANYTHING ABOVE THIS LINE #### + +ErrorDocument 403 /core/templates/403.php +ErrorDocument 404 /core/templates/404.php diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index aa6a443971..6903a5fd10 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -53,9 +53,9 @@ class ObjectHomeMountProvider implements IHomeMountProvider { */ public function getHomeMountForUser(IUser $user, IStorageFactory $loader) { - $config = $this->multiBucketObjectStore($user); + $config = $this->getMultiBucketObjectStoreConfig($user); if ($config === null) { - $config = $this->singleBucketObjectStore($user); + $config = $this->getSingleBucketObjectStoreConfig($user); } if ($config === null) { @@ -69,7 +69,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider { * @param IUser $user * @return array|null */ - private function singleBucketObjectStore(IUser $user) { + private function getSingleBucketObjectStoreConfig(IUser $user) { $config = $this->config->getSystemValue('objectstore'); if (!is_array($config)) { return null; @@ -93,7 +93,7 @@ class ObjectHomeMountProvider implements IHomeMountProvider { * @param IUser $user * @return array|null */ - private function multiBucketObjectStore(IUser $user) { + private function getMultiBucketObjectStoreConfig(IUser $user) { $config = $this->config->getSystemValue('objectstore_multibucket'); if (!is_array($config)) { return null; diff --git a/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php b/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php index f62dab093b..202c2c02b9 100644 --- a/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php +++ b/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php @@ -42,7 +42,7 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { $this->user->expects($this->never())->method($this->anything()); $this->loader->expects($this->never())->method($this->anything()); - $config = $this->invokePrivate($this->provider, 'singleBucketObjectStore', [$this->user, $this->loader]); + $config = $this->invokePrivate($this->provider, 'getSingleBucketObjectStoreConfig', [$this->user, $this->loader]); $this->assertArrayHasKey('class', $config); $this->assertEquals($config['class'], 'Test\Files\Mount\FakeObjectStore'); @@ -66,7 +66,7 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { ->willReturn('uid'); $this->loader->expects($this->never())->method($this->anything()); - $config = $this->invokePrivate($this->provider, 'multiBucketObjectStore', [$this->user, $this->loader]); + $config = $this->invokePrivate($this->provider, 'getMultiBucketObjectStoreConfig', [$this->user, $this->loader]); $this->assertArrayHasKey('class', $config); $this->assertEquals($config['class'], 'Test\Files\Mount\FakeObjectStore'); @@ -95,7 +95,7 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { ->willReturn('uid'); $this->loader->expects($this->never())->method($this->anything()); - $config = $this->invokePrivate($this->provider, 'multiBucketObjectStore', [$this->user, $this->loader]); + $config = $this->invokePrivate($this->provider, 'getMultiBucketObjectStoreConfig', [$this->user, $this->loader]); $this->assertArrayHasKey('class', $config); $this->assertEquals($config['class'], 'Test\Files\Mount\FakeObjectStore'); From abe338f4335ad4a4f0b6211b032e48f80962292f Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 23 May 2016 21:03:25 +0200 Subject: [PATCH 5/5] Store user bucket in preferences --- .htaccess | 4 - .../Files/Mount/ObjectHomeMountProvider.php | 24 +++-- .../Mount/ObjectHomeMountProviderTest.php | 90 +++++++++++++++++-- 3 files changed, 98 insertions(+), 20 deletions(-) diff --git a/.htaccess b/.htaccess index d19162fd71..5bf7b321f0 100644 --- a/.htaccess +++ b/.htaccess @@ -72,7 +72,3 @@ Options -Indexes ModPagespeed Off -#### DO NOT CHANGE ANYTHING ABOVE THIS LINE #### - -ErrorDocument 403 /core/templates/403.php -ErrorDocument 404 /core/templates/404.php diff --git a/lib/private/Files/Mount/ObjectHomeMountProvider.php b/lib/private/Files/Mount/ObjectHomeMountProvider.php index 6903a5fd10..f82313879d 100644 --- a/lib/private/Files/Mount/ObjectHomeMountProvider.php +++ b/lib/private/Files/Mount/ObjectHomeMountProvider.php @@ -108,15 +108,23 @@ class ObjectHomeMountProvider implements IHomeMountProvider { } $config['arguments']['user'] = $user; - /* - * Use any provided bucket argument as prefix - * and add the mapping from username => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; + $bucket = $this->config->getUserValue($user->getUID(), 'homeobjectstore', 'bucket', null); + + if ($bucket === null) { + /* + * Use any provided bucket argument as prefix + * and add the mapping from username => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $mapper = new \OC\Files\ObjectStore\Mapper($user); + $config['arguments']['bucket'] .= $mapper->getBucket(); + + $this->config->setUserValue($user->getUID(), 'homeobjectstore', 'bucket', $config['arguments']['bucket']); + } else { + $config['arguments']['bucket'] = $bucket; } - $mapper = new \OC\Files\ObjectStore\Mapper($user); - $config['arguments']['bucket'] .= $mapper->getBucket(); // instantiate object store implementation $config['arguments']['objectstore'] = new $config['class']($config['arguments']); diff --git a/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php b/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php index 202c2c02b9..5d987f0d05 100644 --- a/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php +++ b/tests/lib/Files/Mount/ObjectHomeMountProviderTest.php @@ -61,11 +61,29 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { 'class' => 'Test\Files\Mount\FakeObjectStore', ]); - $this->user->expects($this->once()) - ->method('getUID') + $this->user->method('getUID') ->willReturn('uid'); $this->loader->expects($this->never())->method($this->anything()); + $this->config->expects($this->once()) + ->method('getUserValue') + ->with( + $this->equalTo('uid'), + $this->equalTo('homeobjectstore'), + $this->equalTo('bucket'), + $this->equalTo(null) + )->willReturn(null); + + $this->config->expects($this->once()) + ->method('setUserValue') + ->with( + $this->equalTo('uid'), + $this->equalTo('homeobjectstore'), + $this->equalTo('bucket'), + $this->equalTo('987'), + $this->equalTo(null) + ); + $config = $this->invokePrivate($this->provider, 'getMultiBucketObjectStoreConfig', [$this->user, $this->loader]); $this->assertArrayHasKey('class', $config); @@ -90,11 +108,29 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { ], ]); - $this->user->expects($this->once()) - ->method('getUID') + $this->user->method('getUID') ->willReturn('uid'); $this->loader->expects($this->never())->method($this->anything()); + $this->config->expects($this->once()) + ->method('getUserValue') + ->with( + $this->equalTo('uid'), + $this->equalTo('homeobjectstore'), + $this->equalTo('bucket'), + $this->equalTo(null) + )->willReturn(null); + + $this->config->expects($this->once()) + ->method('setUserValue') + ->with( + $this->equalTo('uid'), + $this->equalTo('homeobjectstore'), + $this->equalTo('bucket'), + $this->equalTo('myBucketPrefix987'), + $this->equalTo(null) + ); + $config = $this->invokePrivate($this->provider, 'getMultiBucketObjectStoreConfig', [$this->user, $this->loader]); $this->assertArrayHasKey('class', $config); @@ -108,6 +144,46 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { $this->assertEquals('myBucketPrefix987', $config['arguments']['bucket']); } + public function testMultiBucketBucketAlreadySet() { + $this->config->expects($this->once()) + ->method('getSystemValue') + ->with($this->equalTo('objectstore_multibucket'), '') + ->willReturn([ + 'class' => 'Test\Files\Mount\FakeObjectStore', + 'arguments' => [ + 'bucket' => 'myBucketPrefix', + ], + ]); + + $this->user->method('getUID') + ->willReturn('uid'); + $this->loader->expects($this->never())->method($this->anything()); + + $this->config->expects($this->once()) + ->method('getUserValue') + ->with( + $this->equalTo('uid'), + $this->equalTo('homeobjectstore'), + $this->equalTo('bucket'), + $this->equalTo(null) + )->willReturn('awesomeBucket1'); + + $this->config->expects($this->never()) + ->method('setUserValue'); + + $config = $this->invokePrivate($this->provider, 'getMultiBucketObjectStoreConfig', [$this->user, $this->loader]); + + $this->assertArrayHasKey('class', $config); + $this->assertEquals($config['class'], 'Test\Files\Mount\FakeObjectStore'); + $this->assertArrayHasKey('arguments', $config); + $this->assertArrayHasKey('user', $config['arguments']); + $this->assertSame($this->user, $config['arguments']['user']); + $this->assertArrayHasKey('objectstore', $config['arguments']); + $this->assertInstanceOf('Test\Files\Mount\FakeObjectStore', $config['arguments']['objectstore']); + $this->assertArrayHasKey('bucket', $config['arguments']); + $this->assertEquals('awesomeBucket1', $config['arguments']['bucket']); + } + public function testMultiBucketConfigFirst() { $this->config->expects($this->once()) ->method('getSystemValue') @@ -116,8 +192,7 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { 'class' => 'Test\Files\Mount\FakeObjectStore', ]); - $this->user->expects($this->exactly(2)) - ->method('getUID') + $this->user->method('getUID') ->willReturn('uid'); $this->loader->expects($this->never())->method($this->anything()); @@ -138,8 +213,7 @@ class ObjectHomeMountProviderTest extends \Test\TestCase { 'class' => 'Test\Files\Mount\FakeObjectStore', ]); - $this->user->expects($this->once()) - ->method('getUID') + $this->user->method('getUID') ->willReturn('uid'); $this->loader->expects($this->never())->method($this->anything());