mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #22063 from nextcloud/enh/22033/multibucket-appdata-previews 

Distribute preview folders in appdata in multibucket setup to multiple buckets
This commit is contained in:
Morris Jobke 2020-08-07 09:28:11 +02:00 committed by GitHub
parent 1df567d5ad 45428e4948
commit eb2b1bec85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 415 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
config/config.sample.php
View File

@ -1354,6 +1354,23 @@ $CONFIG = [
],
],
/**
* If this is set to true and a multibucket object store is configured then
* newly created previews are put into 256 dedicated buckets.
*
* Those buckets are named like the mulibucket version but with the postfix
* ``-preview-NUMBER`` where NUMBER is between 0 and 255.
*
* Keep in mind that only previews of files are put in there that don't have
* some already. Otherwise the old bucket will be used.
*
* To migrate existing previews to this new multibucket distribution of previews
* use the occ command ``preview:repair``. For now this will only migrate
* previews that were generated before Nextcloud 19 in the flat
* ``appdata_INSTANCEID/previews/FILEID`` folder structure.
*/
'objectstore.multibucket.preview-distribution' => false,
/**
* Sharing

3
lib/composer/composer/autoload_classmap.php
View File

@ -230,6 +230,7 @@ return array(
'OCP\\Files\\Config\\IHomeMountProvider' => $baseDir . '/lib/public/Files/Config/IHomeMountProvider.php',
'OCP\\Files\\Config\\IMountProvider' => $baseDir . '/lib/public/Files/Config/IMountProvider.php',
'OCP\\Files\\Config\\IMountProviderCollection' => $baseDir . '/lib/public/Files/Config/IMountProviderCollection.php',
'OCP\\Files\\Config\\IRootMountProvider' => $baseDir . '/lib/public/Files/Config/IRootMountProvider.php',
'OCP\\Files\\Config\\IUserMountCache' => $baseDir . '/lib/public/Files/Config/IUserMountCache.php',
'OCP\\Files\\EmptyFileNameException' => $baseDir . '/lib/public/Files/EmptyFileNameException.php',
'OCP\\Files\\EntityTooLargeException' => $baseDir . '/lib/public/Files/EntityTooLargeException.php',
@ -1029,6 +1030,7 @@ return array(
'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php',
'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php',
'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php',
'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php',
'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php',
'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php',
'OC\\Files\\Node\\HookConnector' => $baseDir . '/lib/private/Files/Node/HookConnector.php',
@ -1040,6 +1042,7 @@ return array(
'OC\\Files\\Node\\Root' => $baseDir . '/lib/private/Files/Node/Root.php',
'OC\\Files\\Notify\\Change' => $baseDir . '/lib/private/Files/Notify/Change.php',
'OC\\Files\\Notify\\RenameChange' => $baseDir . '/lib/private/Files/Notify/RenameChange.php',
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',

3
lib/composer/composer/autoload_static.php
View File

@ -259,6 +259,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OCP\\Files\\Config\\IHomeMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IHomeMountProvider.php',
'OCP\\Files\\Config\\IMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProvider.php',
'OCP\\Files\\Config\\IMountProviderCollection' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProviderCollection.php',
'OCP\\Files\\Config\\IRootMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IRootMountProvider.php',
'OCP\\Files\\Config\\IUserMountCache' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IUserMountCache.php',
'OCP\\Files\\EmptyFileNameException' => __DIR__ . '/../../..' . '/lib/public/Files/EmptyFileNameException.php',
'OCP\\Files\\EntityTooLargeException' => __DIR__ . '/../../..' . '/lib/public/Files/EntityTooLargeException.php',
@ -1058,6 +1059,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php',
'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php',
'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php',
'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php',
'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php',
'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php',
'OC\\Files\\Node\\HookConnector' => __DIR__ . '/../../..' . '/lib/private/Files/Node/HookConnector.php',
@ -1069,6 +1071,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Files\\Node\\Root' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Root.php',
'OC\\Files\\Notify\\Change' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/Change.php',
'OC\\Files\\Notify\\RenameChange' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/RenameChange.php',
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',

19
lib/private/Files/Config/MountProviderCollection.php
View File

@ -30,6 +30,7 @@ use OC\Hooks\EmitterTrait;
use OCP\Files\Config\IHomeMountProvider;
use OCP\Files\Config\IMountProvider;
use OCP\Files\Config\IMountProviderCollection;
use OCP\Files\Config\IRootMountProvider;
use OCP\Files\Config\IUserMountCache;
use OCP\Files\Mount\IMountManager;
use OCP\Files\Mount\IMountPoint;
@ -49,6 +50,9 @@ class MountProviderCollection implements IMountProviderCollection, Emitter {
*/
private $providers = [];
/** @var \OCP\Files\Config\IRootMountProvider[] */
private $rootProviders = [];
/**
* @var \OCP\Files\Storage\IStorageFactory
*/
@ -198,4 +202,19 @@ class MountProviderCollection implements IMountProviderCollection, Emitter {
public function getMountCache() {
return $this->mountCache;
}
public function registerRootProvider(IRootMountProvider $provider) {
$this->rootProviders[] = $provider;
}
public function getRootMounts(): array {
$loader = $this->loader;
$mounts = array_map(function (IRootMountProvider $provider) use ($loader) {
return $provider->getRootMounts($loader);
}, $this->rootProviders);
$mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) {
return array_merge($mounts, $providerMounts);
}, []);
return $mounts;
}
}

150
lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php Normal file
View File

@ -0,0 +1,150 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
*
* @author Morris Jobke <hey@morrisjobke.de>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\Mount;
use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage;
use OC\Files\ObjectStore\ObjectStoreStorage;
use OC\Files\Storage\Wrapper\Jail;
use OCP\Files\Config\IRootMountProvider;
use OCP\Files\Storage\IStorageFactory;
use OCP\IConfig;
use OCP\ILogger;
/**
* Mount provider for object store app data folder for previews
*/
class ObjectStorePreviewCacheMountProvider implements IRootMountProvider {
/** @var ILogger */
private $logger;
/** @var IConfig */
private $config;
public function __construct(ILogger $logger, IConfig $config) {
$this->logger = $logger;
$this->config = $config;
}
/**
* @return MountPoint[]
* @throws \Exception
*/
public function getRootMounts(IStorageFactory $loader): array {
if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) {
return [];
}
if ($this->config->getSystemValue('objectstore.multibucket.preview-distribution', false) !== true) {
return [];
}
$instanceId = $this->config->getSystemValueString('instanceid', '');
$mountPoints = [];
$directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'];
$i = 0;
foreach ($directoryRange as $parent) {
foreach ($directoryRange as $child) {
$mountPoints[] = new MountPoint(
AppdataPreviewObjectStoreStorage::class,
'/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child,
$this->getMultiBucketObjectStore($i),
$loader
);
$i++;
}
}
$rootStorageArguments = $this->getMultiBucketObjectStoreForRoot();
$fakeRootStorage = new ObjectStoreStorage($rootStorageArguments);
$fakeRootStorageJail = new Jail([
'storage' => $fakeRootStorage,
'root' => '/appdata_' . $instanceId . '/preview',
]);
// add a fallback location to be able to fetch existing previews from the old bucket
$mountPoints[] = new MountPoint(
$fakeRootStorageJail,
'/appdata_' . $instanceId . '/preview/old-multibucket',
null,
$loader
);
return $mountPoints;
}
protected function getMultiBucketObjectStore(int $number): array {
$config = $this->config->getSystemValue('objectstore_multibucket');
// sanity checks
if (empty($config['class'])) {
$this->logger->error('No class given for objectstore', ['app' => 'files']);
}
if (!isset($config['arguments'])) {
$config['arguments'] = [];
}
/*
* Use any provided bucket argument as prefix
* and add the mapping from parent/child => bucket
*/
if (!isset($config['arguments']['bucket'])) {
$config['arguments']['bucket'] = '';
}
$config['arguments']['bucket'] .= "-preview-$number";
// instantiate object store implementation
$config['arguments']['objectstore'] = new $config['class']($config['arguments']);
$config['arguments']['internal-id'] = $number;
return $config['arguments'];
}
protected function getMultiBucketObjectStoreForRoot(): array {
$config = $this->config->getSystemValue('objectstore_multibucket');
// sanity checks
if (empty($config['class'])) {
$this->logger->error('No class given for objectstore', ['app' => 'files']);
}
if (!isset($config['arguments'])) {
$config['arguments'] = [];
}
/*
* Use any provided bucket argument as prefix
* and add the mapping from parent/child => bucket
*/
if (!isset($config['arguments']['bucket'])) {
$config['arguments']['bucket'] = '';
}
$config['arguments']['bucket'] .= '0';
// instantiate object store implementation
$config['arguments']['objectstore'] = new $config['class']($config['arguments']);
return $config['arguments'];
}
}

44
lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php Normal file
View File

@ -0,0 +1,44 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
*
* @author Morris Jobke <hey@morrisjobke.de>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;
class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage {
/** @var string */
private $internalId;
public function __construct($params) {
if (!isset($params['internal-id'])) {
throw new \Exception('missing id in parameters');
}
$this->internalId = (string)$params['internal-id'];
parent::__construct($params);
}
public function getId() {
return 'object::appdata::preview:' . $this->internalId;
}
}

12
lib/private/Preview/Storage/Root.php
View File

@ -1,7 +1,6 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2020, Roeland Jago Douma <roeland@famdouma.nl>
*
@ -33,14 +32,25 @@ use OCP\Files\NotFoundException;
use OCP\Files\SimpleFS\ISimpleFolder;
class Root extends AppData {
private $isMultibucketPreviewDistributionEnabled = false;
public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig) {
parent::__construct($rootFolder, $systemConfig, 'preview');
$this->isMultibucketPreviewDistributionEnabled = $systemConfig->getValue('objectstore.multibucket.preview-distribution', false) === true;
}
public function getFolder(string $name): ISimpleFolder {
$internalFolder = $this->getInternalFolder($name);
if ($this->isMultibucketPreviewDistributionEnabled) {
try {
return parent::getFolder('old-multibucket/' . $internalFolder);
} catch (NotFoundException $e) {
// not in multibucket fallback
}
}
try {
return parent::getFolder($internalFolder);
} catch (NotFoundException $e) {

3
lib/private/Server.php
View File

@ -88,6 +88,7 @@ use OC\Files\Config\UserMountCacheListener;
use OC\Files\Mount\CacheMountProvider;
use OC\Files\Mount\LocalHomeMountProvider;
use OC\Files\Mount\ObjectHomeMountProvider;
use OC\Files\Mount\ObjectStorePreviewCacheMountProvider;
use OC\Files\Node\HookConnector;
use OC\Files\Node\LazyRoot;
use OC\Files\Node\Root;
@ -904,9 +905,11 @@ class Server extends ServerContainer implements IServerContainer {
// builtin providers
$config = $c->getConfig();
$logger = $c->getLogger();
$manager->registerProvider(new CacheMountProvider($config));
$manager->registerHomeProvider(new LocalHomeMountProvider());
$manager->registerHomeProvider(new ObjectHomeMountProvider($config));
$manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config));
return $manager;
});

11
lib/private/legacy/OC_Util.php
View File

@ -297,6 +297,17 @@ class OC_Util {
self::initLocalStorageRootFS();
}
/** @var \OCP\Files\Config\IMountProviderCollection $mountProviderCollection */
$mountProviderCollection = \OC::$server->query(\OCP\Files\Config\IMountProviderCollection::class);
/** @var \OCP\Files\Mount\IMountPoint[] $rootMountProviders */
$rootMountProviders = $mountProviderCollection->getRootMounts();
/** @var \OC\Files\Mount\Manager $mountManager */
$mountManager = \OC\Files\Filesystem::getMountManager();
foreach ($rootMountProviders as $rootMountProvider) {
$mountManager->addMount($rootMountProvider);
}
if ($user != '' && !\OC::$server->getUserManager()->userExists($user)) {
\OC::$server->getEventLogger()->end('setup_fs');
return false;

41
lib/public/Files/Config/IRootMountProvider.php Normal file
View File

@ -0,0 +1,41 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
*
* @author Morris Jobke <hey@morrisjobke.de>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCP\Files\Config;
use OCP\Files\Storage\IStorageFactory;
/**
* @since 20.0.0
*/
interface IRootMountProvider {
/**
* Get all root mountpoints
*
* @return \OCP\Files\Mount\IMountPoint[]
* @since 20.0.0
*/
public function getRootMounts(IStorageFactory $loader): array;
}

113
tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php Normal file
View File

@ -0,0 +1,113 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
*
* @author Morris Jobke <hey@morrisjobke.de>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Files\Mount;
use OC\Files\Mount\ObjectStorePreviewCacheMountProvider;
use OC\Files\ObjectStore\S3;
use OC\Files\Storage\StorageFactory;
use OCP\Files\Storage\IStorageFactory;
use OCP\IConfig;
use OCP\ILogger;
use PHPUnit\Framework\MockObject\MockObject;
/**
* @group DB
*
* The DB permission is needed for the fake root storage initialization
*/
class ObjectStorePreviewCacheMountProviderTest extends \Test\TestCase {
/** @var ObjectStorePreviewCacheMountProvider */
protected $provider;
/** @var ILogger|MockObject */
protected $logger;
/** @var IConfig|MockObject */
protected $config;
/** @var IStorageFactory|MockObject */
protected $loader;
protected function setUp(): void {
parent::setUp();
$this->logger = $this->createMock(ILogger::class);
$this->config = $this->createMock(IConfig::class);
$this->loader = $this->createMock(StorageFactory::class);
$this->provider = new ObjectStorePreviewCacheMountProvider($this->logger, $this->config);
}
public function testNoMultibucketObjectStorage() {
$this->config->expects($this->once())
->method('getSystemValue')
->with('objectstore_multibucket')
->willReturn(null);
$this->assertEquals([], $this->provider->getRootMounts($this->loader));
}
public function testMultibucketObjectStorage() {
$objectstoreConfig = [
'class' => S3::class,
'arguments' => [
'bucket' => 'abc',
'num_buckets' => 64,
'key' => 'KEY',
'secret' => 'SECRET',
'hostname' => 'IP',
'port' => 'PORT',
'use_ssl' => false,
'use_path_style' => true,
],
];
$this->config->expects($this->any())
->method('getSystemValue')
->willReturnCallback(function ($config) use ($objectstoreConfig) {
if ($config === 'objectstore_multibucket') {
return $objectstoreConfig;
} elseif ($config === 'objectstore.multibucket.preview-distribution') {
return true;
}
return null;
});
$this->config->expects($this->once())
->method('getSystemValueString')
->with('instanceid')
->willReturn('INSTANCEID');
$mounts = $this->provider->getRootMounts($this->loader);
// 256 mounts for the subfolders and 1 for the fake root
$this->assertCount(257, $mounts);
// do some sanity checks if they have correct mount point paths
$this->assertEquals('/appdata_INSTANCEID/preview/0/0/', $mounts[0]->getMountPoint());
$this->assertEquals('/appdata_INSTANCEID/preview/2/5/', $mounts[37]->getMountPoint());
// also test the path of the fake bucket
$this->assertEquals('/appdata_INSTANCEID/preview/old-multibucket/', $mounts[256]->getMountPoint());
}
}