Distribute preview folders in appdata in multibucket setup to multiple buckets
* introduces a new IRootMountProvider to register mount points inside the root storage * adds a AppdataPreviewObjectStoreStorage to handle the split between preview folders and bucket number Ref #22033 Signed-off-by: Morris Jobke <hey@morrisjobke.de>
This commit is contained in:
Morris Jobke
parent
c24f3d1336
commit
438ac23e2a
|
|
@ -230,6 +230,7 @@ return array(
|
|||
'OCP\\Files\\Config\\IHomeMountProvider' => $baseDir . '/lib/public/Files/Config/IHomeMountProvider.php',
|
||||
'OCP\\Files\\Config\\IMountProvider' => $baseDir . '/lib/public/Files/Config/IMountProvider.php',
|
||||
'OCP\\Files\\Config\\IMountProviderCollection' => $baseDir . '/lib/public/Files/Config/IMountProviderCollection.php',
|
||||
'OCP\\Files\\Config\\IRootMountProvider' => $baseDir . '/lib/public/Files/Config/IRootMountProvider.php',
|
||||
'OCP\\Files\\Config\\IUserMountCache' => $baseDir . '/lib/public/Files/Config/IUserMountCache.php',
|
||||
'OCP\\Files\\EmptyFileNameException' => $baseDir . '/lib/public/Files/EmptyFileNameException.php',
|
||||
'OCP\\Files\\EntityTooLargeException' => $baseDir . '/lib/public/Files/EntityTooLargeException.php',
|
||||
|
|
@ -1029,6 +1030,7 @@ return array(
|
|||
'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php',
|
||||
'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php',
|
||||
'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php',
|
||||
'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php',
|
||||
'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php',
|
||||
'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php',
|
||||
'OC\\Files\\Node\\HookConnector' => $baseDir . '/lib/private/Files/Node/HookConnector.php',
|
||||
|
|
@ -1040,6 +1042,7 @@ return array(
|
|||
'OC\\Files\\Node\\Root' => $baseDir . '/lib/private/Files/Node/Root.php',
|
||||
'OC\\Files\\Notify\\Change' => $baseDir . '/lib/private/Files/Notify/Change.php',
|
||||
'OC\\Files\\Notify\\RenameChange' => $baseDir . '/lib/private/Files/Notify/RenameChange.php',
|
||||
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
|
||||
'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
|
||||
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
|
||||
'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',
|
||||
|
|
|
|||
|
|
@ -259,6 +259,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
|
|||
'OCP\\Files\\Config\\IHomeMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IHomeMountProvider.php',
|
||||
'OCP\\Files\\Config\\IMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProvider.php',
|
||||
'OCP\\Files\\Config\\IMountProviderCollection' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProviderCollection.php',
|
||||
'OCP\\Files\\Config\\IRootMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IRootMountProvider.php',
|
||||
'OCP\\Files\\Config\\IUserMountCache' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IUserMountCache.php',
|
||||
'OCP\\Files\\EmptyFileNameException' => __DIR__ . '/../../..' . '/lib/public/Files/EmptyFileNameException.php',
|
||||
'OCP\\Files\\EntityTooLargeException' => __DIR__ . '/../../..' . '/lib/public/Files/EntityTooLargeException.php',
|
||||
|
|
@ -1058,6 +1059,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
|
|||
'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php',
|
||||
'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php',
|
||||
'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php',
|
||||
'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php',
|
||||
'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php',
|
||||
'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php',
|
||||
'OC\\Files\\Node\\HookConnector' => __DIR__ . '/../../..' . '/lib/private/Files/Node/HookConnector.php',
|
||||
|
|
@ -1069,6 +1071,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
|
|||
'OC\\Files\\Node\\Root' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Root.php',
|
||||
'OC\\Files\\Notify\\Change' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/Change.php',
|
||||
'OC\\Files\\Notify\\RenameChange' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/RenameChange.php',
|
||||
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
|
||||
'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
|
||||
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
|
||||
'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ use OC\Hooks\EmitterTrait;
|
|||
use OCP\Files\Config\IHomeMountProvider;
|
||||
use OCP\Files\Config\IMountProvider;
|
||||
use OCP\Files\Config\IMountProviderCollection;
|
||||
use OCP\Files\Config\IRootMountProvider;
|
||||
use OCP\Files\Config\IUserMountCache;
|
||||
use OCP\Files\Mount\IMountManager;
|
||||
use OCP\Files\Mount\IMountPoint;
|
||||
|
|
@ -49,6 +50,9 @@ class MountProviderCollection implements IMountProviderCollection, Emitter {
|
|||
*/
|
||||
private $providers = [];
|
||||
|
||||
/** @var \OCP\Files\Config\IRootMountProvider[] */
|
||||
private $rootProviders = [];
|
||||
|
||||
/**
|
||||
* @var \OCP\Files\Storage\IStorageFactory
|
||||
*/
|
||||
|
|
@ -198,4 +202,19 @@ class MountProviderCollection implements IMountProviderCollection, Emitter {
|
|||
public function getMountCache() {
|
||||
return $this->mountCache;
|
||||
}
|
||||
|
||||
public function registerRootProvider(IRootMountProvider $provider) {
|
||||
$this->rootProviders[] = $provider;
|
||||
}
|
||||
|
||||
public function getRootMounts(): array {
|
||||
$loader = $this->loader;
|
||||
$mounts = array_map(function (IRootMountProvider $provider) use ($loader) {
|
||||
return $provider->getRootMounts($loader);
|
||||
}, $this->rootProviders);
|
||||
$mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) {
|
||||
return array_merge($mounts, $providerMounts);
|
||||
}, []);
|
||||
return $mounts;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
|
||||
*
|
||||
* @author Morris Jobke <hey@morrisjobke.de>
|
||||
*
|
||||
* @license GNU AGPL version 3 or any later version
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
namespace OC\Files\Mount;
|
||||
|
||||
use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage;
|
||||
use OCP\Files\Config\IRootMountProvider;
|
||||
use OCP\Files\Storage\IStorageFactory;
|
||||
use OCP\IConfig;
|
||||
use OCP\ILogger;
|
||||
|
||||
/**
|
||||
* Mount provider for object store app data folder for previews
|
||||
*/
|
||||
class ObjectStorePreviewCacheMountProvider implements IRootMountProvider {
|
||||
/** @var ILogger */
|
||||
private $logger;
|
||||
/** @var IConfig */
|
||||
private $config;
|
||||
|
||||
public function __construct(ILogger $logger, IConfig $config) {
|
||||
$this->logger = $logger;
|
||||
$this->config = $config;
|
||||
}
|
||||
|
||||
public function getRootMounts(IStorageFactory $loader): array {
|
||||
if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$instanceId = $this->config->getSystemValueString('instanceid', '');
|
||||
$mountPoints = [];
|
||||
$directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'];
|
||||
$i = 0;
|
||||
foreach ($directoryRange as $parent) {
|
||||
foreach ($directoryRange as $child) {
|
||||
$mountPoints[] = new MountPoint(
|
||||
AppdataPreviewObjectStoreStorage::class,
|
||||
'/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child,
|
||||
$this->getMultiBucketObjectStore($i),
|
||||
$loader
|
||||
);
|
||||
$i++;
|
||||
}
|
||||
}
|
||||
return $mountPoints;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array
|
||||
*/
|
||||
protected function getMultiBucketObjectStore(int $number): array {
|
||||
$config = $this->config->getSystemValue('objectstore_multibucket');
|
||||
|
||||
// sanity checks
|
||||
if (empty($config['class'])) {
|
||||
$this->logger->error('No class given for objectstore', ['app' => 'files']);
|
||||
}
|
||||
if (!isset($config['arguments'])) {
|
||||
$config['arguments'] = [];
|
||||
}
|
||||
|
||||
/*
|
||||
* Use any provided bucket argument as prefix
|
||||
* and add the mapping from parent/child => bucket
|
||||
*/
|
||||
if (!isset($config['arguments']['bucket'])) {
|
||||
$config['arguments']['bucket'] = '';
|
||||
}
|
||||
$numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64;
|
||||
$config['arguments']['bucket'] .= (string)($number % $numBuckets);
|
||||
|
||||
// instantiate object store implementation
|
||||
$config['arguments']['objectstore'] = new $config['class']($config['arguments']);
|
||||
|
||||
$config['arguments']['internal-id'] = $number;
|
||||
|
||||
return $config['arguments'];
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
|
||||
*
|
||||
* @author Morris Jobke <hey@morrisjobke.de>
|
||||
*
|
||||
* @license GNU AGPL version 3 or any later version
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
namespace OC\Files\ObjectStore;
|
||||
|
||||
class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage {
|
||||
|
||||
/** @var string */
|
||||
private $internalId;
|
||||
|
||||
public function __construct($params) {
|
||||
if (!isset($params['internal-id'])) {
|
||||
throw new \Exception('missing id in parameters');
|
||||
}
|
||||
$this->internalId = (string)$params['internal-id'];
|
||||
parent::__construct($params);
|
||||
}
|
||||
|
||||
public function getId() {
|
||||
return 'object::appdata::preview:' . $this->internalId;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* @copyright Copyright (c) 2020, Roeland Jago Douma <roeland@famdouma.nl>
|
||||
*
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ use OC\Files\Config\UserMountCacheListener;
|
|||
use OC\Files\Mount\CacheMountProvider;
|
||||
use OC\Files\Mount\LocalHomeMountProvider;
|
||||
use OC\Files\Mount\ObjectHomeMountProvider;
|
||||
use OC\Files\Mount\ObjectStorePreviewCacheMountProvider;
|
||||
use OC\Files\Node\HookConnector;
|
||||
use OC\Files\Node\LazyRoot;
|
||||
use OC\Files\Node\Root;
|
||||
|
|
@ -904,9 +905,11 @@ class Server extends ServerContainer implements IServerContainer {
|
|||
// builtin providers
|
||||
|
||||
$config = $c->getConfig();
|
||||
$logger = $c->getLogger();
|
||||
$manager->registerProvider(new CacheMountProvider($config));
|
||||
$manager->registerHomeProvider(new LocalHomeMountProvider());
|
||||
$manager->registerHomeProvider(new ObjectHomeMountProvider($config));
|
||||
$manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config));
|
||||
|
||||
return $manager;
|
||||
});
|
||||
|
|
|
|||
|
|
@ -297,6 +297,17 @@ class OC_Util {
|
|||
self::initLocalStorageRootFS();
|
||||
}
|
||||
|
||||
/** @var \OCP\Files\Config\IMountProviderCollection $mountProviderCollection */
|
||||
$mountProviderCollection = \OC::$server->query(\OCP\Files\Config\IMountProviderCollection::class);
|
||||
/** @var \OCP\Files\Mount\IMountPoint[] $rootMountProviders */
|
||||
$rootMountProviders = $mountProviderCollection->getRootMounts();
|
||||
|
||||
/** @var \OC\Files\Mount\Manager $mountManager */
|
||||
$mountManager = \OC\Files\Filesystem::getMountManager();
|
||||
foreach ($rootMountProviders as $rootMountProvider) {
|
||||
$mountManager->addMount($rootMountProvider);
|
||||
}
|
||||
|
||||
if ($user != '' && !\OC::$server->getUserManager()->userExists($user)) {
|
||||
\OC::$server->getEventLogger()->end('setup_fs');
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
|
||||
*
|
||||
* @author Morris Jobke <hey@morrisjobke.de>
|
||||
*
|
||||
* @license GNU AGPL version 3 or any later version
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
namespace OCP\Files\Config;
|
||||
|
||||
use OCP\Files\Storage\IStorageFactory;
|
||||
|
||||
/**
|
||||
* @since 20.0.0
|
||||
*/
|
||||
interface IRootMountProvider {
|
||||
/**
|
||||
* Get all root mountpoints
|
||||
*
|
||||
* @return \OCP\Files\Mount\IMountPoint[]
|
||||
* @since 20.0.0
|
||||
*/
|
||||
public function getRootMounts(IStorageFactory $loader): array;
|
||||
}
|
||||
Loading…
Reference in New Issue