mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #10724 from nextcloud/large-share-count-performance 

Improve performance when dealing with large numbers of shares
This commit is contained in:
Morris Jobke 2018-08-24 17:10:17 +02:00 committed by GitHub
parent 6099786c8d 67f3d6f0a6
commit e8817ed3ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 128 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
apps/files_sharing/lib/MountProvider.php
View File

@ -27,6 +27,8 @@
namespace OCA\Files_Sharing;
use OC\Cache\CappedMemoryCache;
use OC\Files\View;
use OCP\Files\Config\IMountProvider;
use OCP\Files\Storage\IStorageFactory;
use OCP\IConfig;
@ -84,20 +86,35 @@ class MountProvider implements IMountProvider {
$superShares = $this->buildSuperShares($shares, $user);
$mounts = [];
$view = new View('/' . $user->getUID() . '/files');
$ownerViews = [];
$sharingDisabledForUser = $this->shareManager->sharingDisabledForUser($user->getUID());
$foldersExistCache = new CappedMemoryCache();
foreach ($superShares as $share) {
try {
$mounts[] = new SharedMount(
/** @var \OCP\Share\IShare $parentShare */
$parentShare = $share[0];
$owner = $parentShare->getShareOwner();
if (!isset($ownerViews[$owner])) {
$ownerViews[$owner] = new View('/' . $parentShare->getShareOwner() . '/files');
}
$mount = new SharedMount(
'\OCA\Files_Sharing\SharedStorage',
$mounts,
[
'user' => $user->getUID(),
// parent share
'superShare' => $share[0],
'superShare' => $parentShare,
// children/component of the superShare
'groupedShares' => $share[1],
'ownerView' => $ownerViews[$owner],
'sharingDisabledForUser' => $sharingDisabledForUser
],
$storageFactory
$storageFactory,
$view,
$foldersExistCache
);
$mounts[$mount->getMountPoint()] = $mount;
} catch (\Exception $e) {
$this->logger->logException($e);
$this->logger->error('Error while trying to create shared mount');
@ -105,7 +122,7 @@ class MountProvider implements IMountProvider {
}
// array_filter removes the null values from the array
return array_filter($mounts);
return array_values(array_filter($mounts));
}
/**

38
apps/files_sharing/lib/SharedMount.php
View File

@ -28,10 +28,12 @@
namespace OCA\Files_Sharing;
use OC\Cache\CappedMemoryCache;
use OC\Files\Filesystem;
use OC\Files\Mount\MountPoint;
use OC\Files\Mount\MoveableMount;
use OC\Files\View;
use OCP\Files\Storage\IStorageFactory;
/**
* Shared mount points can be moved by the user
@ -61,19 +63,19 @@ class SharedMount extends MountPoint implements MoveableMount {
/**
* @param string $storage
* @param SharedMount[] $mountpoints
* @param array|null $arguments
* @param \OCP\Files\Storage\IStorageFactory $loader
* @param array $arguments
* @param IStorageFactory $loader
* @param View $recipientView
*/
public function __construct($storage, array $mountpoints, $arguments = null, $loader = null) {
public function __construct($storage, array $mountpoints, $arguments, IStorageFactory $loader, View $recipientView, CappedMemoryCache $folderExistCache) {
$this->user = $arguments['user'];
$this->recipientView = new View('/' . $this->user . '/files');
$this->recipientView = $recipientView;
$this->superShare = $arguments['superShare'];
$this->groupedShares = $arguments['groupedShares'];
$newMountPoint = $this->verifyMountPoint($this->superShare, $mountpoints);
$newMountPoint = $this->verifyMountPoint($this->superShare, $mountpoints, $folderExistCache);
$absMountPoint = '/' . $this->user . '/files' . $newMountPoint;
$arguments['ownerView'] = new View('/' . $this->superShare->getShareOwner() . '/files');
parent::__construct($storage, $absMountPoint, $arguments, $loader);
}
@ -84,12 +86,18 @@ class SharedMount extends MountPoint implements MoveableMount {
* @param SharedMount[] $mountpoints
* @return string
*/
private function verifyMountPoint(\OCP\Share\IShare $share, array $mountpoints) {
private function verifyMountPoint(\OCP\Share\IShare $share, array $mountpoints, CappedMemoryCache $folderExistCache) {
$mountPoint = basename($share->getTarget());
$parent = dirname($share->getTarget());
if (!$this->recipientView->is_dir($parent)) {
if ($folderExistCache->hasKey($parent)) {
$parentExists = $folderExistCache->get($parent);
} else {
$parentExists = $this->recipientView->is_dir($parent);
$folderExistCache->set($parent, $parentExists);
}
if (!$parentExists) {
$parent = Helper::getShareFolder($this->recipientView);
}
@ -135,19 +143,11 @@ class SharedMount extends MountPoint implements MoveableMount {
$name = $pathinfo['filename'];
$dir = $pathinfo['dirname'];
// Helper function to find existing mount points
$mountpointExists = function ($path) use ($mountpoints) {
foreach ($mountpoints as $mountpoint) {
if ($mountpoint->getShare()->getTarget() === $path) {
return true;
}
}
return false;
};
$i = 2;
while ($view->file_exists($path) || $mountpointExists($path)) {
$absolutePath = $this->recipientView->getAbsolutePath($path) . '/';
while ($view->file_exists($path) || isset($mountpoints[$absolutePath])) {
$path = Filesystem::normalizePath($dir . '/' . $name . ' (' . $i . ')' . $ext);
$absolutePath = $this->recipientView->getAbsolutePath($path) . '/';
$i++;
}

10
apps/files_sharing/lib/SharedStorage.php
View File

@ -79,6 +79,9 @@ class SharedStorage extends \OC\Files\Storage\Wrapper\Jail implements ISharedSto
private $options;
/** @var boolean */
private $sharingDisabledForUser;
public function __construct($arguments) {
$this->ownerView = $arguments['ownerView'];
$this->logger = \OC::$server->getLogger();
@ -87,6 +90,11 @@ class SharedStorage extends \OC\Files\Storage\Wrapper\Jail implements ISharedSto
$this->groupedShares = $arguments['groupedShares'];
$this->user = $arguments['user'];
if (isset($arguments['sharingDisabledForUser'])) {
$this->sharingDisabledForUser = $arguments['sharingDisabledForUser'];
} else {
$this->sharingDisabledForUser = false;
}
parent::__construct([
'storage' => null,
@ -193,7 +201,7 @@ class SharedStorage extends \OC\Files\Storage\Wrapper\Jail implements ISharedSto
$permissions |= \OCP\Constants::PERMISSION_DELETE;
}
if (\OCP\Util::isSharingDisabledForUser()) {
if ($this->sharingDisabledForUser) {
$permissions &= ~\OCP\Constants::PERMISSION_SHARE;
}

51
lib/private/Files/Config/UserMountCache.php
View File

@ -101,17 +101,31 @@ class UserMountCache implements IUserMountCache {
}
}, $mounts);
$newMounts = array_values(array_filter($newMounts));
$newMountRootIds = array_map(function (ICachedMountInfo $mount) {
return $mount->getRootId();
}, $newMounts);
$newMounts = array_combine($newMountRootIds, $newMounts);
$cachedMounts = $this->getMountsForUser($user);
$mountDiff = function (ICachedMountInfo $mount1, ICachedMountInfo $mount2) {
// since we are only looking for mounts for a specific user comparing on root id is enough
return $mount1->getRootId() - $mount2->getRootId();
};
$cachedMountRootIds = array_map(function (ICachedMountInfo $mount) {
return $mount->getRootId();
}, $cachedMounts);
$cachedMounts = array_combine($cachedMountRootIds, $cachedMounts);
/** @var ICachedMountInfo[] $addedMounts */
$addedMounts = array_udiff($newMounts, $cachedMounts, $mountDiff);
/** @var ICachedMountInfo[] $removedMounts */
$removedMounts = array_udiff($cachedMounts, $newMounts, $mountDiff);
$addedMounts = [];
$removedMounts = [];
foreach ($newMounts as $rootId => $newMount) {
if (!isset($cachedMounts[$rootId])) {
$addedMounts[] = $newMount;
}
}
foreach ($cachedMounts as $rootId => $cachedMount) {
if (!isset($newMounts[$rootId])) {
$removedMounts[] = $cachedMount;
}
}
$changedMounts = $this->findChangedMounts($newMounts, $cachedMounts);
@ -135,16 +149,19 @@ class UserMountCache implements IUserMountCache {
* @return ICachedMountInfo[]
*/
private function findChangedMounts(array $newMounts, array $cachedMounts) {
$new = [];
foreach ($newMounts as $mount) {
$new[$mount->getRootId()] = $mount;
}
$changed = [];
foreach ($newMounts as $newMount) {
foreach ($cachedMounts as $cachedMount) {
foreach ($cachedMounts as $cachedMount) {
$rootId = $cachedMount->getRootId();
if (isset($new[$rootId])) {
$newMount = $new[$rootId];
if (
$newMount->getRootId() === $cachedMount->getRootId() &&
(
$newMount->getMountPoint() !== $cachedMount->getMountPoint() ||
$newMount->getStorageId() !== $cachedMount->getStorageId() ||
$newMount->getMountId() !== $cachedMount->getMountId()
)
$newMount->getMountPoint() !== $cachedMount->getMountPoint() ||
$newMount->getStorageId() !== $cachedMount->getStorageId() ||
$newMount->getMountId() !== $cachedMount->getMountId()
) {
$changed[] = $newMount;
}
@ -197,7 +214,7 @@ class UserMountCache implements IUserMountCache {
}
$mount_id = $row['mount_id'];
if (!is_null($mount_id)) {
$mount_id = (int) $mount_id;
$mount_id = (int)$mount_id;
}
return new CachedMountInfo($user, (int)$row['storage_id'], (int)$row['root_id'], $row['mount_point'], $mount_id, isset($row['path']) ? $row['path'] : '');
}

52
lib/private/Files/Mount/Manager.php
View File

@ -38,8 +38,12 @@ class Manager implements IMountManager {
/** @var CappedMemoryCache */
private $pathCache;
/** @var CappedMemoryCache */
private $inPathCache;
public function __construct() {
$this->pathCache = new CappedMemoryCache();
$this->inPathCache = new CappedMemoryCache();
}
/**
@ -48,6 +52,7 @@ class Manager implements IMountManager {
public function addMount(IMountPoint $mount) {
$this->mounts[$mount->getMountPoint()] = $mount;
$this->pathCache->clear();
$this->inPathCache->clear();
}
/**
@ -60,16 +65,18 @@ class Manager implements IMountManager {
}
unset($this->mounts[$mountPoint]);
$this->pathCache->clear();
$this->inPathCache->clear();
}
/**
* @param string $mountPoint
* @param string $target
*/
public function moveMount(string $mountPoint, string $target){
public function moveMount(string $mountPoint, string $target) {
$this->mounts[$target] = $this->mounts[$mountPoint];
unset($this->mounts[$mountPoint]);
$this->pathCache->clear();
$this->inPathCache->clear();
}
/**
@ -80,32 +87,29 @@ class Manager implements IMountManager {
*/
public function find(string $path) {
\OC_Util::setupFS();
$path = $this->formatPath($path);
if (isset($this->mounts[$path])) {
return $this->mounts[$path];
}
$path = Filesystem::normalizePath($path);
if (isset($this->pathCache[$path])) {
return $this->pathCache[$path];
}
\OC_Hook::emit('OC_Filesystem', 'get_mountpoint', ['path' => $path]);
$foundMountPoint = '';
$mountPoints = array_keys($this->mounts);
$foundMountPointLength = 0;
foreach ($mountPoints as $mountpoint) {
if (\strlen($mountpoint) > $foundMountPointLength && strpos($path, $mountpoint) === 0) {
$foundMountPoint = $mountpoint;
$foundMountPointLength = \strlen($foundMountPoint);
$current = $path;
while (true) {
$mountPoint = $current . '/';
if (isset($this->mounts[$mountPoint])) {
$this->pathCache[$path] = $this->mounts[$mountPoint];
return $this->mounts[$mountPoint];
}
if ($current === '') {
return null;
}
$current = dirname($current);
if ($current === '.' || $current === '/') {
$current = '';
}
}
if (isset($this->mounts[$foundMountPoint])) {
$this->pathCache[$path] = $this->mounts[$foundMountPoint];
return $this->mounts[$foundMountPoint];
}
return null;
}
/**
@ -117,6 +121,11 @@ class Manager implements IMountManager {
public function findIn(string $path): array {
\OC_Util::setupFS();
$path = $this->formatPath($path);
if (isset($this->inPathCache[$path])) {
return $this->inPathCache[$path];
}
$result = [];
$pathLength = \strlen($path);
$mountPoints = array_keys($this->mounts);
@ -125,12 +134,15 @@ class Manager implements IMountManager {
$result[] = $this->mounts[$mountPoint];
}
}
$this->inPathCache[$path] = $result;
return $result;
}
public function clear() {
$this->mounts = [];
$this->pathCache->clear();
$this->inPathCache->clear();
}
/**

20
lib/private/Files/View.php
View File

@ -1433,16 +1433,21 @@ class View {
$contents = $cache->getFolderContentsById($folderId); //TODO: mimetype_filter
$sharingDisabled = \OCP\Util::isSharingDisabledForUser();
$fileNames = array_map(function(ICacheEntry $content) {
return $content->getName();
}, $contents);
/**
* @var \OC\Files\FileInfo[] $files
* @var \OC\Files\FileInfo[] $fileInfos
*/
$files = array_map(function (ICacheEntry $content) use ($path, $storage, $mount, $sharingDisabled) {
$fileInfos = array_map(function (ICacheEntry $content) use ($path, $storage, $mount, $sharingDisabled) {
if ($sharingDisabled) {
$content['permissions'] = $content['permissions'] & ~\OCP\Constants::PERMISSION_SHARE;
}
$owner = $this->getUserObjectForOwner($storage->getOwner($content['path']));
return new FileInfo($path . '/' . $content['name'], $storage, $content['path'], $content, $mount, $owner);
}, $contents);
$files = array_combine($fileNames, $fileInfos);
//add a folder for any mountpoint in this directory and add the sizes of other mountpoints to the folders
$mounts = Filesystem::getMountManager()->findIn($path);
@ -1496,13 +1501,6 @@ class View {
$rootEntry['permissions'] = $permissions & (\OCP\Constants::PERMISSION_ALL - (\OCP\Constants::PERMISSION_UPDATE | \OCP\Constants::PERMISSION_DELETE));
}
//remove any existing entry with the same name
foreach ($files as $i => $file) {
if ($file['name'] === $rootEntry['name']) {
unset($files[$i]);
break;
}
}
$rootEntry['path'] = substr(Filesystem::normalizePath($path . '/' . $rootEntry['name']), strlen($user) + 2); // full path without /$user/
// if sharing was disabled for the user we remove the share permissions
@ -1511,7 +1509,7 @@ class View {
}
$owner = $this->getUserObjectForOwner($subStorage->getOwner(''));
$files[] = new FileInfo($path . '/' . $rootEntry['name'], $subStorage, '', $rootEntry, $mount, $owner);
$files[$rootEntry->getName()] = new FileInfo($path . '/' . $rootEntry['name'], $subStorage, '', $rootEntry, $mount, $owner);
}
}
}
@ -1527,7 +1525,7 @@ class View {
});
}
return $files;
return array_values($files);
} else {
return [];
}

4
lib/private/Share20/Manager.php
View File

@ -1208,6 +1208,10 @@ class Manager implements IManager {
* @throws ShareNotFound
*/
public function getShareByToken($token) {
// tokens can't be valid local user names
if ($this->userManager->userExists($token)) {
throw new ShareNotFound();
}
$share = null;
try {
if($this->shareApiAllowLinks()) {