From 02be8a3a12fbd63384f3dd79653ac94b05e646e6 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 10 Nov 2015 13:32:05 +0100 Subject: [PATCH 1/7] Split getting cache entry and checking update to a seperate method --- lib/private/files/view.php | 104 ++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 58 deletions(-) diff --git a/lib/private/files/view.php b/lib/private/files/view.php index cee4b18242..2bac35cb18 100644 --- a/lib/private/files/view.php +++ b/lib/private/files/view.php @@ -1178,6 +1178,43 @@ class View { } } + /** + * @param \OC\Files\Storage\Storage $storage + * @param string $internalPath + * @param string $relativePath + * @return array|bool + */ + private function getCacheEntry($storage, $internalPath, $relativePath) { + $cache = $storage->getCache($internalPath); + $data = $cache->get($internalPath); + $watcher = $storage->getWatcher($internalPath); + + try { + // if the file is not in the cache or needs to be updated, trigger the scanner and reload the data + if (!$data) { + $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); + if (!$storage->file_exists($internalPath)) { + $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); + return false; + } + $scanner = $storage->getScanner($internalPath); + $scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW); + $data = $cache->get($internalPath); + $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); + } else if (!Cache\Scanner::isPartialFile($internalPath) && $watcher->needsUpdate($internalPath, $data)) { + $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); + $watcher->update($internalPath, $data); + $this->updater->propagate($path); + $data = $cache->get($internalPath); + $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); + } + } catch (LockedException $e) { + // if the file is locked we just use the old cache info + } + + return $data; + } + /** * get the filesystem info * @@ -1204,46 +1241,15 @@ class View { $internalPath = $mount->getInternalPath($path); $data = null; if ($storage) { - $cache = $storage->getCache($internalPath); - - $data = $cache->get($internalPath); - $watcher = $storage->getWatcher($internalPath); - - try { - // if the file is not in the cache or needs to be updated, trigger the scanner and reload the data - if (!$data) { - $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); - if (!$storage->file_exists($internalPath)) { - $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); - return false; - } - $scanner = $storage->getScanner($internalPath); - $scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW); - $data = $cache->get($internalPath); - $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); - } else if (!Cache\Scanner::isPartialFile($internalPath) && $watcher->needsUpdate($internalPath, $data)) { - $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); - $watcher->update($internalPath, $data); - $this->updater->propagate($path); - $data = $cache->get($internalPath); - $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); - } - } catch (LockedException $e) { - // if the file is locked we just use the old cache info - } + $data = $this->getCacheEntry($storage, $internalPath, $relativePath); if ($data and isset($data['fileid'])) { - // upgrades from oc6 or lower might not have the permissions set in the file cache - if ($data['permissions'] === 0) { - $data['permissions'] = $storage->getPermissions($data['path']); - $cache->update($data['fileid'], array('permissions' => $data['permissions'])); - } if ($includeMountPoints and $data['mimetype'] === 'httpd/unix-directory') { //add the sizes of other mount points to the folder $extOnly = ($includeMountPoints === 'ext'); - $mountPoints = Filesystem::getMountPoints($path); - foreach ($mountPoints as $mountPoint) { - $subStorage = Filesystem::getStorage($mountPoint); + $mounts = Filesystem::getMountManager()->findIn($path); + foreach ($mounts as $mount) { + $subStorage = $mount->getStorage(); if ($subStorage) { // exclude shared storage ? if ($extOnly && $subStorage instanceof \OC\Files\Storage\Shared) { @@ -1294,30 +1300,12 @@ class View { /** * @var \OC\Files\FileInfo[] $files */ - $files = array(); + $files = []; - $data = $cache->get($internalPath); - $watcher = $storage->getWatcher($internalPath); - try { - if (!$data or $data['size'] === -1) { - $this->lockFile($directory, ILockingProvider::LOCK_SHARED); - if (!$storage->file_exists($internalPath)) { - $this->unlockFile($directory, ILockingProvider::LOCK_SHARED); - return array(); - } - $scanner = $storage->getScanner($internalPath); - $scanner->scan($internalPath, Cache\Scanner::SCAN_SHALLOW); - $data = $cache->get($internalPath); - $this->unlockFile($directory, ILockingProvider::LOCK_SHARED); - } else if ($watcher->needsUpdate($internalPath, $data)) { - $this->lockFile($directory, ILockingProvider::LOCK_SHARED); - $watcher->update($internalPath, $data); - $this->updater->propagate($path); - $data = $cache->get($internalPath); - $this->unlockFile($directory, ILockingProvider::LOCK_SHARED); - } - } catch (LockedException $e) { - // if the file is locked we just use the old cache info + $data = $this->getCacheEntry($storage, $internalPath, $directory); + + if (!is_array($data) || !isset($data['fileid'])) { + return []; } $folderId = $data['fileid']; From 1736c70075c97aa4a10caf3c1a5790a11cce8675 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 10 Nov 2015 13:52:02 +0100 Subject: [PATCH 2/7] Some more cleanup for getFIleInfo/getDirectoryContent --- lib/private/files/fileinfo.php | 11 ++++++ lib/private/files/view.php | 72 +++++++++++++++------------------- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/lib/private/files/fileinfo.php b/lib/private/files/fileinfo.php index bb810dd45e..2d2ef325d3 100644 --- a/lib/private/files/fileinfo.php +++ b/lib/private/files/fileinfo.php @@ -285,4 +285,15 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { public function getOwner() { return $this->owner; } + + /** + * Add a cache entry which is the child of this folder + * + * Sets the size, etag and size to for cross-storage childs + * + * @param array $data cache entry for the child + */ + public function addSubEntry($data) { + $this->data['size'] += isset($data['size']) ? $data['size'] : 0; + } } diff --git a/lib/private/files/view.php b/lib/private/files/view.php index 2bac35cb18..25bf7b547c 100644 --- a/lib/private/files/view.php +++ b/lib/private/files/view.php @@ -1226,9 +1226,8 @@ class View { */ public function getFileInfo($path, $includeMountPoints = true) { $this->assertPathLength($path); - $data = array(); if (!Filesystem::isValidPath($path)) { - return $data; + return false; } if (Cache\Scanner::isPartialFile($path)) { return $this->getPartFileInfo($path); @@ -1239,10 +1238,16 @@ class View { $mount = Filesystem::getMountManager()->find($path); $storage = $mount->getStorage(); $internalPath = $mount->getInternalPath($path); - $data = null; if ($storage) { $data = $this->getCacheEntry($storage, $internalPath, $relativePath); + if ($mount instanceof MoveableMount && $internalPath === '') { + $data['permissions'] |= \OCP\Constants::PERMISSION_DELETE; + } + + $owner = $this->getUserObjectForOwner($storage->getOwner($internalPath)); + $info = new FileInfo($path, $storage, $internalPath, $data, $mount, $owner); + if ($data and isset($data['fileid'])) { if ($includeMountPoints and $data['mimetype'] === 'httpd/unix-directory') { //add the sizes of other mount points to the folder @@ -1257,22 +1262,16 @@ class View { } $subCache = $subStorage->getCache(''); $rootEntry = $subCache->get(''); - $data['size'] += isset($rootEntry['size']) ? $rootEntry['size'] : 0; + $info->addSubEntry($rootEntry); } } } } - } - if (!$data) { - return false; + + return $info; } - if ($mount instanceof MoveableMount && $internalPath === '') { - $data['permissions'] |= \OCP\Constants::PERMISSION_DELETE; - } - - $owner = $this->getUserObjectForOwner($storage->getOwner($internalPath)); - return new FileInfo($path, $storage, $internalPath, $data, $mount, $owner); + return false; } /** @@ -1284,9 +1283,8 @@ class View { */ public function getDirectoryContent($directory, $mimetype_filter = '') { $this->assertPathLength($directory); - $result = array(); if (!Filesystem::isValidPath($directory)) { - return $result; + return []; } $path = $this->getAbsolutePath($directory); $path = Filesystem::normalizePath($path); @@ -1297,11 +1295,6 @@ class View { $cache = $storage->getCache($internalPath); $user = \OC_User::getUser(); - /** - * @var \OC\Files\FileInfo[] $files - */ - $files = []; - $data = $this->getCacheEntry($storage, $internalPath, $directory); if (!is_array($data) || !isset($data['fileid'])) { @@ -1311,18 +1304,16 @@ class View { $folderId = $data['fileid']; $contents = $cache->getFolderContentsById($folderId); //TODO: mimetype_filter - foreach ($contents as $content) { - if ($content['permissions'] === 0) { - $content['permissions'] = $storage->getPermissions($content['path']); - $cache->update($content['fileid'], array('permissions' => $content['permissions'])); - } - // if sharing was disabled for the user we remove the share permissions + /** + * @var \OC\Files\FileInfo[] $files + */ + $files = array_map(function (array $content) use ($path, $storage, $mount) { if (\OCP\Util::isSharingDisabledForUser()) { $content['permissions'] = $content['permissions'] & ~\OCP\Constants::PERMISSION_SHARE; } $owner = $this->getUserObjectForOwner($storage->getOwner($content['path'])); - $files[] = new FileInfo($path . '/' . $content['name'], $storage, $content['path'], $content, $mount, $owner); - } + return new FileInfo($path . '/' . $content['name'], $storage, $content['path'], $content, $mount, $owner); + }, $contents); //add a folder for any mountpoint in this directory and add the sizes of other mountpoints to the folders $mounts = Filesystem::getMountManager()->findIn($path); @@ -1333,7 +1324,8 @@ class View { if ($subStorage) { $subCache = $subStorage->getCache(''); - if ($subCache->getStatus('') === Cache\Cache::NOT_FOUND) { + $rootEntry = $subCache->get(''); + if (!$rootEntry) { $subScanner = $subStorage->getScanner(''); try { $subScanner->scanFile(''); @@ -1351,17 +1343,17 @@ class View { ); continue; } + $rootEntry = $subCache->get(''); } - $rootEntry = $subCache->get(''); if ($rootEntry) { $relativePath = trim(substr($mountPoint, $dirLength), '/'); if ($pos = strpos($relativePath, '/')) { //mountpoint inside subfolder add size to the correct folder $entryName = substr($relativePath, 0, $pos); foreach ($files as &$entry) { - if ($entry['name'] === $entryName) { - $entry['size'] += $rootEntry['size']; + if ($entry->getName() === $entryName) { + $entry->addSubEntry($rootEntry); } } } else { //mountpoint in this folder, add an entry for it @@ -1398,23 +1390,23 @@ class View { } if ($mimetype_filter) { - foreach ($files as $file) { + $files = array_filter($files, function (FileInfo $file) use ($mimetype_filter) { if (strpos($mimetype_filter, '/')) { - if ($file['mimetype'] === $mimetype_filter) { + if ($file->getMimetype() === $mimetype_filter) { $result[] = $file; } } else { - if ($file['mimepart'] === $mimetype_filter) { + if ($file->getMimePart() === $mimetype_filter) { $result[] = $file; } } - } - } else { - $result = $files; + }); } - } - return $result; + return $files; + } else { + return []; + } } /** From d5061b8139b4513c30aee782be94da7d69582710 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 10 Nov 2015 14:34:38 +0100 Subject: [PATCH 3/7] fix fileinfo for non existing files --- lib/private/files/view.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/private/files/view.php b/lib/private/files/view.php index 25bf7b547c..70e8db8338 100644 --- a/lib/private/files/view.php +++ b/lib/private/files/view.php @@ -1241,6 +1241,10 @@ class View { if ($storage) { $data = $this->getCacheEntry($storage, $internalPath, $relativePath); + if(!is_array($data)) { + return false; + } + if ($mount instanceof MoveableMount && $internalPath === '') { $data['permissions'] |= \OCP\Constants::PERMISSION_DELETE; } From 02f847bc66ea9238c10a5a04d3980c81c7b09293 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 10 Nov 2015 14:40:32 +0100 Subject: [PATCH 4/7] use relative path --- lib/private/files/view.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/files/view.php b/lib/private/files/view.php index 70e8db8338..0d5078fb3d 100644 --- a/lib/private/files/view.php +++ b/lib/private/files/view.php @@ -1204,7 +1204,7 @@ class View { } else if (!Cache\Scanner::isPartialFile($internalPath) && $watcher->needsUpdate($internalPath, $data)) { $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); $watcher->update($internalPath, $data); - $this->updater->propagate($path); + $this->updater->propagate($relativePath); $data = $cache->get($internalPath); $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); } From d006a7c723e4c51a4132b0f8817392bf1f3cd534 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 10 Nov 2015 14:48:19 +0100 Subject: [PATCH 5/7] Fix scanning of incomplete folders --- lib/private/files/view.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/files/view.php b/lib/private/files/view.php index 0d5078fb3d..5f19558a4f 100644 --- a/lib/private/files/view.php +++ b/lib/private/files/view.php @@ -1191,7 +1191,7 @@ class View { try { // if the file is not in the cache or needs to be updated, trigger the scanner and reload the data - if (!$data) { + if (!$data || $data['size'] === -1) { $this->lockFile($relativePath, ILockingProvider::LOCK_SHARED); if (!$storage->file_exists($internalPath)) { $this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED); From 888df3933df7f3588de11085035d2d3ae9292fb0 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 10 Nov 2015 16:14:08 +0100 Subject: [PATCH 6/7] take the etag of child mounts into account for the folder etag this replaces shared etag propagation --- apps/files_sharing/appinfo/app.php | 1 - apps/files_sharing/appinfo/application.php | 31 +--- apps/files_sharing/lib/mountprovider.php | 16 +- .../lib/propagation/changewatcher.php | 110 ----------- .../propagation/grouppropagationmanager.php | 133 -------------- .../lib/propagation/propagationmanager.php | 144 --------------- .../lib/propagation/recipientpropagator.php | 164 ----------------- apps/files_sharing/lib/sharedmount.php | 14 -- apps/files_sharing/lib/sharedpropagator.php | 43 +++++ apps/files_sharing/lib/sharedstorage.php | 16 +- .../tests/grouppropagationmanager.php | 173 ------------------ apps/files_sharing/tests/testcase.php | 1 - lib/private/files/cache/cache.php | 16 +- lib/private/files/cache/changepropagator.php | 24 ++- lib/private/files/cache/propagator.php | 66 +++++++ lib/private/files/fileinfo.php | 27 ++- lib/private/files/storage/common.php | 18 ++ lib/private/files/storage/storage.php | 8 + lib/private/files/storage/wrapper/wrapper.php | 7 + lib/private/files/view.php | 13 +- tests/lib/files/cache/changepropagator.php | 33 ++++ 21 files changed, 243 insertions(+), 815 deletions(-) delete mode 100644 apps/files_sharing/lib/propagation/changewatcher.php delete mode 100644 apps/files_sharing/lib/propagation/grouppropagationmanager.php delete mode 100644 apps/files_sharing/lib/propagation/propagationmanager.php delete mode 100644 apps/files_sharing/lib/propagation/recipientpropagator.php create mode 100644 apps/files_sharing/lib/sharedpropagator.php delete mode 100644 apps/files_sharing/tests/grouppropagationmanager.php create mode 100644 lib/private/files/cache/propagator.php diff --git a/apps/files_sharing/appinfo/app.php b/apps/files_sharing/appinfo/app.php index 03e448be0a..5f56340d25 100644 --- a/apps/files_sharing/appinfo/app.php +++ b/apps/files_sharing/appinfo/app.php @@ -42,7 +42,6 @@ $l = \OC::$server->getL10N('files_sharing'); $application = new Application(); $application->registerMountProviders(); -$application->setupPropagation(); \OCP\App::registerAdmin('files_sharing', 'settings-admin'); \OCP\App::registerPersonal('files_sharing', 'settings-personal'); diff --git a/apps/files_sharing/appinfo/application.php b/apps/files_sharing/appinfo/application.php index 545a942508..ffe3a6a513 100644 --- a/apps/files_sharing/appinfo/application.php +++ b/apps/files_sharing/appinfo/application.php @@ -27,8 +27,6 @@ namespace OCA\Files_Sharing\AppInfo; use OCA\Files_Sharing\Helper; use OCA\Files_Sharing\MountProvider; -use OCA\Files_Sharing\Propagation\PropagationManager; -use OCA\Files_Sharing\Propagation\GroupPropagationManager; use OCP\AppFramework\App; use OC\AppFramework\Utility\SimpleContainer; use OCA\Files_Sharing\Controllers\ExternalSharesController; @@ -116,8 +114,7 @@ class Application extends App { /** @var \OCP\IServerContainer $server */ $server = $c->query('ServerContainer'); return new MountProvider( - $server->getConfig(), - $c->query('PropagationManager') + $server->getConfig() ); }); @@ -132,25 +129,6 @@ class Application extends App { ); }); - $container->registerService('PropagationManager', function (IContainer $c) { - /** @var \OCP\IServerContainer $server */ - $server = $c->query('ServerContainer'); - return new PropagationManager( - $server->getUserSession(), - $server->getConfig() - ); - }); - - $container->registerService('GroupPropagationManager', function (IContainer $c) { - /** @var \OCP\IServerContainer $server */ - $server = $c->query('ServerContainer'); - return new GroupPropagationManager( - $server->getUserSession(), - $server->getGroupManager(), - $c->query('PropagationManager') - ); - }); - /* * Register capabilities */ @@ -164,11 +142,4 @@ class Application extends App { $mountProviderCollection->registerProvider($this->getContainer()->query('MountProvider')); $mountProviderCollection->registerProvider($this->getContainer()->query('ExternalMountProvider')); } - - public function setupPropagation() { - $propagationManager = $this->getContainer()->query('PropagationManager'); - \OCP\Util::connectHook('OC_Filesystem', 'setup', $propagationManager, 'globalSetup'); - - $this->getContainer()->query('GroupPropagationManager')->globalSetup(); - } } diff --git a/apps/files_sharing/lib/mountprovider.php b/apps/files_sharing/lib/mountprovider.php index 458e7f2619..74a2a3ff4d 100644 --- a/apps/files_sharing/lib/mountprovider.php +++ b/apps/files_sharing/lib/mountprovider.php @@ -24,7 +24,6 @@ namespace OCA\Files_Sharing; use OC\Files\Filesystem; use OC\User\NoUserException; -use OCA\Files_Sharing\Propagation\PropagationManager; use OCP\Files\Config\IMountProvider; use OCP\Files\Storage\IStorageFactory; use OCP\IConfig; @@ -36,18 +35,11 @@ class MountProvider implements IMountProvider { */ protected $config; - /** - * @var \OCA\Files_Sharing\Propagation\PropagationManager - */ - protected $propagationManager; - /** * @param \OCP\IConfig $config - * @param \OCA\Files_Sharing\Propagation\PropagationManager $propagationManager */ - public function __construct(IConfig $config, PropagationManager $propagationManager) { + public function __construct(IConfig $config) { $this->config = $config; - $this->propagationManager = $propagationManager; } @@ -60,21 +52,15 @@ class MountProvider implements IMountProvider { */ public function getMountsForUser(IUser $user, IStorageFactory $storageFactory) { $shares = \OCP\Share::getItemsSharedWithUser('file', $user->getUID()); - $propagator = $this->propagationManager->getSharePropagator($user->getUID()); - $propagator->propagateDirtyMountPoints($shares); $shares = array_filter($shares, function ($share) { return $share['permissions'] > 0; }); $shares = array_map(function ($share) use ($user, $storageFactory) { - // for updating etags for the share owner when we make changes to this share. - $ownerPropagator = $this->propagationManager->getChangePropagator($share['uid_owner']); return new SharedMount( '\OC\Files\Storage\Shared', '/' . $user->getUID() . '/' . $share['file_target'], array( - 'propagationManager' => $this->propagationManager, - 'propagator' => $ownerPropagator, 'share' => $share, 'user' => $user->getUID() ), diff --git a/apps/files_sharing/lib/propagation/changewatcher.php b/apps/files_sharing/lib/propagation/changewatcher.php deleted file mode 100644 index e61c161da1..0000000000 --- a/apps/files_sharing/lib/propagation/changewatcher.php +++ /dev/null @@ -1,110 +0,0 @@ - - * - * @copyright Copyright (c) 2015, ownCloud, Inc. - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OCA\Files_Sharing\Propagation; - -use OC\Files\Cache\ChangePropagator; -use OC\Files\Filesystem; -use OC\Files\View; -use OCA\Files_Sharing\SharedMount; - -/** - * Watch for changes made in a shared mount and propagate the changes to the share owner - */ -class ChangeWatcher { - /** - * The user view for the logged in user - * - * @var \OC\Files\View - */ - private $baseView; - - /** - * @var RecipientPropagator - */ - private $recipientPropagator; - - /** - * @param \OC\Files\View $baseView the view for the logged in user - * @param RecipientPropagator $recipientPropagator - */ - public function __construct(View $baseView, RecipientPropagator $recipientPropagator) { - $this->baseView = $baseView; - $this->recipientPropagator = $recipientPropagator; - } - - - public function writeHook($params) { - $path = $params['path']; - $fullPath = $this->baseView->getAbsolutePath($path); - $mount = $this->baseView->getMount($path); - if ($mount instanceof SharedMount) { - $this->propagateForOwner($mount->getShare(), $mount->getInternalPath($fullPath), $mount->getOwnerPropagator()); - } - $info = $this->baseView->getFileInfo($path); - if ($info) { - // trigger propagation if the subject of the write hook is shared. - // if a parent folder of $path is shared the propagation will be triggered from the change propagator hooks - $this->recipientPropagator->propagateById($info->getId()); - } - } - - public function renameHook($params) { - $path1 = $params['oldpath']; - $path2 = $params['newpath']; - $fullPath1 = $this->baseView->getAbsolutePath($path1); - $fullPath2 = $this->baseView->getAbsolutePath($path2); - $mount1 = $this->baseView->getMount($path1); - $mount2 = $this->baseView->getMount($path2); - if ($mount1 instanceof SharedMount and $mount1->getInternalPath($fullPath1) !== '') { - $this->propagateForOwner($mount1->getShare(), $mount1->getInternalPath($fullPath1), $mount1->getOwnerPropagator()); - } - if ($mount1 !== $mount2 and $mount2 instanceof SharedMount and $mount2->getInternalPath($fullPath2) !== '') { - $this->propagateForOwner($mount2->getShare(), $mount2->getInternalPath($fullPath2), $mount2->getOwnerPropagator()); - } - } - - /** - * @param array $share - * @param string $internalPath - * @param \OC\Files\Cache\ChangePropagator $propagator - */ - private function propagateForOwner($share, $internalPath, ChangePropagator $propagator) { - // note that we have already set up the filesystem for the owner when mounting the share - $view = new View('/' . $share['uid_owner'] . '/files'); - - $shareRootPath = $view->getPath($share['item_source']); - if (!is_null($shareRootPath)) { - $path = $shareRootPath . '/' . $internalPath; - $path = Filesystem::normalizePath($path); - $propagator->addChange($path); - $propagator->propagateChanges(); - } - } - - public function permissionsHook($params) { - $share = $params['share']; - - if ($share['item_type'] === 'file' || $share['item_type'] === 'folder') { - $this->recipientPropagator->markDirty($share, microtime(true)); - } - } -} diff --git a/apps/files_sharing/lib/propagation/grouppropagationmanager.php b/apps/files_sharing/lib/propagation/grouppropagationmanager.php deleted file mode 100644 index ba550dccec..0000000000 --- a/apps/files_sharing/lib/propagation/grouppropagationmanager.php +++ /dev/null @@ -1,133 +0,0 @@ - - * - * @copyright Copyright (c) 2015, ownCloud, Inc. - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OCA\Files_Sharing\Propagation; - -use OC\Files\Filesystem; -use OC\Files\View; -use OCP\IConfig; -use OCP\IUserSession; -use OCP\IGroup; -use OCP\IUser; -use OCP\IGroupManager; -use OCA\Files_Sharing\Propagation\PropagationManager; - -/** - * Propagate changes on group changes - */ -class GroupPropagationManager { - /** - * @var \OCP\IUserSession - */ - private $userSession; - - /** - * @var \OCP\IGroupManager - */ - private $groupManager; - - /** - * @var PropagationManager - */ - private $propagationManager; - - /** - * Items shared with a given user. - * Key is user id and value is an array of shares. - * - * @var array - */ - private $userShares = []; - - public function __construct(IUserSession $userSession, IGroupManager $groupManager, PropagationManager $propagationManager) { - $this->userSession = $userSession; - $this->groupManager = $groupManager; - $this->propagationManager = $propagationManager; - } - - public function onPreProcessUser(IGroup $group, IUser $targetUser) { - $this->userShares[$targetUser->getUID()] = $this->getUserShares($targetUser->getUID()); - } - - public function onPostAddUser(IGroup $group, IUser $targetUser) { - $targetUserId = $targetUser->getUID(); - $sharesAfter = $this->getUserShares($targetUserId); - - $this->propagateSharesDiff($targetUserId, $sharesAfter, $this->userShares[$targetUserId]); - unset($this->userShares[$targetUserId]); - } - - public function onPostRemoveUser(IGroup $group, IUser $targetUser) { - $targetUserId = $targetUser->getUID(); - $sharesAfter = $this->getUserShares($targetUserId); - - $this->propagateSharesDiff($targetUserId, $this->userShares[$targetUserId], $sharesAfter); - unset($this->userShares[$targetUserId]); - } - - private function getUserShares($targetUserId) { - return \OCP\Share::getItemsSharedWithUser('file', $targetUserId); - } - - /** - * Propagate etag for the shares that are in $shares1 but not in $shares2. - * - * @param string $targetUserId user id for which to propagate shares - * @param array $shares1 - * @param array $shares2 - */ - private function propagateSharesDiff($targetUserId, $shares1, $shares2) { - $sharesToPropagate = array_udiff( - $shares1, - $shares2, - function($share1, $share2) { - return ($share2['id'] - $share1['id']); - } - ); - - \OC\Files\Filesystem::initMountPoints($targetUserId); - $this->propagationManager->propagateSharesToUser($sharesToPropagate, $targetUserId); - } - - /** - * To be called from setupFS trough a hook - * - * Sets up listening to changes made to shares owned by the current user - */ - public function globalSetup() { - $user = $this->userSession->getUser(); - if (!$user) { - return; - } - - $this->groupManager->listen('\OC\Group', 'preAddUser', [$this, 'onPreProcessUser']); - $this->groupManager->listen('\OC\Group', 'postAddUser', [$this, 'onPostAddUser']); - $this->groupManager->listen('\OC\Group', 'preRemoveUser', [$this, 'onPreProcessUser']); - $this->groupManager->listen('\OC\Group', 'postRemoveUser', [$this, 'onPostRemoveUser']); - } - - public function tearDown() { - $this->groupManager->removeListener('\OC\Group', 'preAddUser', [$this, 'onPreProcessUser']); - $this->groupManager->removeListener('\OC\Group', 'postAddUser', [$this, 'onPostAddUser']); - $this->groupManager->removeListener('\OC\Group', 'preRemoveUser', [$this, 'onPreProcessUser']); - $this->groupManager->removeListener('\OC\Group', 'postRemoveUser', [$this, 'onPostRemoveUser']); - } -} diff --git a/apps/files_sharing/lib/propagation/propagationmanager.php b/apps/files_sharing/lib/propagation/propagationmanager.php deleted file mode 100644 index aac9428240..0000000000 --- a/apps/files_sharing/lib/propagation/propagationmanager.php +++ /dev/null @@ -1,144 +0,0 @@ - - * @author Vincent Petry - * - * @copyright Copyright (c) 2015, ownCloud, Inc. - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OCA\Files_Sharing\Propagation; - -use OC\Files\Filesystem; -use OC\Files\View; -use OCP\IConfig; -use OCP\IUserSession; -use OCP\Util; - - -/** - * Keep track of all change and share propagators by owner - */ -class PropagationManager { - /** - * @var \OCP\IUserSession - */ - private $userSession; - - /** - * @var \OCP\IConfig - */ - private $config; - - /** - * Change propagators for share owner - * - * @var \OC\Files\Cache\ChangePropagator[] - */ - private $changePropagators = []; - - /** - * Recipient propagators - * - * @var \OCA\Files_Sharing\Propagation\RecipientPropagator[] - */ - private $sharePropagators = []; - - public function __construct(IUserSession $userSession, IConfig $config) { - $this->userSession = $userSession; - $this->config = $config; - } - - /** - * @param string $user - * @return \OC\Files\Cache\ChangePropagator - */ - public function getChangePropagator($user) { - $activeUser = $this->userSession->getUser(); - - // for the local user we want to propagator from the active view, not any cached one - if ($activeUser && $activeUser->getUID() === $user && Filesystem::getView() instanceof View) { - // it's important that we take the existing propagator here to make sure we can listen to external changes - $this->changePropagators[$user] = Filesystem::getView()->getUpdater()->getPropagator(); - } - if (isset($this->changePropagators[$user])) { - return $this->changePropagators[$user]; - } - $view = new View('/' . $user . '/files'); - $this->changePropagators[$user] = $view->getUpdater()->getPropagator(); - return $this->changePropagators[$user]; - } - - /** - * Propagates etag changes for the given shares to the given user - * - * @param array array of shares for which to trigger etag change - * @param string $user - */ - public function propagateSharesToUser($shares, $user) { - $changePropagator = $this->getChangePropagator($user); - foreach ($shares as $share) { - $changePropagator->addChange($share['file_target']); - } - $time = microtime(true); - $changePropagator->propagateChanges(floor($time)); - } - - /** - * @param string $user - * @return \OCA\Files_Sharing\Propagation\RecipientPropagator - */ - public function getSharePropagator($user) { - if (isset($this->sharePropagators[$user])) { - return $this->sharePropagators[$user]; - } - $this->sharePropagators[$user] = new RecipientPropagator($user, $this->getChangePropagator($user), $this->config, $this); - return $this->sharePropagators[$user]; - } - - /** - * Attach the recipient propagator for $user to the change propagator of a share owner to mark shares as dirty when the owner makes a change to a share - * - * @param string $shareOwner - * @param string $user - */ - public function listenToOwnerChanges($shareOwner, $user) { - $sharePropagator = $this->getSharePropagator($user); - $ownerPropagator = $this->getChangePropagator($shareOwner); - $sharePropagator->attachToPropagator($ownerPropagator, $shareOwner); - } - - /** - * To be called from setupFS trough a hook - * - * Sets up listening to changes made to shares owned by the current user - */ - public function globalSetup() { - $user = $this->userSession->getUser(); - if (!$user) { - return; - } - $recipientPropagator = $this->getSharePropagator($user->getUID()); - $watcher = new ChangeWatcher(Filesystem::getView(), $recipientPropagator); - - // for marking shares owned by the active user as dirty when a file inside them changes - $this->listenToOwnerChanges($user->getUID(), $user->getUID()); - Util::connectHook('OC_Filesystem', 'post_write', $watcher, 'writeHook'); - Util::connectHook('OC_Filesystem', 'post_delete', $watcher, 'writeHook'); - Util::connectHook('OC_Filesystem', 'post_rename', $watcher, 'renameHook'); - Util::connectHook('OCP\Share', 'post_update_permissions', $watcher, 'permissionsHook'); - } -} diff --git a/apps/files_sharing/lib/propagation/recipientpropagator.php b/apps/files_sharing/lib/propagation/recipientpropagator.php deleted file mode 100644 index 5eacf4c0f6..0000000000 --- a/apps/files_sharing/lib/propagation/recipientpropagator.php +++ /dev/null @@ -1,164 +0,0 @@ - - * @author Morris Jobke - * @author Robin Appelman - * - * @copyright Copyright (c) 2015, ownCloud, Inc. - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OCA\Files_Sharing\Propagation; - -use OC\Files\Cache\ChangePropagator; -use OC\Files\View; -use OC\Share\Share; -use OCP\Files\NotFoundException; - -/** - * Propagate etags for share recipients - */ -class RecipientPropagator { - /** - * @var string - */ - protected $userId; - - /** - * @var \OC\Files\Cache\ChangePropagator - */ - protected $changePropagator; - - /** - * @var \OCP\IConfig - */ - protected $config; - - /** - * @var PropagationManager - */ - private $manager; - - /** - * @param string $userId current user, must match the propagator's - * user - * @param \OC\Files\Cache\ChangePropagator $changePropagator change propagator - * initialized with a view for $user - * @param \OCP\IConfig $config - * @param PropagationManager $manager - */ - public function __construct($userId, $changePropagator, $config, PropagationManager $manager) { - $this->userId = $userId; - $this->changePropagator = $changePropagator; - $this->config = $config; - $this->manager = $manager; - } - - /** - * Propagate the etag changes for all shares marked as dirty and mark the shares as clean - * - * @param array $shares the shares for the users - * @param int $time - */ - public function propagateDirtyMountPoints(array $shares, $time = null) { - if ($time === null) { - $time = microtime(true); - } - $dirtyShares = $this->getDirtyShares($shares); - foreach ($dirtyShares as $share) { - $this->changePropagator->addChange($share['file_target']); - } - if (count($dirtyShares)) { - $this->config->setUserValue($this->userId, 'files_sharing', 'last_propagate', $time); - $this->changePropagator->propagateChanges(floor($time)); - } - } - - /** - * Get all shares we need to update the etag for - * - * @param array $shares the shares for the users - * @return string[] - */ - protected function getDirtyShares($shares) { - $dirty = []; - $userTime = $this->config->getUserValue($this->userId, 'files_sharing', 'last_propagate', 0); - foreach ($shares as $share) { - $updateTime = $this->config->getAppValue('files_sharing', $share['id'], 0); - if ($updateTime >= $userTime) { - $dirty[] = $share; - } - } - return $dirty; - } - - /** - * @param array $share - * @param float $time - */ - public function markDirty($share, $time = null) { - if ($time === null) { - $time = microtime(true); - } - $this->config->setAppValue('files_sharing', $share['id'], $time); - } - - /** - * Listen on the propagator for updates made to shares owned by a user - * - * @param \OC\Files\Cache\ChangePropagator $propagator - * @param string $owner - */ - public function attachToPropagator(ChangePropagator $propagator, $owner) { - $propagator->listen('\OC\Files', 'propagate', function ($path, $entry) use ($owner) { - $this->propagateById($entry['fileid']); - }); - } - - protected $propagatingIds = []; - - /** - * @param int $id - */ - public function propagateById($id) { - if (isset($this->propagatingIds[$id])) { - return; - } - $this->propagatingIds[$id] = true; - $shares = Share::getAllSharesForFileId($id); - foreach ($shares as $share) { - // propagate down the share tree - $this->markDirty($share, microtime(true)); - - // propagate up the share tree - if ($share['share_with'] === $this->userId) { - $user = $share['uid_owner']; - $view = new View('/' . $user . '/files'); - - try { - $path = $view->getPath($share['file_source']); - } catch (NotFoundException $e) { - $path = null; - } - - $watcher = new ChangeWatcher($view, $this->manager->getSharePropagator($user)); - $watcher->writeHook(['path' => $path]); - } - } - - unset($this->propagatingIds[$id]); - } -} diff --git a/apps/files_sharing/lib/sharedmount.php b/apps/files_sharing/lib/sharedmount.php index a138795786..275fea97c7 100644 --- a/apps/files_sharing/lib/sharedmount.php +++ b/apps/files_sharing/lib/sharedmount.php @@ -38,11 +38,6 @@ class SharedMount extends MountPoint implements MoveableMount { */ protected $storage = null; - /** - * @var \OC\Files\Cache\ChangePropagator - */ - protected $ownerPropagator; - /** * @var \OC\Files\View */ @@ -54,8 +49,6 @@ class SharedMount extends MountPoint implements MoveableMount { private $user; public function __construct($storage, $mountpoint, $arguments = null, $loader = null) { - // first update the mount point before creating the parent - $this->ownerPropagator = $arguments['propagator']; $this->user = $arguments['user']; $this->recipientView = new View('/' . $this->user . '/files'); $newMountPoint = $this->verifyMountPoint($arguments['share']); @@ -201,11 +194,4 @@ class SharedMount extends MountPoint implements MoveableMount { public function getShare() { return $this->getStorage()->getShare(); } - - /** - * @return \OC\Files\Cache\ChangePropagator - */ - public function getOwnerPropagator() { - return $this->ownerPropagator; - } } diff --git a/apps/files_sharing/lib/sharedpropagator.php b/apps/files_sharing/lib/sharedpropagator.php new file mode 100644 index 0000000000..fcb4b92dd3 --- /dev/null +++ b/apps/files_sharing/lib/sharedpropagator.php @@ -0,0 +1,43 @@ + + * + * @copyright Copyright (c) 2015, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\Files_Sharing; + +use OC\Files\Cache\Propagator; + +class SharedPropagator extends Propagator { + /** + * @var \OC\Files\Storage\Shared + */ + protected $storage; + + /** + * @param string $internalPath + * @param int $time + * @return array[] all propagated entries + */ + public function propagateChange($internalPath, $time) { + $source = $this->storage->getSourcePath($internalPath); + /** @var \OC\Files\Storage\Storage $storage */ + list($storage, $sourceInternalPath) = \OC\Files\Filesystem::resolvePath($source); + return $storage->getPropagator()->propagateChange($sourceInternalPath, $time); + } +} diff --git a/apps/files_sharing/lib/sharedstorage.php b/apps/files_sharing/lib/sharedstorage.php index 18e0284417..4807b5ee73 100644 --- a/apps/files_sharing/lib/sharedstorage.php +++ b/apps/files_sharing/lib/sharedstorage.php @@ -50,11 +50,6 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { */ private $ownerView; - /** - * @var \OCA\Files_Sharing\Propagation\PropagationManager - */ - private $propagationManager; - /** * @var string */ @@ -65,7 +60,6 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { public function __construct($arguments) { $this->share = $arguments['share']; $this->ownerView = $arguments['ownerView']; - $this->propagationManager = $arguments['propagationManager']; $this->user = $arguments['user']; } @@ -75,9 +69,6 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { } $this->initialized = true; Filesystem::initMountPoints($this->share['uid_owner']); - - // for updating our etags when changes are made to the share from the owners side (probably indirectly by us trough another share) - $this->propagationManager->listenToOwnerChanges($this->share['uid_owner'], $this->user); } /** @@ -571,6 +562,13 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage { return new \OC\Files\Cache\Shared_Watcher($storage); } + public function getPropagator($storage = null) { + if (!$storage) { + $storage = $this; + } + return new \OCA\Files_Sharing\SharedPropagator($storage); + } + public function getOwner($path) { if ($path == '') { $path = $this->getMountPoint(); diff --git a/apps/files_sharing/tests/grouppropagationmanager.php b/apps/files_sharing/tests/grouppropagationmanager.php deleted file mode 100644 index ea32ca4f7e..0000000000 --- a/apps/files_sharing/tests/grouppropagationmanager.php +++ /dev/null @@ -1,173 +0,0 @@ - - * - * @copyright Copyright (c) 2015, ownCloud, Inc. - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OCA\Files_sharing\Tests; - -use OC\Files\View; -use OCP\IGroupManager; -use OCP\IGroup; -use OCP\IUser; -use OCP\Share; -use OCA\Files_Sharing\Propagation\GroupPropagationManager; -use OCA\Files_Sharing\Propagation\PropagationManager; - -class GroupPropagationManagerTest extends TestCase { - - /** - * @var GroupPropagationManager - */ - private $groupPropagationManager; - - /** - * @var IGroupManager - */ - private $groupManager; - - /** - * @var PropagationManager - */ - private $propagationManager; - - /** - * @var IGroup - */ - private $recipientGroup; - - /** - * @var IUser - */ - private $recipientUser; - - /** - * @var array - */ - private $fileInfo; - - protected function setUp() { - parent::setUp(); - - $user = $this->getMockBuilder('\OCP\IUser') - ->disableOriginalConstructor() - ->getMock(); - $user->method('getUID')->willReturn(self::TEST_FILES_SHARING_API_USER1); - $userSession = $this->getMockBuilder('\OCP\IUserSession') - ->disableOriginalConstructor() - ->getMock(); - $userSession->method('getUser')->willReturn(selF::TEST_FILES_SHARING_API_USER1); - - $this->propagationManager = $this->getMockBuilder('OCA\Files_Sharing\Propagation\PropagationManager') - ->disableOriginalConstructor() - ->getMock(); - - $this->groupManager = \OC::$server->getGroupManager(); - $this->groupPropagationManager = new GroupPropagationManager( - $userSession, - $this->groupManager, - $this->propagationManager - ); - $this->groupPropagationManager->globalSetup(); - - // since the sharing code is not mockable, we have to create a real folder - $this->loginAsUser(self::TEST_FILES_SHARING_API_USER1); - $view1 = new View('/' . self::TEST_FILES_SHARING_API_USER1 . '/files'); - $view1->mkdir('/folder'); - - $this->fileInfo = $view1->getFileInfo('/folder'); - - $this->recipientGroup = $this->groupManager->get(self::TEST_FILES_SHARING_API_GROUP1); - $this->recipientUser = \OC::$server->getUserManager()->get(self::TEST_FILES_SHARING_API_USER3); - - Share::shareItem( - 'folder', - $this->fileInfo['fileid'], - Share::SHARE_TYPE_GROUP, - $this->recipientGroup->getGID(), - \OCP\Constants::PERMISSION_READ - ); - - $this->loginAsUser($this->recipientUser->getUID()); - } - - protected function tearDown() { - $this->groupPropagationManager->tearDown(); - $this->recipientGroup->removeUser($this->recipientUser); - parent::tearDown(); - } - - public function testPropagateWhenAddedToGroup() { - $this->propagationManager->expects($this->once()) - ->method('propagateSharesToUser') - ->with($this->callback(function($shares) { - if (count($shares) !== 1) { - return false; - } - $share = array_values($shares)[0]; - return $share['file_source'] === $this->fileInfo['fileid'] && - $share['share_with'] === $this->recipientGroup->getGID() && - $share['file_target'] === '/folder'; - }), $this->recipientUser->getUID()); - - $this->recipientGroup->addUser($this->recipientUser); - } - - public function testPropagateWhenRemovedFromGroup() { - $this->recipientGroup->addUser($this->recipientUser); - - $this->propagationManager->expects($this->once()) - ->method('propagateSharesToUser') - ->with($this->callback(function($shares) { - if (count($shares) !== 1) { - return false; - } - $share = array_values($shares)[0]; - return $share['file_source'] === $this->fileInfo['fileid'] && - $share['share_with'] === $this->recipientGroup->getGID() && - $share['file_target'] === '/folder'; - }), $this->recipientUser->getUID()); - - $this->recipientGroup->removeUser($this->recipientUser); - } - - public function testPropagateWhenRemovedFromGroupWithSubdirTarget() { - $this->recipientGroup->addUser($this->recipientUser); - - // relogin to refresh mount points - $this->loginAsUser($this->recipientUser->getUID()); - $recipientView = new View('/' . $this->recipientUser->getUID() . '/files'); - - $this->assertTrue($recipientView->mkdir('sub')); - $this->assertTrue($recipientView->rename('folder', 'sub/folder')); - - $this->propagationManager->expects($this->once()) - ->method('propagateSharesToUser') - ->with($this->callback(function($shares) { - if (count($shares) !== 1) { - return false; - } - $share = array_values($shares)[0]; - return $share['file_source'] === $this->fileInfo['fileid'] && - $share['share_with'] === $this->recipientGroup->getGID() && - $share['file_target'] === '/sub/folder'; - }), $this->recipientUser->getUID()); - - $this->recipientGroup->removeUser($this->recipientUser); - } -} diff --git a/apps/files_sharing/tests/testcase.php b/apps/files_sharing/tests/testcase.php index c91734a5b0..6a72a34149 100644 --- a/apps/files_sharing/tests/testcase.php +++ b/apps/files_sharing/tests/testcase.php @@ -61,7 +61,6 @@ abstract class TestCase extends \Test\TestCase { $application = new Application(); $application->registerMountProviders(); - $application->setupPropagation(); // reset backend \OC_User::clearBackends(); diff --git a/lib/private/files/cache/cache.php b/lib/private/files/cache/cache.php index 4047724332..e6110c1925 100644 --- a/lib/private/files/cache/cache.php +++ b/lib/private/files/cache/cache.php @@ -160,6 +160,7 @@ class Cache { } else { //fix types $data['fileid'] = (int)$data['fileid']; + $data['parent'] = (int)$data['parent']; $data['size'] = 0 + $data['size']; $data['mtime'] = (int)$data['mtime']; $data['storage_mtime'] = (int)$data['storage_mtime']; @@ -391,14 +392,19 @@ class Cache { if ($file === '') { return -1; } else { - $parent = dirname($file); - if ($parent === '.') { - $parent = ''; - } - return $this->getId($parent); + $parent = $this->getParentPath($file); + return (int) $this->getId($parent); } } + private function getParentPath($path) { + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + return $parent; + } + /** * check if a file is available in the cache * diff --git a/lib/private/files/cache/changepropagator.php b/lib/private/files/cache/changepropagator.php index 9696a82257..2a48eb13c5 100644 --- a/lib/private/files/cache/changepropagator.php +++ b/lib/private/files/cache/changepropagator.php @@ -22,6 +22,7 @@ namespace OC\Files\Cache; +use OC\Files\Filesystem; use OC\Hooks\BasicEmitter; /** @@ -61,23 +62,30 @@ class ChangePropagator extends BasicEmitter { * @param int $time (optional) the mtime to set for the folders, if not set the current time is used */ public function propagateChanges($time = null) { - $parents = $this->getAllParents(); - $this->changedFiles = array(); + $changes = $this->getChanges(); + $this->changedFiles = []; if (!$time) { $time = time(); } - foreach ($parents as $parent) { + foreach ($changes as $change) { /** * @var \OC\Files\Storage\Storage $storage * @var string $internalPath */ - list($storage, $internalPath) = $this->view->resolvePath($parent); + $absolutePath = $this->view->getAbsolutePath($change); + $mount = $this->view->getMount($change); + $storage = $mount->getStorage(); + $internalPath = $mount->getInternalPath($absolutePath); if ($storage) { - $cache = $storage->getCache(); - $entry = $cache->get($internalPath); - $cache->update($entry['fileid'], array('mtime' => max($time, $entry['mtime']), 'etag' => $storage->getETag($internalPath))); - $this->emit('\OC\Files', 'propagate', [$parent, $entry]); + $propagator = $storage->getPropagator(); + $propagatedEntries = $propagator->propagateChange($internalPath, $time); + + foreach ($propagatedEntries as $entry) { + $absolutePath = Filesystem::normalizePath($mount->getMountPoint() . '/' . $entry['path']); + $relativePath = $this->view->getRelativePath($absolutePath); + $this->emit('\OC\Files', 'propagate', [$relativePath, $entry]); + } } } } diff --git a/lib/private/files/cache/propagator.php b/lib/private/files/cache/propagator.php new file mode 100644 index 0000000000..bd11cef599 --- /dev/null +++ b/lib/private/files/cache/propagator.php @@ -0,0 +1,66 @@ + + * + * @copyright Copyright (c) 2015, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\Files\Cache; + +/** + * Propagate etags and mtimes within the storage + */ +class Propagator { + /** + * @var \OC\Files\Storage\Storage + */ + protected $storage; + + /** + * @param \OC\Files\Storage\Storage $storage + */ + public function __construct(\OC\Files\Storage\Storage $storage) { + $this->storage = $storage; + } + + + /** + * @param string $internalPath + * @param int $time + * @return array[] all propagated entries + */ + public function propagateChange($internalPath, $time) { + $cache = $this->storage->getCache($internalPath); + + $parentId = $cache->getParentId($internalPath); + $propagatedEntries = []; + while ($parentId !== -1) { + $entry = $cache->get($parentId); + $propagatedEntries[] = $entry; + if (!$entry) { + return $propagatedEntries; + } + $mtime = max($time, $entry['mtime']); + + $cache->update($parentId, ['mtime' => $mtime, 'etag' => $this->storage->getETag($entry['path'])]); + + $parentId = $entry['parent']; + } + + return $propagatedEntries; + } +} diff --git a/lib/private/files/fileinfo.php b/lib/private/files/fileinfo.php index 2d2ef325d3..5b5e869700 100644 --- a/lib/private/files/fileinfo.php +++ b/lib/private/files/fileinfo.php @@ -61,6 +61,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { */ private $owner; + /** + * @var string[] + */ + private $childEtags = []; + /** * @param string|boolean $path * @param Storage\Storage $storage @@ -93,6 +98,8 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { public function offsetGet($offset) { if ($offset === 'type') { return $this->getType(); + } else if ($offset === 'etag') { + return $this->getEtag(); } elseif (isset($this->data[$offset])) { return $this->data[$offset]; } else { @@ -153,7 +160,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { * @return string */ public function getEtag() { - return $this->data['etag']; + if (count($this->childEtags) > 0) { + $combinedEtag = $this->data['etag'] . '::' . implode('::', $this->childEtags); + return md5($combinedEtag); + } else { + return $this->data['etag']; + } } /** @@ -292,8 +304,19 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess { * Sets the size, etag and size to for cross-storage childs * * @param array $data cache entry for the child + * @param string $entryPath full path of the child entry */ - public function addSubEntry($data) { + public function addSubEntry($data, $entryPath) { $this->data['size'] += isset($data['size']) ? $data['size'] : 0; + if (isset($data['mtime'])) { + $this->data['mtime'] = max($this->data['mtime'], $data['mtime']); + } + if (isset($data['etag'])) { + // prefix the etag with the relative path of the subentry to propagate etag on mount moves + $relativeEntryPath = substr($entryPath, strlen($this->getPath())); + // attach the permissions to propagate etag on permision changes of submounts + $permissions = isset($data['permissions']) ? $data['permissions'] : 0; + $this->childEtags[] = $relativeEntryPath . '/' . $data['etag'] . $permissions; + } } } diff --git a/lib/private/files/storage/common.php b/lib/private/files/storage/common.php index 77a70226b3..a52e7c35f8 100644 --- a/lib/private/files/storage/common.php +++ b/lib/private/files/storage/common.php @@ -37,6 +37,7 @@ namespace OC\Files\Storage; use OC\Files\Cache\Cache; +use OC\Files\Cache\Propagator; use OC\Files\Cache\Scanner; use OC\Files\Filesystem; use OC\Files\Cache\Watcher; @@ -64,6 +65,7 @@ abstract class Common implements Storage { protected $cache; protected $scanner; protected $watcher; + protected $propagator; protected $storageCache; protected $mountOptions = []; @@ -345,6 +347,22 @@ abstract class Common implements Storage { return $this->watcher; } + /** + * get a propagator instance for the cache + * + * @param \OC\Files\Storage\Storage (optional) the storage to pass to the watcher + * @return \OC\Files\Cache\Propagator + */ + public function getPropagator($storage = null) { + if (!$storage) { + $storage = $this; + } + if (!isset($this->propagator)) { + $this->propagator = new Propagator($storage); + } + return $this->propagator; + } + public function getStorageCache($storage = null) { if (!$storage) { $storage = $this; diff --git a/lib/private/files/storage/storage.php b/lib/private/files/storage/storage.php index f46ac544b5..3399c71789 100644 --- a/lib/private/files/storage/storage.php +++ b/lib/private/files/storage/storage.php @@ -67,6 +67,14 @@ interface Storage extends \OCP\Files\Storage { */ public function getWatcher($path = '', $storage = null); + /** + * get a propagator instance for the cache + * + * @param \OC\Files\Storage\Storage (optional) the storage to pass to the watcher + * @return \OC\Files\Cache\Propagator + */ + public function getPropagator($storage = null); + /** * @return \OC\Files\Cache\Storage */ diff --git a/lib/private/files/storage/wrapper/wrapper.php b/lib/private/files/storage/wrapper/wrapper.php index 048738170d..5327b94211 100644 --- a/lib/private/files/storage/wrapper/wrapper.php +++ b/lib/private/files/storage/wrapper/wrapper.php @@ -430,6 +430,13 @@ class Wrapper implements \OC\Files\Storage\Storage { return $this->storage->getWatcher($path, $storage); } + public function getPropagator($storage = null) { + if (!$storage) { + $storage = $this; + } + return $this->storage->getPropagator($storage); + } + /** * @return \OC\Files\Cache\Storage */ diff --git a/lib/private/files/view.php b/lib/private/files/view.php index 5f19558a4f..3ad9d36a38 100644 --- a/lib/private/files/view.php +++ b/lib/private/files/view.php @@ -707,10 +707,6 @@ class View { } else if ($result) { if ($internalPath1 !== '') { // dont do a cache update for moved mounts $this->updater->rename($path1, $path2); - } else { // only do etag propagation - $this->getUpdater()->getPropagator()->addChange($path1); - $this->getUpdater()->getPropagator()->addChange($path2); - $this->getUpdater()->getPropagator()->propagateChanges(); } } @@ -1179,6 +1175,11 @@ class View { } /** + * Get file info from cache + * + * If the file is not in cached it will be scanned + * If the file has changed on storage the cache will be updated + * * @param \OC\Files\Storage\Storage $storage * @param string $internalPath * @param string $relativePath @@ -1266,7 +1267,7 @@ class View { } $subCache = $subStorage->getCache(''); $rootEntry = $subCache->get(''); - $info->addSubEntry($rootEntry); + $info->addSubEntry($rootEntry, $mount->getMountPoint()); } } } @@ -1357,7 +1358,7 @@ class View { $entryName = substr($relativePath, 0, $pos); foreach ($files as &$entry) { if ($entry->getName() === $entryName) { - $entry->addSubEntry($rootEntry); + $entry->addSubEntry($rootEntry, $mountPoint); } } } else { //mountpoint in this folder, add an entry for it diff --git a/tests/lib/files/cache/changepropagator.php b/tests/lib/files/cache/changepropagator.php index 1b56da5e97..9108330eb9 100644 --- a/tests/lib/files/cache/changepropagator.php +++ b/tests/lib/files/cache/changepropagator.php @@ -94,4 +94,37 @@ class ChangePropagator extends \Test\TestCase { $this->assertEquals(100, $time - $cache->get('foo')['mtime']); $this->assertEquals(100, $time - $cache->get('foo/bar')['mtime']); } + + public function testPropagateCrossStorage() { + $storage = new Temporary(); + $this->view->mkdir('/foo'); + Filesystem::mount($storage, [], $this->view->getAbsolutePath('/foo/submount')); + $this->view->mkdir('/foo/submount/bar'); + $this->view->file_put_contents('/foo/submount/bar/sad.txt', 'qwerty'); + + $oldInfo1 = $this->view->getFileInfo('/'); + $oldInfo2 = $this->view->getFileInfo('/foo'); + $oldInfo3 = $this->view->getFileInfo('/foo/submount'); + $oldInfo4 = $this->view->getFileInfo('/foo/submount/bar'); + + $time = time() + 50; + + $this->propagator->addChange('/foo/submount/bar/sad.txt'); + $this->propagator->propagateChanges($time); + + $newInfo1 = $this->view->getFileInfo('/'); + $newInfo2 = $this->view->getFileInfo('/foo'); + $newInfo3 = $this->view->getFileInfo('/foo/submount'); + $newInfo4 = $this->view->getFileInfo('/foo/submount/bar'); + + $this->assertEquals($newInfo1->getMTime(), $time); + $this->assertEquals($newInfo2->getMTime(), $time); + $this->assertEquals($newInfo3->getMTime(), $time); + $this->assertEquals($newInfo4->getMTime(), $time); + + $this->assertNotSame($oldInfo1->getEtag(), $newInfo1->getEtag()); + $this->assertNotSame($oldInfo2->getEtag(), $newInfo2->getEtag()); + $this->assertNotSame($oldInfo3->getEtag(), $newInfo3->getEtag()); + $this->assertNotSame($oldInfo4->getEtag(), $newInfo3->getEtag()); + } } From a95d4c2b22e2cf0deb4958bc3aa3b20ed9d0463c Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 10 Nov 2015 16:31:30 +0100 Subject: [PATCH 7/7] update tests --- apps/files_sharing/tests/etagpropagation.php | 3 +- apps/files_trashbin/tests/trashbin.php | 1 - apps/files_versions/tests/versions.php | 1 - tests/lib/files/cache/updaterlegacy.php | 47 +++++--------------- tests/lib/files/etagtest.php | 1 - 5 files changed, 12 insertions(+), 41 deletions(-) diff --git a/apps/files_sharing/tests/etagpropagation.php b/apps/files_sharing/tests/etagpropagation.php index 1abf04df84..2a33732d63 100644 --- a/apps/files_sharing/tests/etagpropagation.php +++ b/apps/files_sharing/tests/etagpropagation.php @@ -193,7 +193,8 @@ class EtagPropagation extends TestCase { public function testOwnerWritesToSingleFileShare() { $this->loginAsUser(self::TEST_FILES_SHARING_API_USER1); - Filesystem::file_put_contents('/foo.txt', 'bar'); + Filesystem::file_put_contents('/foo.txt', 'longer_bar'); + Filesystem::touch('/foo.txt', time() - 1); $this->assertEtagsNotChanged([self::TEST_FILES_SHARING_API_USER4, self::TEST_FILES_SHARING_API_USER3]); $this->assertEtagsChanged([self::TEST_FILES_SHARING_API_USER1, self::TEST_FILES_SHARING_API_USER2]); diff --git a/apps/files_trashbin/tests/trashbin.php b/apps/files_trashbin/tests/trashbin.php index e28b854ca1..934f831ef7 100644 --- a/apps/files_trashbin/tests/trashbin.php +++ b/apps/files_trashbin/tests/trashbin.php @@ -65,7 +65,6 @@ class Test_Trashbin extends \Test\TestCase { \OC::registerShareHooks(); $application = new \OCA\Files_Sharing\AppInfo\Application(); $application->registerMountProviders(); - $application->setupPropagation(); //disable encryption \OC_App::disable('encryption'); diff --git a/apps/files_versions/tests/versions.php b/apps/files_versions/tests/versions.php index b9bc0932a8..6ebbf830e7 100644 --- a/apps/files_versions/tests/versions.php +++ b/apps/files_versions/tests/versions.php @@ -51,7 +51,6 @@ class Test_Files_Versioning extends \Test\TestCase { $application = new \OCA\Files_Sharing\AppInfo\Application(); $application->registerMountProviders(); - $application->setupPropagation(); // create test user self::loginHelper(self::TEST_VERSIONS_USER2, true); diff --git a/tests/lib/files/cache/updaterlegacy.php b/tests/lib/files/cache/updaterlegacy.php index f4d52e9a39..c1a0d3d823 100644 --- a/tests/lib/files/cache/updaterlegacy.php +++ b/tests/lib/files/cache/updaterlegacy.php @@ -10,6 +10,7 @@ namespace Test\Files\Cache; use \OC\Files\Filesystem as Filesystem; use OC\Files\Storage\Temporary; +use OC\Files\View; class UpdaterLegacy extends \Test\TestCase { /** @@ -111,7 +112,8 @@ class UpdaterLegacy extends \Test\TestCase { $storage2->getScanner()->scan(''); //initialize etags $cache2 = $storage2->getCache(); Filesystem::mount($storage2, array(), '/' . self::$user . '/files/folder/substorage'); - $folderCachedData = $this->cache->get('folder'); + $view = new View('/' . self::$user . '/files'); + $folderCachedData = $view->getFileInfo('folder'); $substorageCachedData = $cache2->get(''); Filesystem::file_put_contents('folder/substorage/foo.txt', 'asd'); $this->assertTrue($cache2->inCache('foo.txt')); @@ -124,7 +126,7 @@ class UpdaterLegacy extends \Test\TestCase { $this->assertInternalType('string', $cachedData['etag']); $this->assertNotSame($substorageCachedData['etag'], $cachedData['etag']); - $cachedData = $this->cache->get('folder'); + $cachedData = $view->getFileInfo('folder'); $this->assertInternalType('string', $folderCachedData['etag']); $this->assertInternalType('string', $cachedData['etag']); $this->assertNotSame($folderCachedData['etag'], $cachedData['etag']); @@ -168,8 +170,9 @@ class UpdaterLegacy extends \Test\TestCase { $cache2 = $storage2->getCache(); Filesystem::mount($storage2, array(), '/' . self::$user . '/files/folder/substorage'); Filesystem::file_put_contents('folder/substorage/foo.txt', 'asd'); + $view = new View('/' . self::$user . '/files'); $this->assertTrue($cache2->inCache('foo.txt')); - $folderCachedData = $this->cache->get('folder'); + $folderCachedData = $view->getFileInfo('folder'); $substorageCachedData = $cache2->get(''); Filesystem::unlink('folder/substorage/foo.txt'); $this->assertFalse($cache2->inCache('foo.txt')); @@ -180,7 +183,7 @@ class UpdaterLegacy extends \Test\TestCase { $this->assertNotSame($substorageCachedData['etag'], $cachedData['etag']); $this->assertGreaterThanOrEqual($substorageCachedData['mtime'], $cachedData['mtime']); - $cachedData = $this->cache->get('folder'); + $cachedData = $view->getFileInfo('folder'); $this->assertInternalType('string', $folderCachedData['etag']); $this->assertInternalType('string', $cachedData['etag']); $this->assertNotSame($folderCachedData['etag'], $cachedData['etag']); @@ -222,8 +225,9 @@ class UpdaterLegacy extends \Test\TestCase { $cache2 = $storage2->getCache(); Filesystem::mount($storage2, array(), '/' . self::$user . '/files/folder/substorage'); Filesystem::file_put_contents('folder/substorage/foo.txt', 'asd'); + $view = new View('/' . self::$user . '/files'); $this->assertTrue($cache2->inCache('foo.txt')); - $folderCachedData = $this->cache->get('folder'); + $folderCachedData = $view->getFileInfo('folder'); $substorageCachedData = $cache2->get(''); $fooCachedData = $cache2->get('foo.txt'); Filesystem::rename('folder/substorage/foo.txt', 'folder/substorage/bar.txt'); @@ -240,7 +244,7 @@ class UpdaterLegacy extends \Test\TestCase { // rename can cause mtime change - invalid assert // $this->assertEquals($mtime, $cachedData['mtime']); - $cachedData = $this->cache->get('folder'); + $cachedData = $view->getFileInfo('folder'); $this->assertInternalType('string', $folderCachedData['etag']); $this->assertInternalType('string', $cachedData['etag']); $this->assertNotSame($folderCachedData['etag'], $cachedData['etag']); @@ -287,35 +291,4 @@ class UpdaterLegacy extends \Test\TestCase { $this->assertEquals($time, $cachedData['mtime']); } - public function testTouchWithMountPoints() { - $storage2 = new \OC\Files\Storage\Temporary(array()); - $cache2 = $storage2->getCache(); - Filesystem::mount($storage2, array(), '/' . self::$user . '/files/folder/substorage'); - Filesystem::file_put_contents('folder/substorage/foo.txt', 'asd'); - $this->assertTrue($cache2->inCache('foo.txt')); - $folderCachedData = $this->cache->get('folder'); - $substorageCachedData = $cache2->get(''); - $fooCachedData = $cache2->get('foo.txt'); - $cachedData = $cache2->get('foo.txt'); - $time = 1371006070; - $this->cache->put('folder', ['mtime' => $time - 100]); - Filesystem::touch('folder/substorage/foo.txt', $time); - $cachedData = $cache2->get('foo.txt'); - $this->assertInternalType('string', $fooCachedData['etag']); - $this->assertInternalType('string', $cachedData['etag']); - $this->assertNotSame($fooCachedData['etag'], $cachedData['etag']); - $this->assertEquals($time, $cachedData['mtime']); - - $cachedData = $cache2->get(''); - $this->assertInternalType('string', $substorageCachedData['etag']); - $this->assertInternalType('string', $cachedData['etag']); - $this->assertNotSame($substorageCachedData['etag'], $cachedData['etag']); - - $cachedData = $this->cache->get('folder'); - $this->assertInternalType('string', $folderCachedData['etag']); - $this->assertInternalType('string', $cachedData['etag']); - $this->assertNotSame($folderCachedData['etag'], $cachedData['etag']); - $this->assertEquals($time, $cachedData['mtime']); - } - } diff --git a/tests/lib/files/etagtest.php b/tests/lib/files/etagtest.php index 1a11b29cf0..192768d04a 100644 --- a/tests/lib/files/etagtest.php +++ b/tests/lib/files/etagtest.php @@ -27,7 +27,6 @@ class EtagTest extends \Test\TestCase { \OC_Hook::clear('OC_Filesystem', 'setup'); $application = new \OCA\Files_Sharing\AppInfo\Application(); $application->registerMountProviders(); - $application->setupPropagation(); \OCP\Share::registerBackend('file', 'OC_Share_Backend_File'); \OCP\Share::registerBackend('folder', 'OC_Share_Backend_Folder', 'file');