From 645e7035a45e400d02c56e588f6946d20b522731 Mon Sep 17 00:00:00 2001 From: ringmaster Date: Thu, 15 May 2014 22:38:46 -0400 Subject: [PATCH 1/7] Allow apps to control via a hook to skip adding/removing a file during filescan --- lib/private/files/cache/scanner.php | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index b3ab94f459..19321da8ec 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -161,7 +161,11 @@ class Scanner extends BasicEmitter { \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); } } else { - $this->cache->remove($file); + $removeFromCache = true; + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file, 'removeFromCache' => &$removeFromCache)); + if($removeFromCache) { + $this->cache->remove($file); + } } return $data; } @@ -244,7 +248,11 @@ class Scanner extends BasicEmitter { $removedChildren = \array_diff($existingChildren, $newChildren); foreach ($removedChildren as $childName) { $child = ($path) ? $path . '/' . $childName : $childName; - $this->cache->remove($child); + $addToCache = true; + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child, 'removeFromCache' => &$addToCache)); + if($addToCache) { + $this->cache->remove($child); + } } \OC_DB::commit(); if ($exceptionOccurred){ @@ -263,7 +271,11 @@ class Scanner extends BasicEmitter { $size += $childSize; } } - $this->cache->put($path, array('size' => $size)); + $addToCache = true; + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'addToCache' => &$addToCache, 'size' => &$size)); + if($addToCache) { + $this->cache->put($path, array('size' => $size)); + } } $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId)); return $size; From fdf26c5a3f4718fcadc214eb69a91f780ce4045c Mon Sep 17 00:00:00 2001 From: ringmaster Date: Fri, 16 May 2014 12:12:27 -0400 Subject: [PATCH 2/7] Rename variable to something more appropriate. --- lib/private/files/cache/scanner.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index 19321da8ec..2790d8581f 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -248,9 +248,9 @@ class Scanner extends BasicEmitter { $removedChildren = \array_diff($existingChildren, $newChildren); foreach ($removedChildren as $childName) { $child = ($path) ? $path . '/' . $childName : $childName; - $addToCache = true; - \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child, 'removeFromCache' => &$addToCache)); - if($addToCache) { + $removeFromCache = true; + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child, 'removeFromCache' => &$removeFromCache)); + if($removeFromCache) { $this->cache->remove($child); } } From 3b287f8274ef263d3144410bb6e3bf1a966f9d6d Mon Sep 17 00:00:00 2001 From: ringmaster Date: Tue, 20 May 2014 12:48:48 -0400 Subject: [PATCH 3/7] Allow the default behavior to come from the config. --- lib/private/files/cache/scanner.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index 2790d8581f..24d406f7cd 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -10,6 +10,7 @@ namespace OC\Files\Cache; use OC\Files\Filesystem; use OC\Hooks\BasicEmitter; +use OCP\Config; /** * Class Scanner @@ -156,12 +157,16 @@ class Scanner extends BasicEmitter { } } if (!empty($newData)) { - $data['fileid'] = $this->cache->put($file, $newData); + $addToCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'addToCache' => &$addToCache, 'data' => &$newData)); + if($addToCache) { + $data['fileid'] = $this->cache->put($file, $newData); + } $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); } } else { - $removeFromCache = true; + $removeFromCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file, 'removeFromCache' => &$removeFromCache)); if($removeFromCache) { $this->cache->remove($file); @@ -248,7 +253,7 @@ class Scanner extends BasicEmitter { $removedChildren = \array_diff($existingChildren, $newChildren); foreach ($removedChildren as $childName) { $child = ($path) ? $path . '/' . $childName : $childName; - $removeFromCache = true; + $removeFromCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child, 'removeFromCache' => &$removeFromCache)); if($removeFromCache) { $this->cache->remove($child); @@ -271,10 +276,11 @@ class Scanner extends BasicEmitter { $size += $childSize; } } - $addToCache = true; - \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'addToCache' => &$addToCache, 'size' => &$size)); + $newData = array('size' => $size); + $addToCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'addToCache' => &$addToCache, 'data' => &$newData)); if($addToCache) { - $this->cache->put($path, array('size' => $size)); + $this->cache->put($path, $newData); } } $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId)); From 26d169b27c43d06db1158990e33939b85d14a60d Mon Sep 17 00:00:00 2001 From: ringmaster Date: Tue, 27 May 2014 16:01:16 -0400 Subject: [PATCH 4/7] Use 'filesystem_check_enable' as a config option. --- config/config.sample.php | 5 ++++- lib/private/files/cache/scanner.php | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index 80694837ed..8fb782e7cf 100755 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -291,6 +291,9 @@ $CONFIG = array( * 1 -> check each file or folder at most once per request, recomended for general use if outside changes might happen * 2 -> check every time the filesystem is used, causes a performance hit when using external storages, not recomended for regular use */ -'filesystem_check_changes' => 1 +'filesystem_check_changes' => 1, + +/* specifies whether changes in the filesystem outside of owncloud affect cached data about those files */ +'filesystem_check_enable' => 1, ); diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index 24d406f7cd..e838a8b625 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -157,7 +157,7 @@ class Scanner extends BasicEmitter { } } if (!empty($newData)) { - $addToCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); + $addToCache = Config::getSystemValue('filesystem_check_enable', true); \OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'addToCache' => &$addToCache, 'data' => &$newData)); if($addToCache) { $data['fileid'] = $this->cache->put($file, $newData); @@ -166,7 +166,7 @@ class Scanner extends BasicEmitter { \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); } } else { - $removeFromCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); + $removeFromCache = Config::getSystemValue('filesystem_check_enable', true); \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file, 'removeFromCache' => &$removeFromCache)); if($removeFromCache) { $this->cache->remove($file); @@ -253,7 +253,7 @@ class Scanner extends BasicEmitter { $removedChildren = \array_diff($existingChildren, $newChildren); foreach ($removedChildren as $childName) { $child = ($path) ? $path . '/' . $childName : $childName; - $removeFromCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); + $removeFromCache = Config::getSystemValue('filesystem_check_enable', true); \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child, 'removeFromCache' => &$removeFromCache)); if($removeFromCache) { $this->cache->remove($child); @@ -277,7 +277,7 @@ class Scanner extends BasicEmitter { } } $newData = array('size' => $size); - $addToCache = Config::getSystemValue('allow_scanner_to_affect_cache', true); + $addToCache = Config::getSystemValue('filesystem_check_enable', true); \OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'addToCache' => &$addToCache, 'data' => &$newData)); if($addToCache) { $this->cache->put($path, $newData); From f79948f519f5532f95fd744b8ba28329cc45b022 Mon Sep 17 00:00:00 2001 From: ringmaster Date: Wed, 28 May 2014 13:20:20 -0400 Subject: [PATCH 5/7] Prevent apps from affecting the scanner via hook. --- lib/private/files/cache/scanner.php | 34 +++++++++++++++-------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index e838a8b625..d4942d7dc2 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -138,9 +138,12 @@ class Scanner extends BasicEmitter { $parent = ''; } $parentCacheData = $this->cache->get($parent); - $this->cache->update($parentCacheData['fileid'], array( - 'etag' => $this->storage->getETag($parent), - )); + \OC_Hook::emit('Scanner', 'updateCache', array('file' => $file, 'data' => $data)); + if(Config::getSystemValue('filesystem_check_enable', true)) { + $this->cache->update($parentCacheData['fileid'], array( + 'etag' => $this->storage->getETag($parent), + )); + } } } } @@ -157,18 +160,16 @@ class Scanner extends BasicEmitter { } } if (!empty($newData)) { - $addToCache = Config::getSystemValue('filesystem_check_enable', true); - \OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'addToCache' => &$addToCache, 'data' => &$newData)); - if($addToCache) { + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'data' => $newData)); + if(Config::getSystemValue('filesystem_check_enable', true)) { $data['fileid'] = $this->cache->put($file, $newData); } $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); } } else { - $removeFromCache = Config::getSystemValue('filesystem_check_enable', true); - \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file, 'removeFromCache' => &$removeFromCache)); - if($removeFromCache) { + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file)); + if(Config::getSystemValue('filesystem_check_enable', true)) { $this->cache->remove($file); } } @@ -253,9 +254,8 @@ class Scanner extends BasicEmitter { $removedChildren = \array_diff($existingChildren, $newChildren); foreach ($removedChildren as $childName) { $child = ($path) ? $path . '/' . $childName : $childName; - $removeFromCache = Config::getSystemValue('filesystem_check_enable', true); - \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child, 'removeFromCache' => &$removeFromCache)); - if($removeFromCache) { + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child)); + if(Config::getSystemValue('filesystem_check_enable', true)) { $this->cache->remove($child); } } @@ -277,9 +277,8 @@ class Scanner extends BasicEmitter { } } $newData = array('size' => $size); - $addToCache = Config::getSystemValue('filesystem_check_enable', true); - \OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'addToCache' => &$addToCache, 'data' => &$newData)); - if($addToCache) { + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'data' => $newData)); + if(Config::getSystemValue('filesystem_check_enable', true)) { $this->cache->put($path, $newData); } } @@ -308,7 +307,10 @@ class Scanner extends BasicEmitter { $lastPath = null; while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { $this->scan($path, self::SCAN_RECURSIVE, self::REUSE_ETAG); - $this->cache->correctFolderSize($path); + \OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path)); + if(Config::getSystemValue('filesystem_check_enable', true)) { + $this->cache->correctFolderSize($path); + } $lastPath = $path; } } From 16ae63bdfd8556dceb251fd284dc97d330ca8092 Mon Sep 17 00:00:00 2001 From: ringmaster Date: Fri, 30 May 2014 09:42:41 -0400 Subject: [PATCH 6/7] Updates per comments on PR: * Use "filesystem_cache_readonly" config setting, update comment in config.sample * Use $this->cacheActive to cache config setting * Add public Scanner::setCacheActive() to set $cacheActive programmatically --- config/config.sample.php | 4 ++-- lib/private/files/cache/scanner.php | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index 8fb782e7cf..e2b4c9b31a 100755 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -293,7 +293,7 @@ $CONFIG = array( */ 'filesystem_check_changes' => 1, -/* specifies whether changes in the filesystem outside of owncloud affect cached data about those files */ -'filesystem_check_enable' => 1, +/* If true, prevent owncloud from changing the cache due to changes in the filesystem for all storage */ +'filesystem_cache_readonly' => false, ); diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index d4942d7dc2..875d941fb7 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -44,6 +44,11 @@ class Scanner extends BasicEmitter { */ private $permissionsCache; + /** + * @var boolean $cacheActive If true, perform cache operations, if false, do not affect cache + */ + protected $cacheActive; + const SCAN_RECURSIVE = true; const SCAN_SHALLOW = false; @@ -55,6 +60,7 @@ class Scanner extends BasicEmitter { $this->storageId = $this->storage->getId(); $this->cache = $storage->getCache(); $this->permissionsCache = $storage->getPermissionsCache(); + $this->cacheActive = !Config::getSystemValue('filesystem_cache_readonly', false); } /** @@ -139,7 +145,7 @@ class Scanner extends BasicEmitter { } $parentCacheData = $this->cache->get($parent); \OC_Hook::emit('Scanner', 'updateCache', array('file' => $file, 'data' => $data)); - if(Config::getSystemValue('filesystem_check_enable', true)) { + if($this->cacheActive) { $this->cache->update($parentCacheData['fileid'], array( 'etag' => $this->storage->getETag($parent), )); @@ -161,7 +167,7 @@ class Scanner extends BasicEmitter { } if (!empty($newData)) { \OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'data' => $newData)); - if(Config::getSystemValue('filesystem_check_enable', true)) { + if(!Config::getSystemValue('filesystem_cache_readonly', false)) { $data['fileid'] = $this->cache->put($file, $newData); } $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); @@ -169,7 +175,7 @@ class Scanner extends BasicEmitter { } } else { \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file)); - if(Config::getSystemValue('filesystem_check_enable', true)) { + if($this->cacheActive) { $this->cache->remove($file); } } @@ -255,7 +261,7 @@ class Scanner extends BasicEmitter { foreach ($removedChildren as $childName) { $child = ($path) ? $path . '/' . $childName : $childName; \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child)); - if(Config::getSystemValue('filesystem_check_enable', true)) { + if($this->cacheActive) { $this->cache->remove($child); } } @@ -278,7 +284,7 @@ class Scanner extends BasicEmitter { } $newData = array('size' => $size); \OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'data' => $newData)); - if(Config::getSystemValue('filesystem_check_enable', true)) { + if($this->cacheActive) { $this->cache->put($path, $newData); } } @@ -308,10 +314,18 @@ class Scanner extends BasicEmitter { while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { $this->scan($path, self::SCAN_RECURSIVE, self::REUSE_ETAG); \OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path)); - if(Config::getSystemValue('filesystem_check_enable', true)) { + if($this->cacheActive) { $this->cache->correctFolderSize($path); } $lastPath = $path; } } + + /** + * Set whether the cache is affected by scan operations + * @param boolean $active The active state of the cache + */ + public function setCacheActive($active) { + $this->cacheActive = $active; + } } From 19f0c47842527d565df3383a2c65830df3f3274f Mon Sep 17 00:00:00 2001 From: ringmaster Date: Fri, 30 May 2014 10:40:26 -0400 Subject: [PATCH 7/7] Missed one. --- lib/private/files/cache/scanner.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index 875d941fb7..6dceaf13ec 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -167,7 +167,7 @@ class Scanner extends BasicEmitter { } if (!empty($newData)) { \OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'data' => $newData)); - if(!Config::getSystemValue('filesystem_cache_readonly', false)) { + if($this->cacheActive) { $data['fileid'] = $this->cache->put($file, $newData); } $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId));