From 56d10e9054c5f2699e3e0df00bd71a40f53be738 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Sun, 10 Feb 2013 18:15:23 +0100 Subject: [PATCH 1/7] Cache: simplify scanner logic a bit when handeling with unknown folder sizes --- lib/files/cache/scanner.php | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 5a9a119458..ff37c94424 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -101,18 +101,16 @@ class Scanner { $child = ($path) ? $path . '/' . $file : $file; $data = $this->scanFile($child); if ($data) { - if ($data['mimetype'] === 'httpd/unix-directory') { + if ($data['size'] === -1) { if ($recursive === self::SCAN_RECURSIVE) { $childQueue[] = $child; $data['size'] = 0; } else { - $data['size'] = -1; + $size = -1; } - } else { } - if ($data['size'] === -1) { - $size = -1; - } elseif ($size !== -1) { + + if ($size !== -1) { $size += $data['size']; } } @@ -133,7 +131,7 @@ class Scanner { } return $size; } - + /** * @brief check if the file should be ignored when scanning * NOTE: files with a '.part' extension are ignored as well! @@ -143,8 +141,8 @@ class Scanner { */ private function isIgnoredFile($file) { if ($file === '.' || $file === '..' - || pathinfo($file,PATHINFO_EXTENSION) === 'part') - { + || pathinfo($file, PATHINFO_EXTENSION) === 'part' + ) { return true; } return false; From 299649b40e1a87eee7bcead74b269fe8c452e04d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Sun, 10 Feb 2013 18:24:24 +0100 Subject: [PATCH 2/7] Cache: reuse known folder sizes when doing a shallow scan --- lib/files/cache/scanner.php | 10 +++++++--- tests/lib/files/cache/watcher.php | 1 - 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index ff37c94424..b27b3555c9 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -58,9 +58,10 @@ class Scanner { * scan a single file and store it in the cache * * @param string $file + * @param bool $checkExisting check existing folder sizes in the cache instead of always using -1 for folder size * @return array with metadata of the scanned file */ - public function scanFile($file) { + public function scanFile($file, $checkExisting = false) { \OC_Hook::emit('\OC\Files\Cache\Scanner', 'scan_file', array('path' => $file, 'storage' => $this->storageId)); $data = $this->getData($file); if ($data) { @@ -73,7 +74,10 @@ class Scanner { $this->scanFile($parent); } } - $id = $this->cache->put($file, $data); + if ($data['size'] === -1 and $cacheData = $this->cache->get($file)) { + $data['size'] = $cacheData['size']; + } + $this->cache->put($file, $data); } return $data; } @@ -99,7 +103,7 @@ class Scanner { while ($file = readdir($dh)) { if (!$this->isIgnoredFile($file)) { $child = ($path) ? $path . '/' . $file : $file; - $data = $this->scanFile($child); + $data = $this->scanFile($child, $recursive === self::SCAN_SHALLOW); if ($data) { if ($data['size'] === -1) { if ($recursive === self::SCAN_RECURSIVE) { diff --git a/tests/lib/files/cache/watcher.php b/tests/lib/files/cache/watcher.php index e8a1689cab..8ef6ab44d1 100644 --- a/tests/lib/files/cache/watcher.php +++ b/tests/lib/files/cache/watcher.php @@ -76,7 +76,6 @@ class Watcher extends \PHPUnit_Framework_TestCase { $updater->checkUpdate(''); $entry = $cache->get('foo.txt'); - $this->assertEquals(-1, $entry['size']); $this->assertEquals('httpd/unix-directory', $entry['mimetype']); $this->assertFalse($cache->inCache('folder')); $this->assertFalse($cache->inCache('folder/bar.txt')); From d84c3cd014fd73930cd15aee64d57aad3383b2aa Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 11 Feb 2013 13:24:56 +0100 Subject: [PATCH 3/7] Cache: actually use parameter --- lib/files/cache/scanner.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index b27b3555c9..ec21626442 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -74,7 +74,7 @@ class Scanner { $this->scanFile($parent); } } - if ($data['size'] === -1 and $cacheData = $this->cache->get($file)) { + if ($checkExisting and $data['size'] === -1 and $cacheData = $this->cache->get($file)) { $data['size'] = $cacheData['size']; } $this->cache->put($file, $data); From 2921d2fb785f265bb3bf17873ada304145d49aec Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 11 Feb 2013 13:27:34 +0100 Subject: [PATCH 4/7] Cache: don't create a new etag when the mtime hasn't changed --- lib/files/cache/scanner.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index ec21626442..7f19261d97 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -76,6 +76,9 @@ class Scanner { } if ($checkExisting and $data['size'] === -1 and $cacheData = $this->cache->get($file)) { $data['size'] = $cacheData['size']; + if ($data['mtime'] === $cacheData['mtime']) { + $data['etag'] = $cacheData['etag']; + } } $this->cache->put($file, $data); } From e1fe5279ddd848b1ee367184c2c6ba7c763bd4a7 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 12 Feb 2013 14:56:57 +0100 Subject: [PATCH 5/7] Cache: also preserve etags for files --- lib/files/cache/scanner.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 7f19261d97..70266c26e6 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -74,8 +74,11 @@ class Scanner { $this->scanFile($parent); } } - if ($checkExisting and $data['size'] === -1 and $cacheData = $this->cache->get($file)) { - $data['size'] = $cacheData['size']; + if ($checkExisting) { + $cacheData = $this->cache->get($file) + if ($data['size'] === -1) { + $data['size'] = $cacheData['size']; + } if ($data['mtime'] === $cacheData['mtime']) { $data['etag'] = $cacheData['etag']; } From f2baf1ae0eb7f5cd34773b05de2e286eab2cd0a1 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 12 Feb 2013 16:18:48 +0100 Subject: [PATCH 6/7] fixing syntax error --- lib/files/cache/scanner.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 70266c26e6..9f72e20614 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -75,7 +75,7 @@ class Scanner { } } if ($checkExisting) { - $cacheData = $this->cache->get($file) + $cacheData = $this->cache->get($file); if ($data['size'] === -1) { $data['size'] = $cacheData['size']; } From b54dcd1999b0ce447dc6920ac8cd361095b8346d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 12 Feb 2013 16:48:21 +0100 Subject: [PATCH 7/7] Cache: fix scanner trying to use existing data when file isn't in cache --- lib/files/cache/scanner.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/files/cache/scanner.php b/lib/files/cache/scanner.php index 9f72e20614..88f208547f 100644 --- a/lib/files/cache/scanner.php +++ b/lib/files/cache/scanner.php @@ -74,8 +74,7 @@ class Scanner { $this->scanFile($parent); } } - if ($checkExisting) { - $cacheData = $this->cache->get($file); + if ($checkExisting and $cacheData = $this->cache->get($file)) { if ($data['size'] === -1) { $data['size'] = $cacheData['size']; }