diff --git a/lib/private/files/cache/changepropagator.php b/lib/private/files/cache/changepropagator.php new file mode 100644 index 0000000000..30f2e675e2 --- /dev/null +++ b/lib/private/files/cache/changepropagator.php @@ -0,0 +1,98 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\Files\Cache; + +/** + * Propagates changes in etag and mtime up the filesystem tree + * + * @package OC\Files\Cache + */ +class ChangePropagator { + /** + * @var string[] + */ + protected $changedFiles = array(); + + /** + * @var \OC\Files\View + */ + protected $view; + + /** + * @param \OC\Files\View $view + */ + public function __construct(\OC\Files\View $view) { + $this->view = $view; + } + + public function addChange($path) { + $this->changedFiles[] = $path; + } + + public function getChanges() { + return $this->changedFiles; + } + + /** + * propagate the registered changes to their parent folders + * + * @param int $time (optional) the mtime to set for the folders, if not set the current time is used + */ + public function propagateChanges($time = null) { + $parents = $this->getAllParents(); + $this->changedFiles = array(); + if (!$time) { + $time = time(); + } + foreach ($parents as $parent) { + /** + * @var \OC\Files\Storage\Storage $storage + * @var string $internalPath + */ + + list($storage, $internalPath) = $this->view->resolvePath($parent); + $cache = $storage->getCache(); + $id = $cache->getId($internalPath); + $cache->update($id, array('mtime' => $time, 'etag' => $storage->getETag($internalPath))); + } + } + + /** + * @return string[] + */ + public function getAllParents() { + $parents = array(); + foreach ($this->getChanges() as $path) { + $parents = array_values(array_unique(array_merge($parents, $this->getParents($path)))); + } + return $parents; + } + + /** + * get all parent folders of $path + * + * @param string $path + * @return string[] + */ + protected function getParents($path) { + $parts = explode('/', $path); + + // remove the singe file + array_pop($parts); + $result = array('/'); + $resultPath = ''; + foreach ($parts as $part) { + if ($part) { + $resultPath .= '/' . $part; + $result[] = $resultPath; + } + } + return $result; + } +} diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index b97070fcdf..6cb9807f5a 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -124,10 +124,8 @@ class Scanner extends BasicEmitter { // prevent empty etag if (empty($cacheData['etag'])) { $etag = $data['etag']; - $propagateETagChange = true; } else { $etag = $cacheData['etag']; - $propagateETagChange = false; } // only reuse data if the file hasn't explicitly changed if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) { @@ -136,22 +134,6 @@ class Scanner extends BasicEmitter { } if ($reuseExisting & self::REUSE_ETAG) { $data['etag'] = $etag; - if ($propagateETagChange) { - $parent = $file; - while ($parent !== '') { - $parent = dirname($parent); - if ($parent === '.') { - $parent = ''; - } - $parentCacheData = $this->cache->get($parent); - \OC_Hook::emit('Scanner', 'updateCache', array('file' => $file, 'data' => $data)); - if($this->cacheActive) { - $this->cache->update($parentCacheData['fileid'], array( - 'etag' => $this->storage->getETag($parent), - )); - } - } - } } } // Only update metadata that has changed @@ -166,24 +148,53 @@ class Scanner extends BasicEmitter { } } if (!empty($newData)) { - \OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'data' => $newData)); - if($this->cacheActive) { - $data['fileid'] = $this->cache->put($file, $newData); - } + $data['fileid'] = $this->addToCache($file, $newData); $this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId)); \OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId)); } } else { - \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file)); - if($this->cacheActive) { - $this->cache->remove($file); - } + $this->removeFromCache($file); } return $data; } return null; } + protected function removeFromCache($path) { + \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $path)); + $this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', array($path)); + if ($this->cacheActive) { + $this->cache->remove($path); + } + } + + /** + * @param string $path + * @param array $data + * @return int the id of the added file + */ + protected function addToCache($path, $data) { + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); + $this->emit('\OC\Files\Cache\Scanner', 'addToCache', array($path, $this->storageId, $data)); + if ($this->cacheActive) { + return $this->cache->put($path, $data); + } else { + return -1; + } + } + + /** + * @param string $path + * @param array $data + */ + protected function updateCache($path, $data) { + \OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data)); + $this->emit('\OC\Files\Cache\Scanner', 'updateCache', array($path, $this->storageId, $data)); + if ($this->cacheActive) { + $this->cache->put($path, $data); + } + } + /** * scan a folder and all it's children * @@ -236,18 +247,15 @@ class Scanner extends BasicEmitter { try { $data = $this->scanFile($child, $reuse, true); if ($data) { - if ($data['size'] === -1) { - if ($recursive === self::SCAN_RECURSIVE) { - $childQueue[] = $child; - } else { - $size = -1; - } + if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { + $childQueue[] = $child; + } else if ($data['size'] === -1) { + $size = -1; } else if ($size !== -1) { $size += $data['size']; } } - } - catch (\Doctrine\DBAL\DBALException $ex){ + } catch (\Doctrine\DBAL\DBALException $ex) { // might happen if inserting duplicate while a scanning // process is running in parallel // log and ignore @@ -260,13 +268,10 @@ class Scanner extends BasicEmitter { $removedChildren = \array_diff($existingChildren, $newChildren); foreach ($removedChildren as $childName) { $child = ($path) ? $path . '/' . $childName : $childName; - \OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child)); - if($this->cacheActive) { - $this->cache->remove($child); - } + $this->removeFromCache($child); } \OC_DB::commit(); - if ($exceptionOccurred){ + if ($exceptionOccurred) { // It might happen that the parallel scan process has already // inserted mimetypes but those weren't available yet inside the transaction // To make sure to have the updated mime types in such cases, @@ -278,15 +283,11 @@ class Scanner extends BasicEmitter { $childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse); if ($childSize === -1) { $size = -1; - } else { + } else if ($size !== -1) { $size += $childSize; } } - $newData = array('size' => $size); - \OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'data' => $newData)); - if($this->cacheActive) { - $this->cache->put($path, $newData); - } + $this->updateCache($path, array('size' => $size)); } $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId)); return $size; @@ -296,6 +297,7 @@ class Scanner extends BasicEmitter { * check if the file should be ignored when scanning * NOTE: files with a '.part' extension are ignored as well! * prevents unfinished put requests to be scanned + * * @param string $file * @return boolean */ @@ -314,7 +316,7 @@ class Scanner extends BasicEmitter { while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) { $this->scan($path, self::SCAN_RECURSIVE, self::REUSE_ETAG); \OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path)); - if($this->cacheActive) { + if ($this->cacheActive) { $this->cache->correctFolderSize($path); } $lastPath = $path; @@ -323,6 +325,7 @@ class Scanner extends BasicEmitter { /** * Set whether the cache is affected by scan operations + * * @param boolean $active The active state of the cache */ public function setCacheActive($active) { diff --git a/lib/private/files/utils/scanner.php b/lib/private/files/utils/scanner.php index a802a8fcb8..1bb3e694c9 100644 --- a/lib/private/files/utils/scanner.php +++ b/lib/private/files/utils/scanner.php @@ -8,6 +8,8 @@ namespace OC\Files\Utils; +use OC\Files\View; +use OC\Files\Cache\ChangePropagator; use OC\Files\Filesystem; use OC\Hooks\PublicEmitter; @@ -26,11 +28,17 @@ class Scanner extends PublicEmitter { */ private $user; + /** + * @var \OC\Files\Cache\ChangePropagator + */ + protected $propagator; + /** * @param string $user */ public function __construct($user) { $this->user = $user; + $this->propagator = new ChangePropagator(new View('')); } /** @@ -67,6 +75,15 @@ class Scanner extends PublicEmitter { $scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount, $emitter) { $emitter->emit('\OC\Files\Utils\Scanner', 'scanFolder', array($mount->getMountPoint() . $path)); }); + + // propagate etag and mtimes when files are changed or removed + $propagator = $this->propagator; + $propagatorListener = function ($path) use ($mount, $propagator) { + $fullPath = Filesystem::normalizePath($mount->getMountPoint() . $path); + $propagator->addChange($fullPath); + }; + $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', $propagatorListener); + $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', $propagatorListener); } /** @@ -82,6 +99,7 @@ class Scanner extends PublicEmitter { $this->attachListener($mount); $scanner->backgroundScan(); } + $this->propagator->propagateChanges(time()); } /** @@ -95,8 +113,9 @@ class Scanner extends PublicEmitter { } $scanner = $mount->getStorage()->getScanner(); $this->attachListener($mount); - $scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG); + $scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); } + $this->propagator->propagateChanges(time()); } } diff --git a/tests/lib/files/cache/changepropagator.php b/tests/lib/files/cache/changepropagator.php new file mode 100644 index 0000000000..9beff27d50 --- /dev/null +++ b/tests/lib/files/cache/changepropagator.php @@ -0,0 +1,72 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test\Files\Cache; + +use OC\Files\Filesystem; +use OC\Files\Storage\Temporary; +use OC\Files\View; + +class ChangePropagator extends \PHPUnit_Framework_TestCase { + /** + * @var \OC\Files\Cache\ChangePropagator + */ + private $propagator; + + /** + * @var \OC\Files\View + */ + private $view; + + public function setUp() { + $storage = new Temporary(array()); + $root = '/' . uniqid(); + Filesystem::mount($storage, array(), $root); + $this->view = new View($root); + $this->propagator = new \OC\Files\Cache\ChangePropagator($this->view); + } + + public function testGetParentsSingle() { + $this->propagator->addChange('/foo/bar/asd'); + $this->assertEquals(array('/', '/foo', '/foo/bar'), $this->propagator->getAllParents()); + } + + public function testGetParentsMultiple() { + $this->propagator->addChange('/foo/bar/asd'); + $this->propagator->addChange('/foo/qwerty'); + $this->propagator->addChange('/foo/asd/bar'); + $this->assertEquals(array('/', '/foo', '/foo/bar', '/foo/asd'), $this->propagator->getAllParents()); + } + + public function testSinglePropagate() { + $this->view->mkdir('/foo'); + $this->view->mkdir('/foo/bar'); + $this->view->file_put_contents('/foo/bar/sad.txt', 'qwerty'); + + $oldInfo1 = $this->view->getFileInfo('/'); + $oldInfo2 = $this->view->getFileInfo('/foo'); + $oldInfo3 = $this->view->getFileInfo('/foo/bar'); + + $time = time() + 50; + + $this->propagator->addChange('/foo/bar/sad.txt'); + $this->propagator->propagateChanges($time); + + $newInfo1 = $this->view->getFileInfo('/'); + $newInfo2 = $this->view->getFileInfo('/foo'); + $newInfo3 = $this->view->getFileInfo('/foo/bar'); + + $this->assertEquals($newInfo1->getMTime(), $time); + $this->assertEquals($newInfo2->getMTime(), $time); + $this->assertEquals($newInfo3->getMTime(), $time); + + $this->assertNotEquals($oldInfo1->getEtag(), $newInfo1->getEtag()); + $this->assertNotEquals($oldInfo2->getEtag(), $newInfo2->getEtag()); + $this->assertNotEquals($oldInfo3->getEtag(), $newInfo3->getEtag()); + } +} diff --git a/tests/lib/files/cache/scanner.php b/tests/lib/files/cache/scanner.php index fb06f2dff3..56c7b65bb0 100644 --- a/tests/lib/files/cache/scanner.php +++ b/tests/lib/files/cache/scanner.php @@ -234,13 +234,5 @@ class Scanner extends \PHPUnit_Framework_TestCase { $newData0 = $this->cache->get('folder/bar.txt'); $this->assertInternalType('string', $newData0['etag']); $this->assertNotEmpty($newData0['etag']); - - $newData1 = $this->cache->get('folder'); - $this->assertInternalType('string', $newData1['etag']); - $this->assertNotSame($data1['etag'], $newData1['etag']); - - $newData2 = $this->cache->get(''); - $this->assertInternalType('string', $newData2['etag']); - $this->assertNotSame($data2['etag'], $newData2['etag']); } } diff --git a/tests/lib/files/utils/scanner.php b/tests/lib/files/utils/scanner.php index a021d215ae..159a2a4867 100644 --- a/tests/lib/files/utils/scanner.php +++ b/tests/lib/files/utils/scanner.php @@ -8,6 +8,7 @@ namespace Test\Files\Utils; +use OC\Files\Filesystem; use OC\Files\Mount\Mount; use OC\Files\Storage\Temporary; @@ -27,12 +28,21 @@ class TestScanner extends \OC\Files\Utils\Scanner { protected function getMounts($dir) { return $this->mounts; } + + public function getPropagator() { + return $this->propagator; + } + + public function setPropagator($propagator) { + $this->propagator = $propagator; + } } class Scanner extends \PHPUnit_Framework_TestCase { public function testReuseExistingRoot() { $storage = new Temporary(array()); $mount = new Mount($storage, ''); + Filesystem::getMountManager()->addMount($mount); $cache = $storage->getCache(); $storage->mkdir('folder'); @@ -54,6 +64,7 @@ class Scanner extends \PHPUnit_Framework_TestCase { public function testReuseExistingFile() { $storage = new Temporary(array()); $mount = new Mount($storage, ''); + Filesystem::getMountManager()->addMount($mount); $cache = $storage->getCache(); $storage->mkdir('folder'); @@ -71,4 +82,57 @@ class Scanner extends \PHPUnit_Framework_TestCase { $new = $cache->get('folder/bar.txt'); $this->assertEquals($old, $new); } + + public function testChangePropagator() { + /** + * @var \OC\Files\Cache\ChangePropagator $propagator + */ + $propagator = $this->getMock('\OC\Files\Cache\ChangePropagator', array('propagateChanges'), array(), '', false); + + $storage = new Temporary(array()); + $mount = new Mount($storage, '/foo'); + Filesystem::getMountManager()->addMount($mount); + $cache = $storage->getCache(); + + $storage->mkdir('folder'); + $storage->file_put_contents('foo.txt', 'qwerty'); + $storage->file_put_contents('folder/bar.txt', 'qwerty'); + + $scanner = new TestScanner(''); + $originalPropagator = $scanner->getPropagator(); + $scanner->setPropagator($propagator); + $scanner->addMount($mount); + + $scanner->scan(''); + + $changes = $propagator->getChanges(); + $parents = $propagator->getAllParents(); + sort($changes); + sort($parents); + $this->assertEquals(array('/foo', '/foo/folder', '/foo/folder/bar.txt', '/foo/foo.txt'), $changes); + $this->assertEquals(array('/', '/foo', '/foo/folder'), $parents); + + $cache->put('foo.txt', array('mtime' => time() - 50)); + + $propagator = $this->getMock('\OC\Files\Cache\ChangePropagator', array('propagateChanges'), array(), '', false); + $scanner->setPropagator($propagator); + $storage->file_put_contents('foo.txt', 'asdasd'); + + $scanner->scan(''); + + $changes = $propagator->getChanges(); + $parents = $propagator->getAllParents(); + $this->assertEquals(array('/foo/foo.txt'), $changes); + $this->assertEquals(array('/', '/foo'), $parents); + + $scanner->setPropagator($originalPropagator); + + $oldInfo = $cache->get(''); + $cache->put('foo.txt', array('mtime' => time() - 70)); + $storage->file_put_contents('foo.txt', 'asdasd'); + + $scanner->scan(''); + $newInfo = $cache->get(''); + $this->assertNotEquals($oldInfo['etag'], $newInfo['etag']); + } } diff --git a/tests/lib/files/view.php b/tests/lib/files/view.php index b5e4d79235..1d8e729fb7 100644 --- a/tests/lib/files/view.php +++ b/tests/lib/files/view.php @@ -52,14 +52,18 @@ class View extends \PHPUnit_Framework_TestCase { $storage1 = $this->getTestStorage(); $storage2 = $this->getTestStorage(); $storage3 = $this->getTestStorage(); - \OC\Files\Filesystem::mount($storage1, array(), '/'); - \OC\Files\Filesystem::mount($storage2, array(), '/substorage'); - \OC\Files\Filesystem::mount($storage3, array(), '/folder/anotherstorage'); + $root = '/' . uniqid(); + \OC\Files\Filesystem::mount($storage1, array(), $root . '/'); + \OC\Files\Filesystem::mount($storage2, array(), $root . '/substorage'); + \OC\Files\Filesystem::mount($storage3, array(), $root . '/folder/anotherstorage'); $textSize = strlen("dummy file data\n"); $imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png'); $storageSize = $textSize * 2 + $imageSize; - $rootView = new \OC\Files\View(''); + $storageInfo = $storage3->getCache()->get(''); + $this->assertEquals($storageSize, $storageInfo['size']); + + $rootView = new \OC\Files\View($root); $cachedData = $rootView->getFileInfo('/foo.txt'); $this->assertEquals($textSize, $cachedData['size']); @@ -110,7 +114,7 @@ class View extends \PHPUnit_Framework_TestCase { $this->assertEquals('foo.png', $folderData[1]['name']); $this->assertEquals('foo.txt', $folderData[2]['name']); - $folderView = new \OC\Files\View('/folder'); + $folderView = new \OC\Files\View($root . '/folder'); $this->assertEquals($rootView->getFileInfo('/folder'), $folderView->getFileInfo('/')); $cachedData = $rootView->getFileInfo('/foo.txt'); @@ -580,9 +584,9 @@ class View extends \PHPUnit_Framework_TestCase { $longPath = ''; // 4000 is the maximum path length in file_cache.path $folderName = 'abcdefghijklmnopqrstuvwxyz012345678901234567890123456789'; - $depth = (4000/57); - foreach (range(0, $depth-1) as $i) { - $longPath .= '/'.$folderName; + $depth = (4000 / 57); + foreach (range(0, $depth - 1) as $i) { + $longPath .= '/' . $folderName; $result = $rootView->mkdir($longPath); $this->assertTrue($result, "mkdir failed on $i - path length: " . strlen($longPath)); @@ -598,7 +602,7 @@ class View extends \PHPUnit_Framework_TestCase { $scanner->scan(''); $longPath = $folderName; - foreach (range(0, $depth-1) as $i) { + foreach (range(0, $depth - 1) as $i) { $cachedFolder = $cache->get($longPath); $this->assertTrue(is_array($cachedFolder), "No cache entry for folder at $i"); $this->assertEquals($folderName, $cachedFolder['name'], "Wrong cache entry for folder at $i"); @@ -652,14 +656,14 @@ class View extends \PHPUnit_Framework_TestCase { * @dataProvider tooLongPathDataProvider * @expectedException \OCP\Files\InvalidPathException */ - public function testTooLongPath($operation, $param0 = NULL) { + public function testTooLongPath($operation, $param0 = null) { $longPath = ''; // 4000 is the maximum path length in file_cache.path $folderName = 'abcdefghijklmnopqrstuvwxyz012345678901234567890123456789'; - $depth = (4000/57); - foreach (range(0, $depth+1) as $i) { - $longPath .= '/'.$folderName; + $depth = (4000 / 57); + foreach (range(0, $depth + 1) as $i) { + $longPath .= '/' . $folderName; } $storage = new \OC\Files\Storage\Temporary(array());