mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #8822 from owncloud/cache-change-propagator 

[WIP] Improved propagation of cache changes
This commit is contained in:
Vincent Petry 2014-06-04 13:03:14 +02:00
parent 5adb8f0a8a 3bcdad62fb
commit b5f0a17918
7 changed files with 320 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
lib/private/files/cache/changepropagator.php vendored Normal file
View File

@ -0,0 +1,98 @@
<?php
/**
* Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\Files\Cache;
/**
* Propagates changes in etag and mtime up the filesystem tree
*
* @package OC\Files\Cache
*/
class ChangePropagator {
/**
* @var string[]
*/
protected $changedFiles = array();
/**
* @var \OC\Files\View
*/
protected $view;
/**
* @param \OC\Files\View $view
*/
public function __construct(\OC\Files\View $view) {
$this->view = $view;
}
public function addChange($path) {
$this->changedFiles[] = $path;
}
public function getChanges() {
return $this->changedFiles;
}
/**
* propagate the registered changes to their parent folders
*
* @param int $time (optional) the mtime to set for the folders, if not set the current time is used
*/
public function propagateChanges($time = null) {
$parents = $this->getAllParents();
$this->changedFiles = array();
if (!$time) {
$time = time();
}
foreach ($parents as $parent) {
/**
* @var \OC\Files\Storage\Storage $storage
* @var string $internalPath
*/
list($storage, $internalPath) = $this->view->resolvePath($parent);
$cache = $storage->getCache();
$id = $cache->getId($internalPath);
$cache->update($id, array('mtime' => $time, 'etag' => $storage->getETag($internalPath)));
}
}
/**
* @return string[]
*/
public function getAllParents() {
$parents = array();
foreach ($this->getChanges() as $path) {
$parents = array_values(array_unique(array_merge($parents, $this->getParents($path))));
}
return $parents;
}
/**
* get all parent folders of $path
*
* @param string $path
* @return string[]
*/
protected function getParents($path) {
$parts = explode('/', $path);
// remove the singe file
array_pop($parts);
$result = array('/');
$resultPath = '';
foreach ($parts as $part) {
if ($part) {
$resultPath .= '/' . $part;
$result[] = $resultPath;
}
}
return $result;
}
}

95
lib/private/files/cache/scanner.php vendored
View File

@ -124,10 +124,8 @@ class Scanner extends BasicEmitter {
// prevent empty etag
if (empty($cacheData['etag'])) {
$etag = $data['etag'];
$propagateETagChange = true;
} else {
$etag = $cacheData['etag'];
$propagateETagChange = false;
}
// only reuse data if the file hasn't explicitly changed
if (isset($data['storage_mtime']) && isset($cacheData['storage_mtime']) && $data['storage_mtime'] === $cacheData['storage_mtime']) {
@ -136,22 +134,6 @@ class Scanner extends BasicEmitter {
}
if ($reuseExisting & self::REUSE_ETAG) {
$data['etag'] = $etag;
if ($propagateETagChange) {
$parent = $file;
while ($parent !== '') {
$parent = dirname($parent);
if ($parent === '.') {
$parent = '';
}
$parentCacheData = $this->cache->get($parent);
\OC_Hook::emit('Scanner', 'updateCache', array('file' => $file, 'data' => $data));
if($this->cacheActive) {
$this->cache->update($parentCacheData['fileid'], array(
'etag' => $this->storage->getETag($parent),
));
}
}
}
}
}
// Only update metadata that has changed
@ -166,24 +148,53 @@ class Scanner extends BasicEmitter {
}
}
if (!empty($newData)) {
\OC_Hook::emit('Scanner', 'addToCache', array('file' => $file, 'data' => $newData));
if($this->cacheActive) {
$data['fileid'] = $this->cache->put($file, $newData);
}
$data['fileid'] = $this->addToCache($file, $newData);
$this->emit('\OC\Files\Cache\Scanner', 'postScanFile', array($file, $this->storageId));
\OC_Hook::emit('\OC\Files\Cache\Scanner', 'post_scan_file', array('path' => $file, 'storage' => $this->storageId));
}
} else {
\OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $file));
if($this->cacheActive) {
$this->cache->remove($file);
}
$this->removeFromCache($file);
}
return $data;
}
return null;
}
protected function removeFromCache($path) {
\OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $path));
$this->emit('\OC\Files\Cache\Scanner', 'removeFromCache', array($path));
if ($this->cacheActive) {
$this->cache->remove($path);
}
}
/**
* @param string $path
* @param array $data
* @return int the id of the added file
*/
protected function addToCache($path, $data) {
\OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data));
$this->emit('\OC\Files\Cache\Scanner', 'addToCache', array($path, $this->storageId, $data));
if ($this->cacheActive) {
return $this->cache->put($path, $data);
} else {
return -1;
}
}
/**
* @param string $path
* @param array $data
*/
protected function updateCache($path, $data) {
\OC_Hook::emit('Scanner', 'addToCache', array('file' => $path, 'data' => $data));
$this->emit('\OC\Files\Cache\Scanner', 'updateCache', array($path, $this->storageId, $data));
if ($this->cacheActive) {
$this->cache->put($path, $data);
}
}
/**
* scan a folder and all it's children
*
@ -236,18 +247,15 @@ class Scanner extends BasicEmitter {
try {
$data = $this->scanFile($child, $reuse, true);
if ($data) {
if ($data['size'] === -1) {
if ($recursive === self::SCAN_RECURSIVE) {
$childQueue[] = $child;
} else {
$size = -1;
}
if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
$childQueue[] = $child;
} else if ($data['size'] === -1) {
$size = -1;
} else if ($size !== -1) {
$size += $data['size'];
}
}
}
catch (\Doctrine\DBAL\DBALException $ex){
} catch (\Doctrine\DBAL\DBALException $ex) {
// might happen if inserting duplicate while a scanning
// process is running in parallel
// log and ignore
@ -260,13 +268,10 @@ class Scanner extends BasicEmitter {
$removedChildren = \array_diff($existingChildren, $newChildren);
foreach ($removedChildren as $childName) {
$child = ($path) ? $path . '/' . $childName : $childName;
\OC_Hook::emit('Scanner', 'removeFromCache', array('file' => $child));
if($this->cacheActive) {
$this->cache->remove($child);
}
$this->removeFromCache($child);
}
\OC_DB::commit();
if ($exceptionOccurred){
if ($exceptionOccurred) {
// It might happen that the parallel scan process has already
// inserted mimetypes but those weren't available yet inside the transaction
// To make sure to have the updated mime types in such cases,
@ -278,15 +283,11 @@ class Scanner extends BasicEmitter {
$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
if ($childSize === -1) {
$size = -1;
} else {
} else if ($size !== -1) {
$size += $childSize;
}
}
$newData = array('size' => $size);
\OC_Hook::emit('Scanner', 'addToCache', array('file' => $child, 'data' => $newData));
if($this->cacheActive) {
$this->cache->put($path, $newData);
}
$this->updateCache($path, array('size' => $size));
}
$this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', array($path, $this->storageId));
return $size;
@ -296,6 +297,7 @@ class Scanner extends BasicEmitter {
* check if the file should be ignored when scanning
* NOTE: files with a '.part' extension are ignored as well!
* prevents unfinished put requests to be scanned
*
* @param string $file
* @return boolean
*/
@ -314,7 +316,7 @@ class Scanner extends BasicEmitter {
while (($path = $this->cache->getIncomplete()) !== false && $path !== $lastPath) {
$this->scan($path, self::SCAN_RECURSIVE, self::REUSE_ETAG);
\OC_Hook::emit('Scanner', 'correctFolderSize', array('path' => $path));
if($this->cacheActive) {
if ($this->cacheActive) {
$this->cache->correctFolderSize($path);
}
$lastPath = $path;
@ -323,6 +325,7 @@ class Scanner extends BasicEmitter {
/**
* Set whether the cache is affected by scan operations
*
* @param boolean $active The active state of the cache
*/
public function setCacheActive($active) {

21
lib/private/files/utils/scanner.php
View File

@ -8,6 +8,8 @@
namespace OC\Files\Utils;
use OC\Files\View;
use OC\Files\Cache\ChangePropagator;
use OC\Files\Filesystem;
use OC\Hooks\PublicEmitter;
@ -26,11 +28,17 @@ class Scanner extends PublicEmitter {
*/
private $user;
/**
* @var \OC\Files\Cache\ChangePropagator
*/
protected $propagator;
/**
* @param string $user
*/
public function __construct($user) {
$this->user = $user;
$this->propagator = new ChangePropagator(new View(''));
}
/**
@ -67,6 +75,15 @@ class Scanner extends PublicEmitter {
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount, $emitter) {
$emitter->emit('\OC\Files\Utils\Scanner', 'scanFolder', array($mount->getMountPoint() . $path));
});
// propagate etag and mtimes when files are changed or removed
$propagator = $this->propagator;
$propagatorListener = function ($path) use ($mount, $propagator) {
$fullPath = Filesystem::normalizePath($mount->getMountPoint() . $path);
$propagator->addChange($fullPath);
};
$scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', $propagatorListener);
$scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', $propagatorListener);
}
/**
@ -82,6 +99,7 @@ class Scanner extends PublicEmitter {
$this->attachListener($mount);
$scanner->backgroundScan();
}
$this->propagator->propagateChanges(time());
}
/**
@ -95,8 +113,9 @@ class Scanner extends PublicEmitter {
}
$scanner = $mount->getStorage()->getScanner();
$this->attachListener($mount);
$scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG);
$scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE);
}
$this->propagator->propagateChanges(time());
}
}

72
tests/lib/files/cache/changepropagator.php vendored Normal file
View File

@ -0,0 +1,72 @@
<?php
/**
* Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test\Files\Cache;
use OC\Files\Filesystem;
use OC\Files\Storage\Temporary;
use OC\Files\View;
class ChangePropagator extends \PHPUnit_Framework_TestCase {
/**
* @var \OC\Files\Cache\ChangePropagator
*/
private $propagator;
/**
* @var \OC\Files\View
*/
private $view;
public function setUp() {
$storage = new Temporary(array());
$root = '/' . uniqid();
Filesystem::mount($storage, array(), $root);
$this->view = new View($root);
$this->propagator = new \OC\Files\Cache\ChangePropagator($this->view);
}
public function testGetParentsSingle() {
$this->propagator->addChange('/foo/bar/asd');
$this->assertEquals(array('/', '/foo', '/foo/bar'), $this->propagator->getAllParents());
}
public function testGetParentsMultiple() {
$this->propagator->addChange('/foo/bar/asd');
$this->propagator->addChange('/foo/qwerty');
$this->propagator->addChange('/foo/asd/bar');
$this->assertEquals(array('/', '/foo', '/foo/bar', '/foo/asd'), $this->propagator->getAllParents());
}
public function testSinglePropagate() {
$this->view->mkdir('/foo');
$this->view->mkdir('/foo/bar');
$this->view->file_put_contents('/foo/bar/sad.txt', 'qwerty');
$oldInfo1 = $this->view->getFileInfo('/');
$oldInfo2 = $this->view->getFileInfo('/foo');
$oldInfo3 = $this->view->getFileInfo('/foo/bar');
$time = time() + 50;
$this->propagator->addChange('/foo/bar/sad.txt');
$this->propagator->propagateChanges($time);
$newInfo1 = $this->view->getFileInfo('/');
$newInfo2 = $this->view->getFileInfo('/foo');
$newInfo3 = $this->view->getFileInfo('/foo/bar');
$this->assertEquals($newInfo1->getMTime(), $time);
$this->assertEquals($newInfo2->getMTime(), $time);
$this->assertEquals($newInfo3->getMTime(), $time);
$this->assertNotEquals($oldInfo1->getEtag(), $newInfo1->getEtag());
$this->assertNotEquals($oldInfo2->getEtag(), $newInfo2->getEtag());
$this->assertNotEquals($oldInfo3->getEtag(), $newInfo3->getEtag());
}
}

8
tests/lib/files/cache/scanner.php vendored
View File

@ -234,13 +234,5 @@ class Scanner extends \PHPUnit_Framework_TestCase {
$newData0 = $this->cache->get('folder/bar.txt');
$this->assertInternalType('string', $newData0['etag']);
$this->assertNotEmpty($newData0['etag']);
$newData1 = $this->cache->get('folder');
$this->assertInternalType('string', $newData1['etag']);
$this->assertNotSame($data1['etag'], $newData1['etag']);
$newData2 = $this->cache->get('');
$this->assertInternalType('string', $newData2['etag']);
$this->assertNotSame($data2['etag'], $newData2['etag']);
}
}

64
tests/lib/files/utils/scanner.php
View File

@ -8,6 +8,7 @@
namespace Test\Files\Utils;
use OC\Files\Filesystem;
use OC\Files\Mount\Mount;
use OC\Files\Storage\Temporary;
@ -27,12 +28,21 @@ class TestScanner extends \OC\Files\Utils\Scanner {
protected function getMounts($dir) {
return $this->mounts;
}
public function getPropagator() {
return $this->propagator;
}
public function setPropagator($propagator) {
$this->propagator = $propagator;
}
}
class Scanner extends \PHPUnit_Framework_TestCase {
public function testReuseExistingRoot() {
$storage = new Temporary(array());
$mount = new Mount($storage, '');
Filesystem::getMountManager()->addMount($mount);
$cache = $storage->getCache();
$storage->mkdir('folder');
@ -54,6 +64,7 @@ class Scanner extends \PHPUnit_Framework_TestCase {
public function testReuseExistingFile() {
$storage = new Temporary(array());
$mount = new Mount($storage, '');
Filesystem::getMountManager()->addMount($mount);
$cache = $storage->getCache();
$storage->mkdir('folder');
@ -71,4 +82,57 @@ class Scanner extends \PHPUnit_Framework_TestCase {
$new = $cache->get('folder/bar.txt');
$this->assertEquals($old, $new);
}
public function testChangePropagator() {
/**
* @var \OC\Files\Cache\ChangePropagator $propagator
*/
$propagator = $this->getMock('\OC\Files\Cache\ChangePropagator', array('propagateChanges'), array(), '', false);
$storage = new Temporary(array());
$mount = new Mount($storage, '/foo');
Filesystem::getMountManager()->addMount($mount);
$cache = $storage->getCache();
$storage->mkdir('folder');
$storage->file_put_contents('foo.txt', 'qwerty');
$storage->file_put_contents('folder/bar.txt', 'qwerty');
$scanner = new TestScanner('');
$originalPropagator = $scanner->getPropagator();
$scanner->setPropagator($propagator);
$scanner->addMount($mount);
$scanner->scan('');
$changes = $propagator->getChanges();
$parents = $propagator->getAllParents();
sort($changes);
sort($parents);
$this->assertEquals(array('/foo', '/foo/folder', '/foo/folder/bar.txt', '/foo/foo.txt'), $changes);
$this->assertEquals(array('/', '/foo', '/foo/folder'), $parents);
$cache->put('foo.txt', array('mtime' => time() - 50));
$propagator = $this->getMock('\OC\Files\Cache\ChangePropagator', array('propagateChanges'), array(), '', false);
$scanner->setPropagator($propagator);
$storage->file_put_contents('foo.txt', 'asdasd');
$scanner->scan('');
$changes = $propagator->getChanges();
$parents = $propagator->getAllParents();
$this->assertEquals(array('/foo/foo.txt'), $changes);
$this->assertEquals(array('/', '/foo'), $parents);
$scanner->setPropagator($originalPropagator);
$oldInfo = $cache->get('');
$cache->put('foo.txt', array('mtime' => time() - 70));
$storage->file_put_contents('foo.txt', 'asdasd');
$scanner->scan('');
$newInfo = $cache->get('');
$this->assertNotEquals($oldInfo['etag'], $newInfo['etag']);
}
}

30
tests/lib/files/view.php
View File

@ -52,14 +52,18 @@ class View extends \PHPUnit_Framework_TestCase {
$storage1 = $this->getTestStorage();
$storage2 = $this->getTestStorage();
$storage3 = $this->getTestStorage();
\OC\Files\Filesystem::mount($storage1, array(), '/');
\OC\Files\Filesystem::mount($storage2, array(), '/substorage');
\OC\Files\Filesystem::mount($storage3, array(), '/folder/anotherstorage');
$root = '/' . uniqid();
\OC\Files\Filesystem::mount($storage1, array(), $root . '/');
\OC\Files\Filesystem::mount($storage2, array(), $root . '/substorage');
\OC\Files\Filesystem::mount($storage3, array(), $root . '/folder/anotherstorage');
$textSize = strlen("dummy file data\n");
$imageSize = filesize(\OC::$SERVERROOT . '/core/img/logo.png');
$storageSize = $textSize * 2 + $imageSize;
$rootView = new \OC\Files\View('');
$storageInfo = $storage3->getCache()->get('');
$this->assertEquals($storageSize, $storageInfo['size']);
$rootView = new \OC\Files\View($root);
$cachedData = $rootView->getFileInfo('/foo.txt');
$this->assertEquals($textSize, $cachedData['size']);
@ -110,7 +114,7 @@ class View extends \PHPUnit_Framework_TestCase {
$this->assertEquals('foo.png', $folderData[1]['name']);
$this->assertEquals('foo.txt', $folderData[2]['name']);
$folderView = new \OC\Files\View('/folder');
$folderView = new \OC\Files\View($root . '/folder');
$this->assertEquals($rootView->getFileInfo('/folder'), $folderView->getFileInfo('/'));
$cachedData = $rootView->getFileInfo('/foo.txt');
@ -580,9 +584,9 @@ class View extends \PHPUnit_Framework_TestCase {
$longPath = '';
// 4000 is the maximum path length in file_cache.path
$folderName = 'abcdefghijklmnopqrstuvwxyz012345678901234567890123456789';
$depth = (4000/57);
foreach (range(0, $depth-1) as $i) {
$longPath .= '/'.$folderName;
$depth = (4000 / 57);
foreach (range(0, $depth - 1) as $i) {
$longPath .= '/' . $folderName;
$result = $rootView->mkdir($longPath);
$this->assertTrue($result, "mkdir failed on $i - path length: " . strlen($longPath));
@ -598,7 +602,7 @@ class View extends \PHPUnit_Framework_TestCase {
$scanner->scan('');
$longPath = $folderName;
foreach (range(0, $depth-1) as $i) {
foreach (range(0, $depth - 1) as $i) {
$cachedFolder = $cache->get($longPath);
$this->assertTrue(is_array($cachedFolder), "No cache entry for folder at $i");
$this->assertEquals($folderName, $cachedFolder['name'], "Wrong cache entry for folder at $i");
@ -652,14 +656,14 @@ class View extends \PHPUnit_Framework_TestCase {
* @dataProvider tooLongPathDataProvider
* @expectedException \OCP\Files\InvalidPathException
*/
public function testTooLongPath($operation, $param0 = NULL) {
public function testTooLongPath($operation, $param0 = null) {
$longPath = '';
// 4000 is the maximum path length in file_cache.path
$folderName = 'abcdefghijklmnopqrstuvwxyz012345678901234567890123456789';
$depth = (4000/57);
foreach (range(0, $depth+1) as $i) {
$longPath .= '/'.$folderName;
$depth = (4000 / 57);
foreach (range(0, $depth + 1) as $i) {
$longPath .= '/' . $folderName;
}
$storage = new \OC\Files\Storage\Temporary(array());