From fa817599170e81b0eb28cb59446c32f0b5645f03 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 15 Jun 2017 13:59:06 +0200 Subject: [PATCH 1/4] fix moving folders out of a cache jail Signed-off-by: Robin Appelman --- lib/private/Files/Cache/Wrapper/CacheJail.php | 12 ++++- .../lib/Files/Cache/Wrapper/CacheJailTest.php | 50 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index ebab20fbae..8f12ca77ee 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -94,7 +94,7 @@ class CacheJail extends CacheWrapper { * get the stored metadata of a file or folder * * @param string /int $file - * @return array|false + * @return ICacheEntry|false */ public function get($file) { if (is_string($file) or $file == '') { @@ -175,6 +175,16 @@ class CacheJail extends CacheWrapper { $this->getCache()->move($this->getSourcePath($source), $this->getSourcePath($target)); } + /** + * Get the storage id and path needed for a move + * + * @param string $path + * @return array [$storageId, $internalPath] + */ + protected function getMoveInfo($path) { + return [$this->getNumericStorageId(), $this->getSourcePath($path)]; + } + /** * remove all entries for files that are stored on the storage from the cache */ diff --git a/tests/lib/Files/Cache/Wrapper/CacheJailTest.php b/tests/lib/Files/Cache/Wrapper/CacheJailTest.php index e3043c50d5..f26e3a59f1 100644 --- a/tests/lib/Files/Cache/Wrapper/CacheJailTest.php +++ b/tests/lib/Files/Cache/Wrapper/CacheJailTest.php @@ -8,6 +8,7 @@ namespace Test\Files\Cache\Wrapper; +use OC\Files\Cache\Wrapper\CacheJail; use Test\Files\Cache\CacheTest; /** @@ -80,4 +81,53 @@ class CacheJailTest extends CacheTest { //not supported $this->assertTrue(true); } + + function testMoveFromJail() { + $folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->sourceCache->put('source', $folderData); + $this->sourceCache->put('source/foo', $folderData); + $this->sourceCache->put('source/foo/bar', $folderData); + $this->sourceCache->put('target', $folderData); + + $jail = new CacheJail($this->sourceCache, 'source'); + + $this->sourceCache->moveFromCache($jail, 'foo', 'target/foo'); + + $this->assertTrue($this->sourceCache->inCache('target/foo')); + $this->assertTrue($this->sourceCache->inCache('target/foo/bar')); + } + + function testMoveToJail() { + $folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->sourceCache->put('source', $folderData); + $this->sourceCache->put('source/foo', $folderData); + $this->sourceCache->put('source/foo/bar', $folderData); + $this->sourceCache->put('target', $folderData); + + $jail = new CacheJail($this->sourceCache, 'target'); + + $jail->moveFromCache($this->sourceCache, 'source/foo', 'foo'); + + $this->assertTrue($this->sourceCache->inCache('target/foo')); + $this->assertTrue($this->sourceCache->inCache('target/foo/bar')); + } + + function testMoveBetweenJail() { + $folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->sourceCache->put('source', $folderData); + $this->sourceCache->put('source/foo', $folderData); + $this->sourceCache->put('source/foo/bar', $folderData); + $this->sourceCache->put('target', $folderData); + + $jail = new CacheJail($this->sourceCache, 'target'); + $sourceJail = new CacheJail($this->sourceCache, 'source'); + + $jail->moveFromCache($sourceJail, 'foo', 'foo'); + + $this->assertTrue($this->sourceCache->inCache('target/foo')); + $this->assertTrue($this->sourceCache->inCache('target/foo/bar')); + } } From d3c20eefca1e8be222022f09384d629f7933c561 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 20 Jun 2017 16:31:52 +0200 Subject: [PATCH 2/4] Add repair step for invalid paths Signed-off-by: Robin Appelman --- lib/private/Repair.php | 6 +- lib/private/Repair/RepairInvalidPaths.php | 121 ++++++++++++++++++++ tests/lib/Repair/RepairInvalidPathsTest.php | 111 ++++++++++++++++++ 3 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 lib/private/Repair/RepairInvalidPaths.php create mode 100644 tests/lib/Repair/RepairInvalidPathsTest.php diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 4d14bf2550..ca8dbd9875 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -42,6 +42,7 @@ use OC\Repair\NC12\UpdateLanguageCodes; use OC\Repair\OldGroupMembershipShares; use OC\Repair\Owncloud\SaveAccountsTableData; use OC\Repair\RemoveRootShares; +use OC\Repair\RepairInvalidPaths; use OC\Repair\SqliteAutoincrement; use OC\Repair\RepairMimeTypes; use OC\Repair\RepairInvalidShares; @@ -143,7 +144,8 @@ class Repair implements IOutput{ \OC::$server->query(BundleFetcher::class), \OC::$server->getConfig(), \OC::$server->query(Installer::class) - ) + ), + new RepairInvalidPaths(\OC::$server->getDatabaseConnection()) ]; } @@ -155,7 +157,7 @@ class Repair implements IOutput{ */ public static function getExpensiveRepairSteps() { return [ - new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()), + new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()) ]; } diff --git a/lib/private/Repair/RepairInvalidPaths.php b/lib/private/Repair/RepairInvalidPaths.php new file mode 100644 index 0000000000..cdd0906295 --- /dev/null +++ b/lib/private/Repair/RepairInvalidPaths.php @@ -0,0 +1,121 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Repair; + + +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class RepairInvalidPaths implements IRepairStep { + /** @var IDBConnection */ + private $connection; + + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + } + + + public function getName() { + return 'Repair invalid paths in file cache'; + } + + private function getInvalidEntries() { + $builder = $this->connection->getQueryBuilder(); + + $computedPath = $builder->func()->concat( + 'p.path', + $builder->func()->concat($builder->createNamedParameter('/'), 'f.name') + ); + + //select f.path, f.parent,p.path from oc_filecache f inner join oc_filecache p on f.parent=p.fileid and p.path!='' where f.path != p.path || '/' || f.name; + $query = $builder->select('f.fileid', 'f.path', 'p.path AS parent_path', 'f.name', 'f.parent', 'f.storage') + ->from('filecache', 'f') + ->innerJoin('f', 'filecache', 'p', $builder->expr()->andX( + $builder->expr()->eq('f.parent', 'p.fileid'), + $builder->expr()->neq('p.name', $builder->createNamedParameter('')) + )) + ->where($builder->expr()->neq('f.path', $computedPath)); + + return $query->execute()->fetchAll(); + } + + private function getId($storage, $path) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->select('fileid') + ->from('filecache') + ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storage))) + ->andWhere($builder->expr()->eq('path', $builder->createNamedParameter($path))); + + return $query->execute()->fetchColumn(); + } + + private function update($fileid, $newPath) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->update('filecache') + ->set('path', $builder->createNamedParameter($newPath)) + ->set('path_hash', $builder->createNamedParameter(md5($newPath))) + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid))); + + $query->execute(); + } + + private function reparent($from, $to) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->update('filecache') + ->set('parent', $builder->createNamedParameter($to)) + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($from))); + $query->execute(); + } + + private function delete($fileid) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->delete('filecache') + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid))); + $query->execute(); + } + + private function repair() { + $entries = $this->getInvalidEntries(); + foreach ($entries as $entry) { + $calculatedPath = $entry['parent_path'] . '/' . $entry['name']; + if ($newId = $this->getId($entry['storage'], $calculatedPath)) { + // a new entry with the correct path has already been created, reuse that one and delete the incorrect entry + $this->reparent($entry['fileid'], $newId); + $this->delete($entry['fileid']); + } else { + $this->update($entry['fileid'], $calculatedPath); + } + } + return count($entries); + } + + public function run(IOutput $output) { + $count = $this->repair(); + + $output->info('Repaired ' . $count . ' paths'); + } +} diff --git a/tests/lib/Repair/RepairInvalidPathsTest.php b/tests/lib/Repair/RepairInvalidPathsTest.php new file mode 100644 index 0000000000..c6c1bbb19b --- /dev/null +++ b/tests/lib/Repair/RepairInvalidPathsTest.php @@ -0,0 +1,111 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace Test\Repair; + +use OC\Files\Cache\Cache; +use OC\Files\Storage\Temporary; +use OC\Repair\RepairInvalidPaths; +use OCP\Migration\IOutput; +use Test\TestCase; + +/** + * @group DB + */ +class RepairInvalidPathsTest extends TestCase { + /** @var Temporary */ + private $storage; + /** @var Cache */ + private $cache; + /** @var RepairInvalidPaths */ + private $repair; + + protected function setUp() { + parent::setUp(); + + $this->storage = new Temporary(); + $this->cache = $this->storage->getCache(); + $this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection()); + } + + protected function tearDown() { + $this->cache->clear(); + + return parent::tearDown(); + } + + public function testRepairNonDuplicate() { + $this->storage->mkdir('foo/bar/asd'); + $this->storage->mkdir('foo2'); + $this->storage->getScanner()->scan(''); + + $folderId = $this->cache->getId('foo/bar'); + $newParentFolderId = $this->cache->getId('foo2'); + // failed rename, moved entry is updated but not it's children + $this->cache->update($folderId, ['path' => 'foo2/bar', 'parent' => $newParentFolderId]); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo/bar/asd')); + $this->assertFalse($this->cache->inCache('foo2/bar/asd')); + + $this->assertEquals($folderId, $this->cache->get('foo/bar/asd')['parent']); + + $this->repair->run($this->createMock(IOutput::class)); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo2/bar/asd')); + $this->assertFalse($this->cache->inCache('foo/bar/asd')); + + $this->assertEquals($folderId, $this->cache->get('foo2/bar/asd')['parent']); + $this->assertEquals($folderId, $this->cache->getId('foo2/bar')); + } + + public function testRepairDuplicate() { + $this->storage->mkdir('foo/bar/asd'); + $this->storage->mkdir('foo2'); + $this->storage->getScanner()->scan(''); + + $folderId = $this->cache->getId('foo/bar'); + $newParentFolderId = $this->cache->getId('foo2'); + // failed rename, moved entry is updated but not it's children + $this->cache->update($folderId, ['path' => 'foo2/bar', 'parent' => $newParentFolderId]); + $this->storage->rename('foo/bar', 'foo2/bar'); + $this->storage->mkdir('foo2/bar/asd/foo'); + + // usage causes the renamed subfolder to be scanned + $this->storage->getScanner()->scan('foo2/bar/asd'); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo/bar/asd')); + $this->assertTrue($this->cache->inCache('foo2/bar/asd')); + + $this->assertEquals($folderId, $this->cache->get('foo/bar/asd')['parent']); + + $this->repair->run($this->createMock(IOutput::class)); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo2/bar/asd')); + $this->assertFalse($this->cache->inCache('foo/bar/asd')); + + $this->assertEquals($this->cache->getId('foo2/bar'), $this->cache->get('foo2/bar/asd')['parent']); + $this->assertEquals($this->cache->getId('foo2/bar/asd'), $this->cache->get('foo2/bar/asd/foo')['parent']); + } +} From 846e62c225070cbe9f8a0c8b14ec2a1f87ba1ca1 Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Wed, 21 Jun 2017 16:50:20 -0500 Subject: [PATCH 3/4] Run repair step only once Signed-off-by: Morris Jobke --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + lib/private/Repair.php | 2 +- lib/private/Repair/{ => NC13}/RepairInvalidPaths.php | 10 +++++++--- version.php | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) rename lib/private/Repair/{ => NC13}/RepairInvalidPaths.php (91%) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 02979b426b..1832e7443b 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -739,6 +739,7 @@ return array( 'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => $baseDir . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php', 'OC\\Repair\\NC12\\InstallCoreBundle' => $baseDir . '/lib/private/Repair/NC12/InstallCoreBundle.php', 'OC\\Repair\\NC12\\UpdateLanguageCodes' => $baseDir . '/lib/private/Repair/NC12/UpdateLanguageCodes.php', + 'OC\\Repair\\NC13\\RepairInvalidPaths' => $baseDir . '/lib/private/Repair/NC13/RepairInvalidPaths.php', 'OC\\Repair\\OldGroupMembershipShares' => $baseDir . '/lib/private/Repair/OldGroupMembershipShares.php', 'OC\\Repair\\Owncloud\\SaveAccountsTableData' => $baseDir . '/lib/private/Repair/Owncloud/SaveAccountsTableData.php', 'OC\\Repair\\RemoveRootShares' => $baseDir . '/lib/private/Repair/RemoveRootShares.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 5f9e571a68..02633a32ff 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -769,6 +769,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php', 'OC\\Repair\\NC12\\InstallCoreBundle' => __DIR__ . '/../../..' . '/lib/private/Repair/NC12/InstallCoreBundle.php', 'OC\\Repair\\NC12\\UpdateLanguageCodes' => __DIR__ . '/../../..' . '/lib/private/Repair/NC12/UpdateLanguageCodes.php', + 'OC\\Repair\\NC13\\RepairInvalidPaths' => __DIR__ . '/../../..' . '/lib/private/Repair/NC13/RepairInvalidPaths.php', 'OC\\Repair\\OldGroupMembershipShares' => __DIR__ . '/../../..' . '/lib/private/Repair/OldGroupMembershipShares.php', 'OC\\Repair\\Owncloud\\SaveAccountsTableData' => __DIR__ . '/../../..' . '/lib/private/Repair/Owncloud/SaveAccountsTableData.php', 'OC\\Repair\\RemoveRootShares' => __DIR__ . '/../../..' . '/lib/private/Repair/RemoveRootShares.php', diff --git a/lib/private/Repair.php b/lib/private/Repair.php index ca8dbd9875..65b131169f 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -42,7 +42,7 @@ use OC\Repair\NC12\UpdateLanguageCodes; use OC\Repair\OldGroupMembershipShares; use OC\Repair\Owncloud\SaveAccountsTableData; use OC\Repair\RemoveRootShares; -use OC\Repair\RepairInvalidPaths; +use OC\Repair\NC13\RepairInvalidPaths; use OC\Repair\SqliteAutoincrement; use OC\Repair\RepairMimeTypes; use OC\Repair\RepairInvalidShares; diff --git a/lib/private/Repair/RepairInvalidPaths.php b/lib/private/Repair/NC13/RepairInvalidPaths.php similarity index 91% rename from lib/private/Repair/RepairInvalidPaths.php rename to lib/private/Repair/NC13/RepairInvalidPaths.php index cdd0906295..8551f8261e 100644 --- a/lib/private/Repair/RepairInvalidPaths.php +++ b/lib/private/Repair/NC13/RepairInvalidPaths.php @@ -19,7 +19,7 @@ * */ -namespace OC\Repair; +namespace OC\Repair\NC13; use OCP\IDBConnection; @@ -114,8 +114,12 @@ class RepairInvalidPaths implements IRepairStep { } public function run(IOutput $output) { - $count = $this->repair(); + $versionFromBeforeUpdate = $this->config->getSystemValue('version', '0.0.0'); + // was added to 12.0.0.30 and 13.0.0.1 + if (version_compare($versionFromBeforeUpdate, '12.0.0.30', '<') || version_compare($versionFromBeforeUpdate, '13.0.0.0', '==')) { + $count = $this->repair(); - $output->info('Repaired ' . $count . ' paths'); + $output->info('Repaired ' . $count . ' paths'); + } } } diff --git a/version.php b/version.php index e237770c82..d9e26eafce 100644 --- a/version.php +++ b/version.php @@ -26,7 +26,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version = array(13, 0, 0, 0); +$OC_Version = array(13, 0, 0, 1); // The human readable string $OC_VersionString = '13.0.0 alpha'; From 601362e164632a1b68e896bd5359d0b7308eed4f Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 29 Jun 2017 14:45:08 +0200 Subject: [PATCH 4/4] adjust to moved repair step Signed-off-by: Robin Appelman --- lib/private/Repair.php | 2 +- lib/private/Repair/NC13/RepairInvalidPaths.php | 6 +++++- tests/lib/Repair/RepairInvalidPathsTest.php | 10 ++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 65b131169f..dae328e634 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -145,7 +145,7 @@ class Repair implements IOutput{ \OC::$server->getConfig(), \OC::$server->query(Installer::class) ), - new RepairInvalidPaths(\OC::$server->getDatabaseConnection()) + new RepairInvalidPaths(\OC::$server->getDatabaseConnection(), \OC::$server->getConfig()) ]; } diff --git a/lib/private/Repair/NC13/RepairInvalidPaths.php b/lib/private/Repair/NC13/RepairInvalidPaths.php index 8551f8261e..076fbb735c 100644 --- a/lib/private/Repair/NC13/RepairInvalidPaths.php +++ b/lib/private/Repair/NC13/RepairInvalidPaths.php @@ -22,6 +22,7 @@ namespace OC\Repair\NC13; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; @@ -29,9 +30,12 @@ use OCP\Migration\IRepairStep; class RepairInvalidPaths implements IRepairStep { /** @var IDBConnection */ private $connection; + /** @var IConfig */ + private $config; - public function __construct(IDBConnection $connection) { + public function __construct(IDBConnection $connection, IConfig $config) { $this->connection = $connection; + $this->config = $config; } diff --git a/tests/lib/Repair/RepairInvalidPathsTest.php b/tests/lib/Repair/RepairInvalidPathsTest.php index c6c1bbb19b..b18758585c 100644 --- a/tests/lib/Repair/RepairInvalidPathsTest.php +++ b/tests/lib/Repair/RepairInvalidPathsTest.php @@ -23,7 +23,8 @@ namespace Test\Repair; use OC\Files\Cache\Cache; use OC\Files\Storage\Temporary; -use OC\Repair\RepairInvalidPaths; +use OC\Repair\NC13\RepairInvalidPaths; +use OCP\IConfig; use OCP\Migration\IOutput; use Test\TestCase; @@ -43,7 +44,12 @@ class RepairInvalidPathsTest extends TestCase { $this->storage = new Temporary(); $this->cache = $this->storage->getCache(); - $this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection()); + $config = $this->createMock(IConfig::class); + $config->expects($this->any()) + ->method('getSystemValue') + ->with('version', '0.0.0') + ->willReturn('12.0.0.0'); + $this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection(), $config); } protected function tearDown() {