From de2ad46c3cd8f6fedf0ba1f1b50b92a00e623b6e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 18 Jan 2021 18:30:19 +0100 Subject: [PATCH 1/4] add command to repair broken filesystem trees Signed-off-by: Robin Appelman --- apps/files/appinfo/info.xml | 1 + .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + apps/files/lib/Command/RepairTree.php | 107 ++++++++++++++++++ 4 files changed, 110 insertions(+) create mode 100644 apps/files/lib/Command/RepairTree.php diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index c5b1e85f9b..5f1ef494b7 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -34,6 +34,7 @@ OCA\Files\Command\DeleteOrphanedFiles OCA\Files\Command\TransferOwnership OCA\Files\Command\ScanAppData + OCA\Files\Command\RepairTree diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index a4a72d59c1..73104f155f 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -28,6 +28,7 @@ return array( 'OCA\\Files\\Collaboration\\Resources\\Listener' => $baseDir . '/../lib/Collaboration/Resources/Listener.php', 'OCA\\Files\\Collaboration\\Resources\\ResourceProvider' => $baseDir . '/../lib/Collaboration/Resources/ResourceProvider.php', 'OCA\\Files\\Command\\DeleteOrphanedFiles' => $baseDir . '/../lib/Command/DeleteOrphanedFiles.php', + 'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php', 'OCA\\Files\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php', 'OCA\\Files\\Command\\ScanAppData' => $baseDir . '/../lib/Command/ScanAppData.php', 'OCA\\Files\\Command\\TransferOwnership' => $baseDir . '/../lib/Command/TransferOwnership.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index 91e29fac48..cc360c9891 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -43,6 +43,7 @@ class ComposerStaticInitFiles 'OCA\\Files\\Collaboration\\Resources\\Listener' => __DIR__ . '/..' . '/../lib/Collaboration/Resources/Listener.php', 'OCA\\Files\\Collaboration\\Resources\\ResourceProvider' => __DIR__ . '/..' . '/../lib/Collaboration/Resources/ResourceProvider.php', 'OCA\\Files\\Command\\DeleteOrphanedFiles' => __DIR__ . '/..' . '/../lib/Command/DeleteOrphanedFiles.php', + 'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php', 'OCA\\Files\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php', 'OCA\\Files\\Command\\ScanAppData' => __DIR__ . '/..' . '/../lib/Command/ScanAppData.php', 'OCA\\Files\\Command\\TransferOwnership' => __DIR__ . '/..' . '/../lib/Command/TransferOwnership.php', diff --git a/apps/files/lib/Command/RepairTree.php b/apps/files/lib/Command/RepairTree.php new file mode 100644 index 0000000000..77b97d7ca0 --- /dev/null +++ b/apps/files/lib/Command/RepairTree.php @@ -0,0 +1,107 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Files\Command; + +use OCP\IDBConnection; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class RepairTree extends Command { + public const CHUNK_SIZE = 200; + + /** + * @var IDBConnection + */ + protected $connection; + + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + parent::__construct(); + } + + protected function configure() { + $this + ->setName('files:repair-tree') + ->setDescription('Try and repair malformed filesystem tree structures') + ->addOption('dry-run'); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $rows = $this->findBrokenTreeBits(); + $fix = !$input->getOption('dry-run'); + + $output->writeln("Found " . count($rows) . " file entries with an invalid path"); + + if ($fix) { + $this->connection->beginTransaction(); + } + + $query = $this->connection->getQueryBuilder(); + $query->update('filecache') + ->set('path', $query->createParameter('path')) + ->set('path_hash', $query->func()->md5($query->createParameter('path'))) + ->where($query->expr()->eq('fileid', $query->createParameter('fileid'))); + + foreach ($rows as $row) { + $output->writeln("Path of file ${row['fileid']} is ${row['path']} but should be ${row['parent_path']}/${row['name']} based on it's parent", OutputInterface::VERBOSITY_VERBOSE); + + if ($fix) { + $query->setParameters([ + 'fileid' => $row['fileid'], + 'path' => $row['parent_path'] . '/' . $row['name'], + ]); + $query->execute(); + } + } + + if ($fix) { + $this->connection->commit(); + } + + return 0; + } + + private function findBrokenTreeBits(): array { + $query = $this->connection->getQueryBuilder(); + + $query->select('f.fileid', 'f.path', 'f.parent', 'f.name') + ->selectAlias('p.path', 'parent_path') + ->from('filecache', 'f') + ->innerJoin('f', 'filecache', 'p', $query->expr()->eq('f.parent', 'p.fileid')) + ->where($query->expr()->orX( + $query->expr()->andX( + $query->expr()->neq('p.path_hash', $query->createNamedParameter(md5(''))), + $query->expr()->neq('f.path', $query->func()->concat('p.path', $query->func()->concat($query->createNamedParameter('/'), 'f.name'))) + ), + $query->expr()->andX( + $query->expr()->eq('p.path_hash', $query->createNamedParameter(md5(''))), + $query->expr()->neq('f.path', 'f.name') + ), + $query->expr()->neq('f.storage', 'p.storage') + )); + + return $query->execute()->fetchAll(); + } +} From 0e790dad314ec4152793b7c1e58049e279fcf6b8 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 22 Jan 2021 16:05:14 +0100 Subject: [PATCH 2/4] also repair storage id Signed-off-by: Robin Appelman --- apps/files/lib/Command/RepairTree.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/files/lib/Command/RepairTree.php b/apps/files/lib/Command/RepairTree.php index 77b97d7ca0..d021b59722 100644 --- a/apps/files/lib/Command/RepairTree.php +++ b/apps/files/lib/Command/RepairTree.php @@ -62,6 +62,7 @@ class RepairTree extends Command { $query->update('filecache') ->set('path', $query->createParameter('path')) ->set('path_hash', $query->func()->md5($query->createParameter('path'))) + ->set('storage', $query->createParameter('storage')) ->where($query->expr()->eq('fileid', $query->createParameter('fileid'))); foreach ($rows as $row) { @@ -71,6 +72,7 @@ class RepairTree extends Command { $query->setParameters([ 'fileid' => $row['fileid'], 'path' => $row['parent_path'] . '/' . $row['name'], + 'storage' => $row['parent_storage'], ]); $query->execute(); } @@ -88,6 +90,7 @@ class RepairTree extends Command { $query->select('f.fileid', 'f.path', 'f.parent', 'f.name') ->selectAlias('p.path', 'parent_path') + ->selectAlias('p.storage', 'parent_storage') ->from('filecache', 'f') ->innerJoin('f', 'filecache', 'p', $query->expr()->eq('f.parent', 'p.fileid')) ->where($query->expr()->orX( From f8346f43f136a060e6938f157338570b60363e97 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 26 Jan 2021 20:24:30 +0100 Subject: [PATCH 3/4] handle the cache where a cache entry with the correct path has already been recreated Signed-off-by: Robin Appelman --- apps/files/lib/Command/RepairTree.php | 34 ++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/apps/files/lib/Command/RepairTree.php b/apps/files/lib/Command/RepairTree.php index d021b59722..1e4607dbe6 100644 --- a/apps/files/lib/Command/RepairTree.php +++ b/apps/files/lib/Command/RepairTree.php @@ -69,12 +69,18 @@ class RepairTree extends Command { $output->writeln("Path of file ${row['fileid']} is ${row['path']} but should be ${row['parent_path']}/${row['name']} based on it's parent", OutputInterface::VERBOSITY_VERBOSE); if ($fix) { - $query->setParameters([ - 'fileid' => $row['fileid'], - 'path' => $row['parent_path'] . '/' . $row['name'], - 'storage' => $row['parent_storage'], - ]); - $query->execute(); + $fileId = $this->getFileId($row['parent_storage'], $row['parent_path'] . '/' . $row['name']); + if ($fileId > 0) { + $output->writeln("Cache entry has already be recreated with id $fileId, deleting instead"); + $this->deleteById($row['fileid']); + } else { + $query->setParameters([ + 'fileid' => $row['fileid'], + 'path' => $row['parent_path'] . '/' . $row['name'], + 'storage' => $row['parent_storage'], + ]); + $query->execute(); + } } } @@ -85,6 +91,22 @@ class RepairTree extends Command { return 0; } + private function getFileId(int $storage, string $path) { + $query = $this->connection->getQueryBuilder(); + $query->select('fileid') + ->from('filecache') + ->where($query->expr()->eq('storage', $query->createNamedParameter($storage))) + ->andWhere($query->expr()->eq('path_hash', $query->createNamedParameter(md5($path)))); + return $query->execute()->fetch(\PDO::FETCH_COLUMN); + } + + private function deleteById(int $fileId) { + $query = $this->connection->getQueryBuilder(); + $query->delete('filecache') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId))); + $query->execute(); + } + private function findBrokenTreeBits(): array { $query = $this->connection->getQueryBuilder(); From d9f06c1792ae6ca63aa33abefb2f11bc15d9bbba Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 27 Jan 2021 18:08:10 +0100 Subject: [PATCH 4/4] cast ints Signed-off-by: Robin Appelman --- apps/files/lib/Command/RepairTree.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/files/lib/Command/RepairTree.php b/apps/files/lib/Command/RepairTree.php index 1e4607dbe6..144ce5654e 100644 --- a/apps/files/lib/Command/RepairTree.php +++ b/apps/files/lib/Command/RepairTree.php @@ -69,10 +69,10 @@ class RepairTree extends Command { $output->writeln("Path of file ${row['fileid']} is ${row['path']} but should be ${row['parent_path']}/${row['name']} based on it's parent", OutputInterface::VERBOSITY_VERBOSE); if ($fix) { - $fileId = $this->getFileId($row['parent_storage'], $row['parent_path'] . '/' . $row['name']); + $fileId = $this->getFileId((int)$row['parent_storage'], $row['parent_path'] . '/' . $row['name']); if ($fileId > 0) { $output->writeln("Cache entry has already be recreated with id $fileId, deleting instead"); - $this->deleteById($row['fileid']); + $this->deleteById((int)$row['fileid']); } else { $query->setParameters([ 'fileid' => $row['fileid'],