Merge pull request #22135 from nextcloud/enh/noid/occ-preview-repair

Add occ preview:migrate to migrate previews from the old flat structure to a subfolder structure

core/Command/Preview/Repair.php

@@ -0,0 +1,279 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de>
+ *
+ * @author Morris Jobke <hey@morrisjobke.de>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Command\Preview;
+
+use bantu\IniGetWrapper\IniGetWrapper;
+use OC\Preview\Storage\Root;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use OCP\IConfig;
+use OCP\ILogger;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
+
+class Repair extends Command {
+	/** @var IConfig */
+	protected $config;
+	/** @var IRootFolder */
+	private $rootFolder;
+	/** @var ILogger */
+	private $logger;
+
+	/** @var bool */
+	private $stopSignalReceived = false;
+	/** @var int */
+	private $memoryLimit;
+	/** @var int */
+	private $memoryTreshold;
+
+	public function __construct(IConfig $config, IRootFolder $rootFolder, ILogger $logger, IniGetWrapper $phpIni) {
+		$this->config = $config;
+		$this->rootFolder = $rootFolder;
+		$this->logger = $logger;
+
+		$this->memoryLimit = $phpIni->getBytes('memory_limit');
+		$this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024;
+
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this
+			->setName('preview:repair')
+			->setDescription('distributes the existing previews into subfolders')
+			->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.')
+			->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		if ($this->memoryLimit !== -1) {
+			$limitInMiB = round($this->memoryLimit / 1024 /1024, 1);
+			$thresholdInMiB = round($this->memoryTreshold / 1024 /1024, 1);
+			$output->writeln("Memory limit is $limitInMiB MiB");
+			$output->writeln("Memory threshold is $thresholdInMiB MiB");
+			$output->writeln("");
+			$memoryCheckEnabled = true;
+		} else {
+			$output->writeln("No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.");
+			$output->writeln("");
+			$memoryCheckEnabled = false;
+		}
+
+		$dryMode = $input->getOption('dry');
+
+		if ($dryMode) {
+			$output->writeln("INFO: The migration is run in dry mode and will not modify anything.");
+			$output->writeln("");
+		}
+
+		$verbose = $output->isVerbose();
+
+		$instanceId = $this->config->getSystemValueString('instanceid');
+
+		$output->writeln("This will migrate all previews from the old preview location to the new one.");
+		$output->writeln('');
+
+		$output->writeln('Fetching previews that need to be migrated …');
+		/** @var \OCP\Files\Folder $currentPreviewFolder */
+		$currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview");
+
+		$directoryListing = $currentPreviewFolder->getDirectoryListing();
+
+		$total = count($directoryListing);
+		/**
+		 * by default there could be 0-9 a-f and the old-multibucket folder which are all fine
+		 */
+		if ($total < 18) {
+			$directoryListing = array_filter($directoryListing, function ($dir) {
+				if ($dir->getName() === 'old-multibucket') {
+					return false;
+				}
+
+				// a-f can't be a file ID -> removing from migration
+				if (preg_match('!^[a-f]$!', $dir->getName())) {
+					return false;
+				}
+
+				if (preg_match('!^[0-9]$!', $dir->getName())) {
+					// ignore folders that only has folders in them
+					if ($dir instanceof Folder) {
+						foreach ($dir->getDirectoryListing() as $entry) {
+							if (!$entry instanceof Folder) {
+								return true;
+							}
+						}
+						return false;
+					}
+				}
+				return true;
+			});
+			$total = count($directoryListing);
+		}
+
+		if ($total === 0) {
+			$output->writeln("All previews are already migrated.");
+			return 0;
+		}
+
+		$output->writeln("A total of $total preview files need to be migrated.");
+		$output->writeln("");
+		$output->writeln("The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This fill finish the current batch and then stop the migration. This migration can then just be started and it will continue.");
+
+		if ($input->getOption('batch')) {
+			$output->writeln('Batch mode active: migration is started right away.');
+		} else {
+			$helper = $this->getHelper('question');
+			$question = new ConfirmationQuestion('<info>Should the migration be started? (y/[n]) </info>', false);
+
+			if (!$helper->ask($input, $output, $question)) {
+				return 0;
+			}
+		}
+
+		// register the SIGINT listener late in here to be able to exit in the early process of this command
+		pcntl_signal(SIGINT, [$this, 'sigIntHandler']);
+
+		$output->writeln("");
+		$output->writeln("");
+		$section1 = $output->section();
+		$section2 = $output->section();
+		$progressBar = new ProgressBar($section2, $total);
+		$progressBar->setFormat("%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%");
+		$time = (new \DateTime())->format('H:i:s');
+		$progressBar->setMessage("$time Starting …");
+		$progressBar->maxSecondsBetweenRedraws(0.2);
+		$progressBar->start();
+
+		foreach ($directoryListing as $oldPreviewFolder) {
+			pcntl_signal_dispatch();
+			$name = $oldPreviewFolder->getName();
+			$time = (new \DateTime())->format('H:i:s');
+			$section1->writeln("$time Migrating previews of file with fileId $name …");
+			$progressBar->display();
+
+			if ($this->stopSignalReceived) {
+				$section1->writeln("$time Stopping migration …");
+				return 0;
+			}
+			if (!$oldPreviewFolder instanceof Folder) {
+				$section1->writeln("         Skipping non-folder $name …");
+				$progressBar->advance();
+				continue;
+			}
+			if ($name === 'old-multibucket') {
+				$section1->writeln("         Skipping fallback mount point $name …");
+				$progressBar->advance();
+				continue;
+			}
+			if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) {
+				$section1->writeln("         Skipping hex-digit folder $name …");
+				$progressBar->advance();
+				continue;
+			}
+			if (!preg_match('!^\d+$!', $name)) {
+				$section1->writeln("         Skipping non-numeric folder $name …");
+				$progressBar->advance();
+				continue;
+			}
+
+			$newFoldername = Root::getInternalFolder($name);
+
+			$memoryUsage = memory_get_usage();
+			if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) {
+				$section1->writeln("");
+				$section1->writeln("");
+				$section1->writeln("");
+				$section1->writeln("         Stopped process 25 MB before reaching the memory limit to avoid a hard crash.");
+				$time = (new \DateTime())->format('H:i:s');
+				$section1->writeln("$time Reached memory limit and stopped to avoid hard crash.");
+				return 1;
+			}
+
+			$previews = $oldPreviewFolder->getDirectoryListing();
+			if ($previews !== []) {
+				try {
+					$this->rootFolder->get("appdata_$instanceId/preview/$newFoldername");
+				} catch (NotFoundException $e) {
+					if ($verbose) {
+						$section1->writeln("         Create folder preview/$newFoldername");
+					}
+					if (!$dryMode) {
+						$this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername");
+					}
+				}
+
+				foreach ($previews as $preview) {
+					pcntl_signal_dispatch();
+					$previewName = $preview->getName();
+
+					if ($preview instanceof Folder) {
+						$section1->writeln("         Skipping folder $name/$previewName …");
+						$progressBar->advance();
+						continue;
+					}
+					if ($verbose) {
+						$section1->writeln("         Move preview/$name/$previewName to preview/$newFoldername");
+					}
+					if (!$dryMode) {
+						try {
+							$preview->move("appdata_$instanceId/preview/$newFoldername/$previewName");
+						} catch (\Exception $e) {
+							$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to move preview from preview/$name/$previewName to preview/$newFoldername"]);
+						}
+					}
+				}
+			}
+			if ($oldPreviewFolder->getDirectoryListing() === []) {
+				if ($verbose) {
+					$section1->writeln("         Delete empty folder preview/$name");
+				}
+				if (!$dryMode) {
+					try {
+						$oldPreviewFolder->delete();
+					} catch (\Exception $e) {
+						$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to delete empty folder preview/$name"]);
+					}
+				}
+			}
+			$section1->writeln("         Finished migrating previews of file with fileId $name …");
+			$progressBar->advance();
+		}
+
+		$progressBar->finish();
+		$output->writeln("");
+		return 0;
+	}
+
+	protected function sigIntHandler() {
+		echo "\nSignal received - will finish the step and then stop the migration.\n\n\n";
+		$this->stopSignalReceived = true;
+	}
+}

core/register_command.php

@@ -155,6 +155,8 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
 		\OC::$server->getAppManager()
 	));
 
+	$application->add(\OC::$server->query(\OC\Core\Command\Preview\Repair::class));
+
 	$application->add(new OC\Core\Command\User\Add(\OC::$server->getUserManager(), \OC::$server->getGroupManager()));
 	$application->add(new OC\Core\Command\User\Delete(\OC::$server->getUserManager()));
 	$application->add(new OC\Core\Command\User\Disable(\OC::$server->getUserManager()));

lib/composer/composer/autoload_classmap.php

@@ -830,6 +830,7 @@ return array(
     'OC\\Core\\Command\\Maintenance\\Repair' => $baseDir . '/core/Command/Maintenance/Repair.php',
     'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => $baseDir . '/core/Command/Maintenance/UpdateHtaccess.php',
     'OC\\Core\\Command\\Maintenance\\UpdateTheme' => $baseDir . '/core/Command/Maintenance/UpdateTheme.php',
+    'OC\\Core\\Command\\Preview\\Repair' => $baseDir . '/core/Command/Preview/Repair.php',
     'OC\\Core\\Command\\Security\\ImportCertificate' => $baseDir . '/core/Command/Security/ImportCertificate.php',
     'OC\\Core\\Command\\Security\\ListCertificates' => $baseDir . '/core/Command/Security/ListCertificates.php',
     'OC\\Core\\Command\\Security\\RemoveCertificate' => $baseDir . '/core/Command/Security/RemoveCertificate.php',

lib/composer/composer/autoload_static.php

@@ -859,6 +859,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Core\\Command\\Maintenance\\Repair' => __DIR__ . '/../../..' . '/core/Command/Maintenance/Repair.php',
         'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateHtaccess.php',
         'OC\\Core\\Command\\Maintenance\\UpdateTheme' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateTheme.php',
+        'OC\\Core\\Command\\Preview\\Repair' => __DIR__ . '/../../..' . '/core/Command/Preview/Repair.php',
         'OC\\Core\\Command\\Security\\ImportCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/ImportCertificate.php',
         'OC\\Core\\Command\\Security\\ListCertificates' => __DIR__ . '/../../..' . '/core/Command/Security/ListCertificates.php',
         'OC\\Core\\Command\\Security\\RemoveCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/RemoveCertificate.php',

lib/private/Preview/Storage/Root.php

@@ -41,7 +41,7 @@ class Root extends AppData {
 
 
 	public function getFolder(string $name): ISimpleFolder {
-		$internalFolder = $this->getInternalFolder($name);
+		$internalFolder = self::getInternalFolder($name);
 
 		if ($this->isMultibucketPreviewDistributionEnabled) {
 			try {
@@ -64,7 +64,7 @@ class Root extends AppData {
 	}
 
 	public function newFolder(string $name): ISimpleFolder {
-		$internalFolder = $this->getInternalFolder($name);
+		$internalFolder = self::getInternalFolder($name);
 		return parent::newFolder($internalFolder);
 	}
 
@@ -76,7 +76,7 @@ class Root extends AppData {
 		return [];
 	}
 
-	private function getInternalFolder(string $name): string {
+	public static function getInternalFolder(string $name): string {
 		return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
 	}
 }

tests/Core/Command/Preview/RepairTest.php

@@ -0,0 +1,141 @@
+<?php
+
+namespace Tests\Core\Command\Preview;
+
+use bantu\IniGetWrapper\IniGetWrapper;
+use OC\Core\Command\Preview\Repair;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\Node;
+use OCP\IConfig;
+use OCP\ILogger;
+use PHPUnit\Framework\MockObject\MockObject;
+use Symfony\Component\Console\Formatter\OutputFormatterInterface;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Test\TestCase;
+
+class RepairTest extends TestCase {
+	/** @var IConfig|MockObject */
+	private $config;
+	/** @var IRootFolder|MockObject */
+	private $rootFolder;
+	/** @var ILogger|MockObject */
+	private $logger;
+	/** @var IniGetWrapper|MockObject */
+	private $iniGetWrapper;
+	/** @var InputInterface|MockObject */
+	private $input;
+	/** @var OutputInterface|MockObject */
+	private $output;
+	/** @var string */
+	private $outputLines = '';
+	/** @var Repair */
+	private $repair;
+
+	protected function setUp(): void {
+		parent::setUp();
+		$this->config = $this->getMockBuilder(IConfig::class)
+			->getMock();
+		$this->rootFolder = $this->getMockBuilder(IRootFolder::class)
+			->getMock();
+		$this->logger = $this->getMockBuilder(ILogger::class)
+			->getMock();
+		$this->iniGetWrapper = $this->getMockBuilder(IniGetWrapper::class)
+			->getMock();
+		$this->repair = new Repair($this->config, $this->rootFolder, $this->logger, $this->iniGetWrapper);
+		$this->input = $this->getMockBuilder(InputInterface::class)
+			->getMock();
+		$this->input->expects($this->any())
+			->method('getOption')
+			->willReturnCallback(function ($parameter) {
+				if ($parameter === 'batch') {
+					return true;
+				}
+				return null;
+			});
+		$this->output = $this->getMockBuilder(OutputInterface::class)
+			->setMethods(['section', 'writeln', 'write', 'setVerbosity', 'getVerbosity', 'isQuiet', 'isVerbose', 'isVeryVerbose', 'isDebug', 'setDecorated', 'isDecorated', 'setFormatter', 'getFormatter'])
+			->getMock();
+		$self = $this;
+		$this->output->expects($this->any())
+			->method('section')
+			->willReturn($this->output);
+		$this->output->expects($this->any())
+			->method('getFormatter')
+			->willReturn($this->getMockBuilder(OutputFormatterInterface::class)->getMock());
+		$this->output->expects($this->any())
+			->method('writeln')
+			->willReturnCallback(function ($line) use ($self) {
+				$self->outputLines .= $line . "\n";
+			});
+	}
+
+	public function emptyTestDataProvider() {
+		/** directoryNames, expectedOutput */
+		return [
+			[
+				[],
+				'All previews are already migrated.'
+			],
+			[
+				[['name' => 'a'], ['name' => 'b'], ['name' => 'c']],
+				'All previews are already migrated.'
+			],
+			[
+				[['name' => '0', 'content' => ['folder', 'folder']], ['name' => 'b'], ['name' => 'c']],
+				'All previews are already migrated.'
+			],
+			[
+				[['name' => '0', 'content' => ['file', 'folder', 'folder']], ['name' => 'b'], ['name' => 'c']],
+				'A total of 1 preview files need to be migrated.'
+			],
+			[
+				[['name' => '23'], ['name' => 'b'], ['name' => 'c']],
+				'A total of 1 preview files need to be migrated.'
+			],
+		];
+	}
+
+	/**
+	 * @dataProvider emptyTestDataProvider
+	 */
+	public function testEmptyExecute($directoryNames, $expectedOutput) {
+		$previewFolder = $this->getMockBuilder(Folder::class)
+			->getMock();
+		$directories = array_map(function ($element) {
+			$dir = $this->getMockBuilder(Folder::class)
+				->getMock();
+			$dir->expects($this->any())
+				->method('getName')
+				->willReturn($element['name']);
+			if (isset($element['content'])) {
+				$list = [];
+				foreach ($element['content'] as $item) {
+					if ($item === 'file') {
+						$list[] = $this->getMockBuilder(Node::class)
+							->getMock();
+					} elseif ($item === 'folder') {
+						$list[] = $this->getMockBuilder(Folder::class)
+							->getMock();
+					}
+				}
+				$dir->expects($this->once())
+					->method('getDirectoryListing')
+					->willReturn($list);
+			}
+			return $dir;
+		}, $directoryNames);
+		$previewFolder->expects($this->once())
+			->method('getDirectoryListing')
+			->willReturn($directories);
+		$this->rootFolder->expects($this->at(0))
+			->method('get')
+			->with("appdata_/preview")
+			->willReturn($previewFolder);
+
+		$this->repair->run($this->input, $this->output);
+
+		$this->assertStringContainsString($expectedOutput, $this->outputLines);
+	}
+}