* @author Blaok * @author Christoph Wurst * @author Daniel Kesselberg * @author Joel S * @author Jörn Friedrich Dreyer * @author martin.mattel@diemattels.at * @author Morris Jobke * @author Robin Appelman * @author Roeland Jago Douma * @author Thomas Müller * @author Vincent Petry * * @license AGPL-3.0 * * This code is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, * as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License, version 3, * along with this program. If not, see * */ namespace OCA\Files\Command; use Doctrine\DBAL\Connection; use OC\Core\Command\Base; use OC\Core\Command\InterruptedException; use OC\ForbiddenException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\StorageNotAvailableException; use OCP\IDBConnection; use OCP\IUserManager; use Symfony\Component\Console\Helper\Table; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class Scan extends Base { /** @var IUserManager $userManager */ private $userManager; /** @var float */ protected $execTime = 0; /** @var int */ protected $foldersCounter = 0; /** @var int */ protected $filesCounter = 0; public function __construct(IUserManager $userManager) { $this->userManager = $userManager; parent::__construct(); } protected function configure() { parent::configure(); $this ->setName('files:scan') ->setDescription('rescan filesystem') ->addArgument( 'user_id', InputArgument::OPTIONAL | InputArgument::IS_ARRAY, 'will rescan all files of the given user(s)' ) ->addOption( 'path', 'p', InputArgument::OPTIONAL, 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' ) ->addOption( 'all', null, InputOption::VALUE_NONE, 'will rescan all files of all known users' )->addOption( 'unscanned', null, InputOption::VALUE_NONE, 'only scan files which are marked as not fully scanned' )->addOption( 'shallow', null, InputOption::VALUE_NONE, 'do not scan folders recursively' )->addOption( 'home-only', null, InputOption::VALUE_NONE, 'only scan the home storage, ignoring any mounted external storage or share' ); } public function checkScanWarning($fullPath, OutputInterface $output) { $normalizedPath = basename(\OC\Files\Filesystem::normalizePath($fullPath)); $path = basename($fullPath); if ($normalizedPath !== $path) { $output->writeln("\tEntry \"" . $fullPath . '" will not be accessible due to incompatible encoding'); } } protected function scanFiles($user, $path, OutputInterface $output, $backgroundScan = false, $recursive = true, $homeOnly = false) { $connection = $this->reconnectToDatabase($output); $scanner = new \OC\Files\Utils\Scanner($user, $connection, \OC::$server->query(IEventDispatcher::class), \OC::$server->getLogger()); # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) { $output->writeln("\tFile\t$path", OutputInterface::VERBOSITY_VERBOSE); ++$this->filesCounter; $this->abortIfInterrupted(); }); $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) { $output->writeln("\tFolder\t$path", OutputInterface::VERBOSITY_VERBOSE); ++$this->foldersCounter; $this->abortIfInterrupted(); }); $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output) { $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); }); $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) { $this->checkScanWarning($path, $output); }); $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) { $this->checkScanWarning($path, $output); }); try { if ($backgroundScan) { $scanner->backgroundScan($path); } else { $scanner->scan($path, $recursive, $homeOnly ? [$this, 'filterHomeMount'] : null); } } catch (ForbiddenException $e) { $output->writeln("Home storage for user $user not writable"); $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); } catch (InterruptedException $e) { # exit the function if ctrl-c has been pressed $output->writeln('Interrupted by user'); } catch (NotFoundException $e) { $output->writeln('Path not found: ' . $e->getMessage() . ''); } catch (\Exception $e) { $output->writeln('Exception during scan: ' . $e->getMessage() . ''); $output->writeln('' . $e->getTraceAsString() . ''); } } public function filterHomeMount(IMountPoint $mountPoint) { // any mountpoint inside '/$user/files/' return substr_count($mountPoint->getMountPoint(), '/') <= 3; } protected function execute(InputInterface $input, OutputInterface $output) { $inputPath = $input->getOption('path'); if ($inputPath) { $inputPath = '/' . trim($inputPath, '/'); list (, $user,) = explode('/', $inputPath, 3); $users = [$user]; } else if ($input->getOption('all')) { $users = $this->userManager->search(''); } else { $users = $input->getArgument('user_id'); } # restrict the verbosity level to VERBOSITY_VERBOSE if ($output->getVerbosity() > OutputInterface::VERBOSITY_VERBOSE) { $output->setVerbosity(OutputInterface::VERBOSITY_VERBOSE); } # check quantity of users to be process and show it on the command line $users_total = count($users); if ($users_total === 0) { $output->writeln('Please specify the user id to scan, --all to scan for all users or --path=...'); return; } $this->initTools(); $user_count = 0; foreach ($users as $user) { if (is_object($user)) { $user = $user->getUID(); } $path = $inputPath ? $inputPath : '/' . $user; ++$user_count; if ($this->userManager->userExists($user)) { $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); $this->scanFiles($user, $path, $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only')); $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); } else { $output->writeln("Unknown user $user_count $user"); $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); } try { $this->abortIfInterrupted(); } catch (InterruptedException $e) { break; } } $this->presentStats($output); } /** * Initialises some useful tools for the Command */ protected function initTools() { // Start the timer $this->execTime = -microtime(true); // Convert PHP errors to exceptions set_error_handler([$this, 'exceptionErrorHandler'], E_ALL); } /** * Processes PHP errors as exceptions in order to be able to keep track of problems * * @see https://secure.php.net/manual/en/function.set-error-handler.php * * @param int $severity the level of the error raised * @param string $message * @param string $file the filename that the error was raised in * @param int $line the line number the error was raised * * @throws \ErrorException */ public function exceptionErrorHandler($severity, $message, $file, $line) { if (!(error_reporting() & $severity)) { // This error code is not included in error_reporting return; } throw new \ErrorException($message, 0, $severity, $file, $line); } /** * @param OutputInterface $output */ protected function presentStats(OutputInterface $output) { // Stop the timer $this->execTime += microtime(true); $headers = [ 'Folders', 'Files', 'Elapsed time' ]; $this->showSummary($headers, null, $output); } /** * Shows a summary of operations * * @param string[] $headers * @param string[] $rows * @param OutputInterface $output */ protected function showSummary($headers, $rows, OutputInterface $output) { $niceDate = $this->formatExecTime(); if (!$rows) { $rows = [ $this->foldersCounter, $this->filesCounter, $niceDate, ]; } $table = new Table($output); $table ->setHeaders($headers) ->setRows([$rows]); $table->render(); } /** * Formats microtime into a human readable format * * @return string */ protected function formatExecTime() { $secs = round($this->execTime); # convert seconds into HH:MM:SS form return sprintf('%02d:%02d:%02d', ($secs/3600), ($secs/60%60), $secs%60); } /** * @return \OCP\IDBConnection */ protected function reconnectToDatabase(OutputInterface $output) { /** @var Connection | IDBConnection $connection */ $connection = \OC::$server->getDatabaseConnection(); try { $connection->close(); } catch (\Exception $ex) { $output->writeln("Error while disconnecting from database: {$ex->getMessage()}"); } while (!$connection->isConnected()) { try { $connection->connect(); } catch (\Exception $ex) { $output->writeln("Error while re-connecting to database: {$ex->getMessage()}"); sleep(60); } } return $connection; } }