From 391bc49dabb39e31739c849b83bb490cabf9da3d Mon Sep 17 00:00:00 2001 From: Lukas Reschke Date: Mon, 16 Nov 2015 22:23:16 +0100 Subject: [PATCH] Move files/ajax/scan.php to background job The background job will now be executed in chunks of 500 users all 10 minutes. --- apps/files/ajax/scan.php | 94 ------------ apps/files/appinfo/info.xml | 2 +- apps/files/appinfo/install.php | 26 ++++ apps/files/appinfo/routes.php | 2 - apps/files/appinfo/update.php | 5 + apps/files/js/files.js | 48 ------- apps/files/lib/backgroundjob/scanfiles.php | 114 +++++++++++++++ .../tests/backgroundjob/ScanFilesTest.php | 134 ++++++++++++++++++ lib/public/backgroundjob/ijoblist.php | 1 - 9 files changed, 280 insertions(+), 146 deletions(-) delete mode 100644 apps/files/ajax/scan.php create mode 100644 apps/files/appinfo/install.php create mode 100644 apps/files/lib/backgroundjob/scanfiles.php create mode 100644 apps/files/tests/backgroundjob/ScanFilesTest.php diff --git a/apps/files/ajax/scan.php b/apps/files/ajax/scan.php deleted file mode 100644 index 7710a28a8c..0000000000 --- a/apps/files/ajax/scan.php +++ /dev/null @@ -1,94 +0,0 @@ - - * @author Jörn Friedrich Dreyer - * @author Lukas Reschke - * @author Robin Appelman - * @author Vincent Petry - * - * @copyright Copyright (c) 2015, ownCloud, Inc. - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ -set_time_limit(0); //scanning can take ages - -\OCP\JSON::checkLoggedIn(); -\OCP\JSON::callCheck(); - -\OC::$server->getSession()->close(); - -$force = (isset($_GET['force']) and ($_GET['force'] === 'true')); -$dir = isset($_GET['dir']) ? (string)$_GET['dir'] : ''; -if (isset($_GET['users'])) { - \OCP\JSON::checkAdminUser(); - if ($_GET['users'] === 'all') { - $users = OC_User::getUsers(); - } else { - $users = json_decode($_GET['users']); - } -} else { - $users = array(OC_User::getUser()); -} - -$eventSource = \OC::$server->createEventSource(); -$listener = new ScanListener($eventSource); - -foreach ($users as $user) { - $eventSource->send('user', $user); - $scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger()); - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file')); - try { - if ($force) { - $scanner->scan($dir); - } else { - $scanner->backgroundScan($dir); - } - } catch (\Exception $e) { - $eventSource->send('error', get_class($e) . ': ' . $e->getMessage()); - } -} - -$eventSource->send('done', $listener->getCount()); -$eventSource->close(); - -class ScanListener { - - private $fileCount = 0; - private $lastCount = 0; - - /** - * @var \OCP\IEventSource event source to pass events to - */ - private $eventSource; - - /** - * @param \OCP\IEventSource $eventSource - */ - public function __construct($eventSource) { - $this->eventSource = $eventSource; - } - - public function file() { - $this->fileCount++; - if ($this->fileCount > $this->lastCount + 20) { //send a count update every 20 files - $this->lastCount = $this->fileCount; - $this->eventSource->send('count', $this->fileCount); - } - } - - public function getCount() { - return $this->fileCount; - } -} diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index 4ab226f396..df12b87397 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -8,7 +8,7 @@ true - 1.4.0 + 1.4.1 diff --git a/apps/files/appinfo/install.php b/apps/files/appinfo/install.php new file mode 100644 index 0000000000..b47bf6ac4b --- /dev/null +++ b/apps/files/appinfo/install.php @@ -0,0 +1,26 @@ + + * + * @copyright Copyright (c) 2015, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +// Cron job for scanning user storages +$jobList = \OC::$server->getJobList(); +$job = 'OCA\Files\BackgroundJob\ScanFiles'; +\OC::$server->getJobList()->add($job); + diff --git a/apps/files/appinfo/routes.php b/apps/files/appinfo/routes.php index 2bb913c30a..844b73b3c4 100644 --- a/apps/files/appinfo/routes.php +++ b/apps/files/appinfo/routes.php @@ -75,8 +75,6 @@ $this->create('files_ajax_newfolder', 'ajax/newfolder.php') ->actionInclude('files/ajax/newfolder.php'); $this->create('files_ajax_rename', 'ajax/rename.php') ->actionInclude('files/ajax/rename.php'); -$this->create('files_ajax_scan', 'ajax/scan.php') - ->actionInclude('files/ajax/scan.php'); $this->create('files_ajax_upload', 'ajax/upload.php') ->actionInclude('files/ajax/upload.php'); diff --git a/apps/files/appinfo/update.php b/apps/files/appinfo/update.php index 6084435fa5..d181dff560 100644 --- a/apps/files/appinfo/update.php +++ b/apps/files/appinfo/update.php @@ -96,6 +96,11 @@ if ($installedVersion === '1.1.9' && ( } } +// Add cron job for scanning user storages +$jobList = \OC::$server->getJobList(); +$job = 'OCA\Files\BackgroundJob\ScanFiles'; +\OC::$server->getJobList()->add($job); + /** * migrate old constant DEBUG to new config value 'debug' * diff --git a/apps/files/js/files.js b/apps/files/js/files.js index e33b835443..6bf4a4cfe5 100644 --- a/apps/files/js/files.js +++ b/apps/files/js/files.js @@ -243,9 +243,6 @@ e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone }); - //do a background scan if needed - scanFiles(); - // display storage warnings setTimeout(Files.displayStorageWarnings, 100); @@ -323,51 +320,6 @@ OCA.Files.Files = Files; })(); -function scanFiles(force, dir, users) { - if (!OC.currentUser) { - return; - } - - if (!dir) { - dir = ''; - } - force = !!force; //cast to bool - scanFiles.scanning = true; - var scannerEventSource; - if (users) { - var usersString; - if (users === 'all') { - usersString = users; - } else { - usersString = JSON.stringify(users); - } - scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir, users: usersString}); - } else { - scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir}); - } - scanFiles.cancel = scannerEventSource.close.bind(scannerEventSource); - scannerEventSource.listen('count',function(count) { - console.log(count + ' files scanned'); - }); - scannerEventSource.listen('folder',function(path) { - console.log('now scanning ' + path); - }); - scannerEventSource.listen('error',function(message) { - console.error('Scanner error: ', message); - }); - scannerEventSource.listen('done',function(count) { - scanFiles.scanning=false; - console.log('done after ' + count + ' files'); - if (OCA.Files.App) { - OCA.Files.App.fileList.updateStorageStatistics(true); - } - }); - scannerEventSource.listen('user',function(user) { - console.log('scanning files for ' + user); - }); -} -scanFiles.scanning=false; - // TODO: move to FileList var createDragShadow = function(event) { // FIXME: inject file list instance somehow diff --git a/apps/files/lib/backgroundjob/scanfiles.php b/apps/files/lib/backgroundjob/scanfiles.php new file mode 100644 index 0000000000..7371429a26 --- /dev/null +++ b/apps/files/lib/backgroundjob/scanfiles.php @@ -0,0 +1,114 @@ + + * + * @copyright Copyright (c) 2015, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\Files\BackgroundJob; + +use OC\Files\Utils\Scanner; +use OCP\IConfig; +use OCP\IDBConnection; +use OCP\ILogger; +use OCP\IUser; +use OCP\IUserManager; + +/** + * Class ScanFiles is a background job used to run the file scanner over the user + * accounts to ensure integrity of the file cache. + * + * @package OCA\Files\BackgroundJob + */ +class ScanFiles extends \OC\BackgroundJob\TimedJob { + /** @var IConfig */ + private $config; + /** @var IUserManager */ + private $userManager; + /** @var IDBConnection */ + private $dbConnection; + /** @var ILogger */ + private $logger; + /** Amount of users that should get scanned per execution */ + const USERS_PER_SESSION = 500; + + /** + * @param IConfig|null $config + * @param IUserManager|null $userManager + * @param IDBConnection|null $dbConnection + * @param ILogger|null $logger + */ + public function __construct(IConfig $config = null, + IUserManager $userManager = null, + IDBConnection $dbConnection = null, + ILogger $logger = null) { + // Run once per 10 minutes + $this->setInterval(60 * 10); + + if (is_null($userManager) || is_null($config)) { + $this->fixDIForJobs(); + } else { + $this->config = $config; + $this->userManager = $userManager; + $this->logger = $logger; + } + } + + protected function fixDIForJobs() { + $this->config = \OC::$server->getConfig(); + $this->userManager = \OC::$server->getUserManager(); + $this->logger = \OC::$server->getLogger(); + } + + /** + * @param IUser $user + */ + protected function runScanner(IUser $user) { + try { + $scanner = new Scanner( + $user->getUID(), + $this->dbConnection, + $this->logger + ); + $scanner->backgroundScan(''); + } catch (\Exception $e) { + $this->logger->logException($e, ['app' => 'files']); + } + \OC_Util::tearDownFS(); + } + + /** + * @param $argument + * @throws \Exception + */ + protected function run($argument) { + $offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0); + $users = $this->userManager->search('', self::USERS_PER_SESSION, $offset); + if (!count($users)) { + // No users found, reset offset and retry + $offset = 0; + $users = $this->userManager->search('', self::USERS_PER_SESSION); + } + + $offset += self::USERS_PER_SESSION; + $this->config->setAppValue('files', 'cronjob_scan_files', $offset); + + foreach ($users as $user) { + $this->runScanner($user); + } + } +} diff --git a/apps/files/tests/backgroundjob/ScanFilesTest.php b/apps/files/tests/backgroundjob/ScanFilesTest.php new file mode 100644 index 0000000000..907cad64f9 --- /dev/null +++ b/apps/files/tests/backgroundjob/ScanFilesTest.php @@ -0,0 +1,134 @@ + + * + * @copyright Copyright (c) 2015, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ +namespace OCA\Files\Tests\BackgroundJob; + +use Test\TestCase; +use OCP\IConfig; +use OCP\IUserManager; +use OCA\Files\BackgroundJob\ScanFiles; +use OCP\ILogger; + +/** + * Class ScanFilesTest + * + * @package OCA\Files\Tests\BackgroundJob + */ +class ScanFilesTest extends TestCase { + /** @var IConfig */ + private $config; + /** @var IUserManager */ + private $userManager; + /** @var ScanFiles */ + private $scanFiles; + + public function setUp() { + parent::setUp(); + + $this->config = $this->getMock('\OCP\IConfig'); + $this->userManager = $this->getMock('\OCP\IUserManager'); + + $this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles') + ->setConstructorArgs([ + $this->config, + $this->userManager, + ]) + ->setMethods(['runScanner']) + ->getMock(); + } + + public function testRunWithoutUsers() { + $this->config + ->expects($this->at(0)) + ->method('getAppValue') + ->with('files', 'cronjob_scan_files', 0) + ->will($this->returnValue(50)); + $this->userManager + ->expects($this->at(0)) + ->method('search') + ->with('', 500, 50) + ->will($this->returnValue([])); + $this->userManager + ->expects($this->at(1)) + ->method('search') + ->with('', 500) + ->will($this->returnValue([])); + $this->config + ->expects($this->at(1)) + ->method('setAppValue') + ->with('files', 'cronjob_scan_files', 500); + + $this->invokePrivate($this->scanFiles, 'run', [[]]); + } + + public function testRunWithUsers() { + $fakeUser = $this->getMock('\OCP\IUser'); + $this->config + ->expects($this->at(0)) + ->method('getAppValue') + ->with('files', 'cronjob_scan_files', 0) + ->will($this->returnValue(50)); + $this->userManager + ->expects($this->at(0)) + ->method('search') + ->with('', 500, 50) + ->will($this->returnValue([ + $fakeUser + ])); + $this->config + ->expects($this->at(1)) + ->method('setAppValue') + ->with('files', 'cronjob_scan_files', 550); + $this->scanFiles + ->expects($this->once()) + ->method('runScanner') + ->with($fakeUser); + + $this->invokePrivate($this->scanFiles, 'run', [[]]); + } + + public function testRunWithUsersAndOffsetAtEndOfUserList() { + $this->config + ->expects($this->at(0)) + ->method('getAppValue') + ->with('files', 'cronjob_scan_files', 0) + ->will($this->returnValue(50)); + $this->userManager + ->expects($this->at(0)) + ->method('search') + ->with('', 500, 50) + ->will($this->returnValue([])); + $this->userManager + ->expects($this->at(1)) + ->method('search') + ->with('', 500) + ->will($this->returnValue([])); + $this->config + ->expects($this->at(1)) + ->method('setAppValue') + ->with('files', 'cronjob_scan_files', 500); + $this->scanFiles + ->expects($this->never()) + ->method('runScanner'); + + $this->invokePrivate($this->scanFiles, 'run', [[]]); + } + +} diff --git a/lib/public/backgroundjob/ijoblist.php b/lib/public/backgroundjob/ijoblist.php index e2dc348e54..384f8b3d80 100644 --- a/lib/public/backgroundjob/ijoblist.php +++ b/lib/public/backgroundjob/ijoblist.php @@ -36,7 +36,6 @@ interface IJobList { * * @param \OCP\BackgroundJob\IJob|string $job * @param mixed $argument The argument to be passed to $job->run() when the job is exectured - * @param string $job * @return void * @since 7.0.0 */