Merge pull request #20545 from owncloud/scan-storage-in-background-job
Move files/ajax/scan.php to background job
This commit is contained in:
commit
2ceae43989
|
@ -1,94 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* @author Bart Visscher <bartv@thisnet.nl>
|
|
||||||
* @author Jörn Friedrich Dreyer <jfd@butonic.de>
|
|
||||||
* @author Lukas Reschke <lukas@owncloud.com>
|
|
||||||
* @author Robin Appelman <icewind@owncloud.com>
|
|
||||||
* @author Vincent Petry <pvince81@owncloud.com>
|
|
||||||
*
|
|
||||||
* @copyright Copyright (c) 2015, ownCloud, Inc.
|
|
||||||
* @license AGPL-3.0
|
|
||||||
*
|
|
||||||
* This code is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License, version 3,
|
|
||||||
* as published by the Free Software Foundation.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License, version 3,
|
|
||||||
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
set_time_limit(0); //scanning can take ages
|
|
||||||
|
|
||||||
\OCP\JSON::checkLoggedIn();
|
|
||||||
\OCP\JSON::callCheck();
|
|
||||||
|
|
||||||
\OC::$server->getSession()->close();
|
|
||||||
|
|
||||||
$force = (isset($_GET['force']) and ($_GET['force'] === 'true'));
|
|
||||||
$dir = isset($_GET['dir']) ? (string)$_GET['dir'] : '';
|
|
||||||
if (isset($_GET['users'])) {
|
|
||||||
\OCP\JSON::checkAdminUser();
|
|
||||||
if ($_GET['users'] === 'all') {
|
|
||||||
$users = OC_User::getUsers();
|
|
||||||
} else {
|
|
||||||
$users = json_decode($_GET['users']);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$users = array(OC_User::getUser());
|
|
||||||
}
|
|
||||||
|
|
||||||
$eventSource = \OC::$server->createEventSource();
|
|
||||||
$listener = new ScanListener($eventSource);
|
|
||||||
|
|
||||||
foreach ($users as $user) {
|
|
||||||
$eventSource->send('user', $user);
|
|
||||||
$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger());
|
|
||||||
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file'));
|
|
||||||
try {
|
|
||||||
if ($force) {
|
|
||||||
$scanner->scan($dir);
|
|
||||||
} else {
|
|
||||||
$scanner->backgroundScan($dir);
|
|
||||||
}
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
$eventSource->send('error', get_class($e) . ': ' . $e->getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$eventSource->send('done', $listener->getCount());
|
|
||||||
$eventSource->close();
|
|
||||||
|
|
||||||
class ScanListener {
|
|
||||||
|
|
||||||
private $fileCount = 0;
|
|
||||||
private $lastCount = 0;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @var \OCP\IEventSource event source to pass events to
|
|
||||||
*/
|
|
||||||
private $eventSource;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param \OCP\IEventSource $eventSource
|
|
||||||
*/
|
|
||||||
public function __construct($eventSource) {
|
|
||||||
$this->eventSource = $eventSource;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function file() {
|
|
||||||
$this->fileCount++;
|
|
||||||
if ($this->fileCount > $this->lastCount + 20) { //send a count update every 20 files
|
|
||||||
$this->lastCount = $this->fileCount;
|
|
||||||
$this->eventSource->send('count', $this->fileCount);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getCount() {
|
|
||||||
return $this->fileCount;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -8,7 +8,7 @@
|
||||||
<shipped>true</shipped>
|
<shipped>true</shipped>
|
||||||
<standalone/>
|
<standalone/>
|
||||||
<default_enable/>
|
<default_enable/>
|
||||||
<version>1.4.0</version>
|
<version>1.4.1</version>
|
||||||
<types>
|
<types>
|
||||||
<filesystem/>
|
<filesystem/>
|
||||||
</types>
|
</types>
|
||||||
|
|
|
@ -0,0 +1,26 @@
|
||||||
|
<?php
|
||||||
|
/**
|
||||||
|
* @author Lukas Reschke <lukas@owncloud.com>
|
||||||
|
*
|
||||||
|
* @copyright Copyright (c) 2015, ownCloud, Inc.
|
||||||
|
* @license AGPL-3.0
|
||||||
|
*
|
||||||
|
* This code is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License, version 3,
|
||||||
|
* as published by the Free Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License, version 3,
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Cron job for scanning user storages
|
||||||
|
$jobList = \OC::$server->getJobList();
|
||||||
|
$job = 'OCA\Files\BackgroundJob\ScanFiles';
|
||||||
|
\OC::$server->getJobList()->add($job);
|
||||||
|
|
|
@ -75,8 +75,6 @@ $this->create('files_ajax_newfolder', 'ajax/newfolder.php')
|
||||||
->actionInclude('files/ajax/newfolder.php');
|
->actionInclude('files/ajax/newfolder.php');
|
||||||
$this->create('files_ajax_rename', 'ajax/rename.php')
|
$this->create('files_ajax_rename', 'ajax/rename.php')
|
||||||
->actionInclude('files/ajax/rename.php');
|
->actionInclude('files/ajax/rename.php');
|
||||||
$this->create('files_ajax_scan', 'ajax/scan.php')
|
|
||||||
->actionInclude('files/ajax/scan.php');
|
|
||||||
$this->create('files_ajax_upload', 'ajax/upload.php')
|
$this->create('files_ajax_upload', 'ajax/upload.php')
|
||||||
->actionInclude('files/ajax/upload.php');
|
->actionInclude('files/ajax/upload.php');
|
||||||
|
|
||||||
|
|
|
@ -96,6 +96,11 @@ if ($installedVersion === '1.1.9' && (
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add cron job for scanning user storages
|
||||||
|
$jobList = \OC::$server->getJobList();
|
||||||
|
$job = 'OCA\Files\BackgroundJob\ScanFiles';
|
||||||
|
\OC::$server->getJobList()->add($job);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* migrate old constant DEBUG to new config value 'debug'
|
* migrate old constant DEBUG to new config value 'debug'
|
||||||
*
|
*
|
||||||
|
|
|
@ -243,9 +243,6 @@
|
||||||
e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone
|
e.preventDefault(); // prevent browser from doing anything, if file isn't dropped in dropZone
|
||||||
});
|
});
|
||||||
|
|
||||||
//do a background scan if needed
|
|
||||||
scanFiles();
|
|
||||||
|
|
||||||
// display storage warnings
|
// display storage warnings
|
||||||
setTimeout(Files.displayStorageWarnings, 100);
|
setTimeout(Files.displayStorageWarnings, 100);
|
||||||
|
|
||||||
|
@ -323,51 +320,6 @@
|
||||||
OCA.Files.Files = Files;
|
OCA.Files.Files = Files;
|
||||||
})();
|
})();
|
||||||
|
|
||||||
function scanFiles(force, dir, users) {
|
|
||||||
if (!OC.currentUser) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!dir) {
|
|
||||||
dir = '';
|
|
||||||
}
|
|
||||||
force = !!force; //cast to bool
|
|
||||||
scanFiles.scanning = true;
|
|
||||||
var scannerEventSource;
|
|
||||||
if (users) {
|
|
||||||
var usersString;
|
|
||||||
if (users === 'all') {
|
|
||||||
usersString = users;
|
|
||||||
} else {
|
|
||||||
usersString = JSON.stringify(users);
|
|
||||||
}
|
|
||||||
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir, users: usersString});
|
|
||||||
} else {
|
|
||||||
scannerEventSource = new OC.EventSource(OC.filePath('files','ajax','scan.php'),{force: force,dir: dir});
|
|
||||||
}
|
|
||||||
scanFiles.cancel = scannerEventSource.close.bind(scannerEventSource);
|
|
||||||
scannerEventSource.listen('count',function(count) {
|
|
||||||
console.log(count + ' files scanned');
|
|
||||||
});
|
|
||||||
scannerEventSource.listen('folder',function(path) {
|
|
||||||
console.log('now scanning ' + path);
|
|
||||||
});
|
|
||||||
scannerEventSource.listen('error',function(message) {
|
|
||||||
console.error('Scanner error: ', message);
|
|
||||||
});
|
|
||||||
scannerEventSource.listen('done',function(count) {
|
|
||||||
scanFiles.scanning=false;
|
|
||||||
console.log('done after ' + count + ' files');
|
|
||||||
if (OCA.Files.App) {
|
|
||||||
OCA.Files.App.fileList.updateStorageStatistics(true);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
scannerEventSource.listen('user',function(user) {
|
|
||||||
console.log('scanning files for ' + user);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
scanFiles.scanning=false;
|
|
||||||
|
|
||||||
// TODO: move to FileList
|
// TODO: move to FileList
|
||||||
var createDragShadow = function(event) {
|
var createDragShadow = function(event) {
|
||||||
// FIXME: inject file list instance somehow
|
// FIXME: inject file list instance somehow
|
||||||
|
|
|
@ -0,0 +1,114 @@
|
||||||
|
<?php
|
||||||
|
/**
|
||||||
|
* @author Lukas Reschke <lukas@owncloud.com>
|
||||||
|
*
|
||||||
|
* @copyright Copyright (c) 2015, ownCloud, Inc.
|
||||||
|
* @license AGPL-3.0
|
||||||
|
*
|
||||||
|
* This code is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License, version 3,
|
||||||
|
* as published by the Free Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License, version 3,
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
namespace OCA\Files\BackgroundJob;
|
||||||
|
|
||||||
|
use OC\Files\Utils\Scanner;
|
||||||
|
use OCP\IConfig;
|
||||||
|
use OCP\IDBConnection;
|
||||||
|
use OCP\ILogger;
|
||||||
|
use OCP\IUser;
|
||||||
|
use OCP\IUserManager;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class ScanFiles is a background job used to run the file scanner over the user
|
||||||
|
* accounts to ensure integrity of the file cache.
|
||||||
|
*
|
||||||
|
* @package OCA\Files\BackgroundJob
|
||||||
|
*/
|
||||||
|
class ScanFiles extends \OC\BackgroundJob\TimedJob {
|
||||||
|
/** @var IConfig */
|
||||||
|
private $config;
|
||||||
|
/** @var IUserManager */
|
||||||
|
private $userManager;
|
||||||
|
/** @var IDBConnection */
|
||||||
|
private $dbConnection;
|
||||||
|
/** @var ILogger */
|
||||||
|
private $logger;
|
||||||
|
/** Amount of users that should get scanned per execution */
|
||||||
|
const USERS_PER_SESSION = 500;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param IConfig|null $config
|
||||||
|
* @param IUserManager|null $userManager
|
||||||
|
* @param IDBConnection|null $dbConnection
|
||||||
|
* @param ILogger|null $logger
|
||||||
|
*/
|
||||||
|
public function __construct(IConfig $config = null,
|
||||||
|
IUserManager $userManager = null,
|
||||||
|
IDBConnection $dbConnection = null,
|
||||||
|
ILogger $logger = null) {
|
||||||
|
// Run once per 10 minutes
|
||||||
|
$this->setInterval(60 * 10);
|
||||||
|
|
||||||
|
if (is_null($userManager) || is_null($config)) {
|
||||||
|
$this->fixDIForJobs();
|
||||||
|
} else {
|
||||||
|
$this->config = $config;
|
||||||
|
$this->userManager = $userManager;
|
||||||
|
$this->logger = $logger;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function fixDIForJobs() {
|
||||||
|
$this->config = \OC::$server->getConfig();
|
||||||
|
$this->userManager = \OC::$server->getUserManager();
|
||||||
|
$this->logger = \OC::$server->getLogger();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param IUser $user
|
||||||
|
*/
|
||||||
|
protected function runScanner(IUser $user) {
|
||||||
|
try {
|
||||||
|
$scanner = new Scanner(
|
||||||
|
$user->getUID(),
|
||||||
|
$this->dbConnection,
|
||||||
|
$this->logger
|
||||||
|
);
|
||||||
|
$scanner->backgroundScan('');
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
$this->logger->logException($e, ['app' => 'files']);
|
||||||
|
}
|
||||||
|
\OC_Util::tearDownFS();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param $argument
|
||||||
|
* @throws \Exception
|
||||||
|
*/
|
||||||
|
protected function run($argument) {
|
||||||
|
$offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0);
|
||||||
|
$users = $this->userManager->search('', self::USERS_PER_SESSION, $offset);
|
||||||
|
if (!count($users)) {
|
||||||
|
// No users found, reset offset and retry
|
||||||
|
$offset = 0;
|
||||||
|
$users = $this->userManager->search('', self::USERS_PER_SESSION);
|
||||||
|
}
|
||||||
|
|
||||||
|
$offset += self::USERS_PER_SESSION;
|
||||||
|
$this->config->setAppValue('files', 'cronjob_scan_files', $offset);
|
||||||
|
|
||||||
|
foreach ($users as $user) {
|
||||||
|
$this->runScanner($user);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,134 @@
|
||||||
|
<?php
|
||||||
|
/**
|
||||||
|
* @author Lukas Reschke <lukas@owncloud.com>
|
||||||
|
*
|
||||||
|
* @copyright Copyright (c) 2015, ownCloud, Inc.
|
||||||
|
* @license AGPL-3.0
|
||||||
|
*
|
||||||
|
* This code is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License, version 3,
|
||||||
|
* as published by the Free Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License, version 3,
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
namespace OCA\Files\Tests\BackgroundJob;
|
||||||
|
|
||||||
|
use Test\TestCase;
|
||||||
|
use OCP\IConfig;
|
||||||
|
use OCP\IUserManager;
|
||||||
|
use OCA\Files\BackgroundJob\ScanFiles;
|
||||||
|
use OCP\ILogger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class ScanFilesTest
|
||||||
|
*
|
||||||
|
* @package OCA\Files\Tests\BackgroundJob
|
||||||
|
*/
|
||||||
|
class ScanFilesTest extends TestCase {
|
||||||
|
/** @var IConfig */
|
||||||
|
private $config;
|
||||||
|
/** @var IUserManager */
|
||||||
|
private $userManager;
|
||||||
|
/** @var ScanFiles */
|
||||||
|
private $scanFiles;
|
||||||
|
|
||||||
|
public function setUp() {
|
||||||
|
parent::setUp();
|
||||||
|
|
||||||
|
$this->config = $this->getMock('\OCP\IConfig');
|
||||||
|
$this->userManager = $this->getMock('\OCP\IUserManager');
|
||||||
|
|
||||||
|
$this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles')
|
||||||
|
->setConstructorArgs([
|
||||||
|
$this->config,
|
||||||
|
$this->userManager,
|
||||||
|
])
|
||||||
|
->setMethods(['runScanner'])
|
||||||
|
->getMock();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRunWithoutUsers() {
|
||||||
|
$this->config
|
||||||
|
->expects($this->at(0))
|
||||||
|
->method('getAppValue')
|
||||||
|
->with('files', 'cronjob_scan_files', 0)
|
||||||
|
->will($this->returnValue(50));
|
||||||
|
$this->userManager
|
||||||
|
->expects($this->at(0))
|
||||||
|
->method('search')
|
||||||
|
->with('', 500, 50)
|
||||||
|
->will($this->returnValue([]));
|
||||||
|
$this->userManager
|
||||||
|
->expects($this->at(1))
|
||||||
|
->method('search')
|
||||||
|
->with('', 500)
|
||||||
|
->will($this->returnValue([]));
|
||||||
|
$this->config
|
||||||
|
->expects($this->at(1))
|
||||||
|
->method('setAppValue')
|
||||||
|
->with('files', 'cronjob_scan_files', 500);
|
||||||
|
|
||||||
|
$this->invokePrivate($this->scanFiles, 'run', [[]]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRunWithUsers() {
|
||||||
|
$fakeUser = $this->getMock('\OCP\IUser');
|
||||||
|
$this->config
|
||||||
|
->expects($this->at(0))
|
||||||
|
->method('getAppValue')
|
||||||
|
->with('files', 'cronjob_scan_files', 0)
|
||||||
|
->will($this->returnValue(50));
|
||||||
|
$this->userManager
|
||||||
|
->expects($this->at(0))
|
||||||
|
->method('search')
|
||||||
|
->with('', 500, 50)
|
||||||
|
->will($this->returnValue([
|
||||||
|
$fakeUser
|
||||||
|
]));
|
||||||
|
$this->config
|
||||||
|
->expects($this->at(1))
|
||||||
|
->method('setAppValue')
|
||||||
|
->with('files', 'cronjob_scan_files', 550);
|
||||||
|
$this->scanFiles
|
||||||
|
->expects($this->once())
|
||||||
|
->method('runScanner')
|
||||||
|
->with($fakeUser);
|
||||||
|
|
||||||
|
$this->invokePrivate($this->scanFiles, 'run', [[]]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testRunWithUsersAndOffsetAtEndOfUserList() {
|
||||||
|
$this->config
|
||||||
|
->expects($this->at(0))
|
||||||
|
->method('getAppValue')
|
||||||
|
->with('files', 'cronjob_scan_files', 0)
|
||||||
|
->will($this->returnValue(50));
|
||||||
|
$this->userManager
|
||||||
|
->expects($this->at(0))
|
||||||
|
->method('search')
|
||||||
|
->with('', 500, 50)
|
||||||
|
->will($this->returnValue([]));
|
||||||
|
$this->userManager
|
||||||
|
->expects($this->at(1))
|
||||||
|
->method('search')
|
||||||
|
->with('', 500)
|
||||||
|
->will($this->returnValue([]));
|
||||||
|
$this->config
|
||||||
|
->expects($this->at(1))
|
||||||
|
->method('setAppValue')
|
||||||
|
->with('files', 'cronjob_scan_files', 500);
|
||||||
|
$this->scanFiles
|
||||||
|
->expects($this->never())
|
||||||
|
->method('runScanner');
|
||||||
|
|
||||||
|
$this->invokePrivate($this->scanFiles, 'run', [[]]);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -36,7 +36,6 @@ interface IJobList {
|
||||||
*
|
*
|
||||||
* @param \OCP\BackgroundJob\IJob|string $job
|
* @param \OCP\BackgroundJob\IJob|string $job
|
||||||
* @param mixed $argument The argument to be passed to $job->run() when the job is exectured
|
* @param mixed $argument The argument to be passed to $job->run() when the job is exectured
|
||||||
* @param string $job
|
|
||||||
* @return void
|
* @return void
|
||||||
* @since 7.0.0
|
* @since 7.0.0
|
||||||
*/
|
*/
|
||||||
|
|
Loading…
Reference in New Issue