Merge pull request #10922 from owncloud/explicit-scan-transactions
Use bigger transactions when doing explicit file system scans
This commit is contained in:
commit
c3d90b96c8
|
@ -20,7 +20,7 @@ $listener = new ScanListener($eventSource);
|
||||||
|
|
||||||
foreach ($users as $user) {
|
foreach ($users as $user) {
|
||||||
$eventSource->send('user', $user);
|
$eventSource->send('user', $user);
|
||||||
$scanner = new \OC\Files\Utils\Scanner($user);
|
$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection());
|
||||||
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file'));
|
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file'));
|
||||||
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', array($listener, 'folder'));
|
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', array($listener, 'folder'));
|
||||||
if ($force) {
|
if ($force) {
|
||||||
|
|
|
@ -46,7 +46,7 @@ class Scan extends Command {
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function scanFiles($user, OutputInterface $output) {
|
protected function scanFiles($user, OutputInterface $output) {
|
||||||
$scanner = new \OC\Files\Utils\Scanner($user);
|
$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection());
|
||||||
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
|
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
|
||||||
$output->writeln("Scanning <info>$path</info>");
|
$output->writeln("Scanning <info>$path</info>");
|
||||||
});
|
});
|
||||||
|
|
|
@ -44,6 +44,11 @@ class Scanner extends BasicEmitter {
|
||||||
*/
|
*/
|
||||||
protected $cacheActive;
|
protected $cacheActive;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @var bool $useTransactions whether to use transactions
|
||||||
|
*/
|
||||||
|
protected $useTransactions = true;
|
||||||
|
|
||||||
const SCAN_RECURSIVE = true;
|
const SCAN_RECURSIVE = true;
|
||||||
const SCAN_SHALLOW = false;
|
const SCAN_SHALLOW = false;
|
||||||
|
|
||||||
|
@ -57,6 +62,16 @@ class Scanner extends BasicEmitter {
|
||||||
$this->cacheActive = !Config::getSystemValue('filesystem_cache_readonly', false);
|
$this->cacheActive = !Config::getSystemValue('filesystem_cache_readonly', false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether to wrap the scanning of a folder in a database transaction
|
||||||
|
* On default transactions are used
|
||||||
|
*
|
||||||
|
* @param bool $useTransactions
|
||||||
|
*/
|
||||||
|
public function setUseTransactions($useTransactions) {
|
||||||
|
$this->useTransactions = $useTransactions;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* get all the metadata of a file or folder
|
* get all the metadata of a file or folder
|
||||||
* *
|
* *
|
||||||
|
@ -234,7 +249,9 @@ class Scanner extends BasicEmitter {
|
||||||
$newChildren = array();
|
$newChildren = array();
|
||||||
if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
|
if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
|
||||||
$exceptionOccurred = false;
|
$exceptionOccurred = false;
|
||||||
\OC_DB::beginTransaction();
|
if ($this->useTransactions) {
|
||||||
|
\OC_DB::beginTransaction();
|
||||||
|
}
|
||||||
if (is_resource($dh)) {
|
if (is_resource($dh)) {
|
||||||
while (($file = readdir($dh)) !== false) {
|
while (($file = readdir($dh)) !== false) {
|
||||||
$child = ($path) ? $path . '/' . $file : $file;
|
$child = ($path) ? $path . '/' . $file : $file;
|
||||||
|
@ -266,7 +283,9 @@ class Scanner extends BasicEmitter {
|
||||||
$child = ($path) ? $path . '/' . $childName : $childName;
|
$child = ($path) ? $path . '/' . $childName : $childName;
|
||||||
$this->removeFromCache($child);
|
$this->removeFromCache($child);
|
||||||
}
|
}
|
||||||
\OC_DB::commit();
|
if ($this->useTransactions) {
|
||||||
|
\OC_DB::commit();
|
||||||
|
}
|
||||||
if ($exceptionOccurred) {
|
if ($exceptionOccurred) {
|
||||||
// It might happen that the parallel scan process has already
|
// It might happen that the parallel scan process has already
|
||||||
// inserted mimetypes but those weren't available yet inside the transaction
|
// inserted mimetypes but those weren't available yet inside the transaction
|
||||||
|
|
|
@ -35,11 +35,18 @@ class Scanner extends PublicEmitter {
|
||||||
protected $propagator;
|
protected $propagator;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param string $user
|
* @var \OCP\IDBConnection
|
||||||
*/
|
*/
|
||||||
public function __construct($user) {
|
protected $db;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $user
|
||||||
|
* @param \OCP\IDBConnection $db
|
||||||
|
*/
|
||||||
|
public function __construct($user, $db) {
|
||||||
$this->user = $user;
|
$this->user = $user;
|
||||||
$this->propagator = new ChangePropagator(new View(''));
|
$this->propagator = new ChangePropagator(new View(''));
|
||||||
|
$this->db = $db;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -121,8 +128,11 @@ class Scanner extends PublicEmitter {
|
||||||
throw new ForbiddenException();
|
throw new ForbiddenException();
|
||||||
}
|
}
|
||||||
$scanner = $storage->getScanner();
|
$scanner = $storage->getScanner();
|
||||||
|
$scanner->useTransactions(false);
|
||||||
$this->attachListener($mount);
|
$this->attachListener($mount);
|
||||||
|
$this->db->beginTransaction();
|
||||||
$scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE);
|
$scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE);
|
||||||
|
$this->db->commit();
|
||||||
}
|
}
|
||||||
$this->propagator->propagateChanges(time());
|
$this->propagator->propagateChanges(time());
|
||||||
}
|
}
|
||||||
|
|
|
@ -62,7 +62,7 @@ class EtagTest extends \PHPUnit_Framework_TestCase {
|
||||||
$files = array('/foo.txt', '/folder/bar.txt', '/folder/subfolder', '/folder/subfolder/qwerty.txt');
|
$files = array('/foo.txt', '/folder/bar.txt', '/folder/subfolder', '/folder/subfolder/qwerty.txt');
|
||||||
$originalEtags = $this->getEtags($files);
|
$originalEtags = $this->getEtags($files);
|
||||||
|
|
||||||
$scanner = new \OC\Files\Utils\Scanner($user1);
|
$scanner = new \OC\Files\Utils\Scanner($user1, \OC::$server->getDatabaseConnection());
|
||||||
$scanner->backgroundScan('/');
|
$scanner->backgroundScan('/');
|
||||||
|
|
||||||
$newEtags = $this->getEtags($files);
|
$newEtags = $this->getEtags($files);
|
||||||
|
|
Loading…
Reference in New Issue