mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Catch duplicate insertion errors while scanning files 

When two scanning processed run at the same time, for example when
scan.php and list.php were called at the same time on an external
storage mountpoint, duplicate insertion errors can occurs.

These errors are now logged and ignored.

Since both scans are running in parallel transactions, they don't see
each other's changes directly in the DB which can cause duplicate
insertion errors for mime types as well, but those mime types can't be
selected yet. The solution to this is to force-reload the mimetypes list
after the transaction is finished.

Fixes #5199
This commit is contained in:
Vincent Petry 2013-10-21 14:48:08 +02:00
parent 2d14daf36b
commit a542c57a7b
2 changed files with 38 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/private/files/cache/cache.php vendored
View File

@ -69,9 +69,15 @@ class Cache {
}
if (!isset(self::$mimetypeIds[$mime])) {
$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
try{
$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
}
catch (\Doctrine\DBAL\DBALException $e){
\OC_Log::write('core', 'Exception during mimetype insertion: ' . $e->getmessage(), \OC_Log::DEBUG);
return -1;
}
}
return self::$mimetypeIds[$mime];
@ -84,8 +90,8 @@ class Cache {
return isset(self::$mimetypes[$id]) ? self::$mimetypes[$id] : null;
}
protected function loadMimetypes(){
public function loadMimetypes(){
$result = \OC_DB::executeAudited('SELECT `id`, `mimetype` FROM `*PREFIX*mimetypes`', array());
if ($result) {
while ($row = $result->fetchRow()) {

36
lib/private/files/cache/scanner.php vendored
View File

@ -190,24 +190,34 @@ class Scanner extends BasicEmitter {
}
$newChildren = array();
if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
$exceptionOccurred = false;
\OC_DB::beginTransaction();
if (is_resource($dh)) {
while (($file = readdir($dh)) !== false) {
$child = ($path) ? $path . '/' . $file : $file;
if (!Filesystem::isIgnoredDir($file)) {
$newChildren[] = $file;
$data = $this->scanFile($child, $reuse, true);
if ($data) {
if ($data['size'] === -1) {
if ($recursive === self::SCAN_RECURSIVE) {
$childQueue[] = $child;
} else {
$size = -1;
try {
$data = $this->scanFile($child, $reuse, true);
if ($data) {
if ($data['size'] === -1) {
if ($recursive === self::SCAN_RECURSIVE) {
$childQueue[] = $child;
} else {
$size = -1;
}
} else if ($size !== -1) {
$size += $data['size'];
}
} else if ($size !== -1) {
$size += $data['size'];
}
}
catch (\Doctrine\DBAL\DBALException $ex){
// might happen if inserting duplicate while a scanning
// process is running in parallel
// log and ignore
\OC_Log::write('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OC_Log::DEBUG);
$exceptionOccurred = true;
}
}
}
}
@ -217,6 +227,14 @@ class Scanner extends BasicEmitter {
$this->cache->remove($child);
}
\OC_DB::commit();
if ($exceptionOccurred){
// It might happen that the parallel scan process has already
// inserted mimetypes but those weren't available yet inside the transaction
// To make sure to have the updated mime types in such cases,
// we reload them here
$this->cache->loadMimetypes();
}
foreach ($childQueue as $child) {
$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
if ($childSize === -1) {