Catch duplicate insertion errors while scanning files
When two scanning processed run at the same time, for example when scan.php and list.php were called at the same time on an external storage mountpoint, duplicate insertion errors can occurs. These errors are now logged and ignored. Since both scans are running in parallel transactions, they don't see each other's changes directly in the DB which can cause duplicate insertion errors for mime types as well, but those mime types can't be selected yet. The solution to this is to force-reload the mimetypes list after the transaction is finished. Fixes #5199
This commit is contained in:
parent
2d14daf36b
commit
a542c57a7b
|
@ -69,9 +69,15 @@ class Cache {
|
|||
}
|
||||
|
||||
if (!isset(self::$mimetypeIds[$mime])) {
|
||||
$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
|
||||
self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
|
||||
self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
|
||||
try{
|
||||
$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
|
||||
self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
|
||||
self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
|
||||
}
|
||||
catch (\Doctrine\DBAL\DBALException $e){
|
||||
\OC_Log::write('core', 'Exception during mimetype insertion: ' . $e->getmessage(), \OC_Log::DEBUG);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return self::$mimetypeIds[$mime];
|
||||
|
@ -85,7 +91,7 @@ class Cache {
|
|||
return isset(self::$mimetypes[$id]) ? self::$mimetypes[$id] : null;
|
||||
}
|
||||
|
||||
protected function loadMimetypes(){
|
||||
public function loadMimetypes(){
|
||||
$result = \OC_DB::executeAudited('SELECT `id`, `mimetype` FROM `*PREFIX*mimetypes`', array());
|
||||
if ($result) {
|
||||
while ($row = $result->fetchRow()) {
|
||||
|
|
|
@ -190,24 +190,34 @@ class Scanner extends BasicEmitter {
|
|||
}
|
||||
$newChildren = array();
|
||||
if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
|
||||
$exceptionOccurred = false;
|
||||
\OC_DB::beginTransaction();
|
||||
if (is_resource($dh)) {
|
||||
while (($file = readdir($dh)) !== false) {
|
||||
$child = ($path) ? $path . '/' . $file : $file;
|
||||
if (!Filesystem::isIgnoredDir($file)) {
|
||||
$newChildren[] = $file;
|
||||
$data = $this->scanFile($child, $reuse, true);
|
||||
if ($data) {
|
||||
if ($data['size'] === -1) {
|
||||
if ($recursive === self::SCAN_RECURSIVE) {
|
||||
$childQueue[] = $child;
|
||||
} else {
|
||||
$size = -1;
|
||||
try {
|
||||
$data = $this->scanFile($child, $reuse, true);
|
||||
if ($data) {
|
||||
if ($data['size'] === -1) {
|
||||
if ($recursive === self::SCAN_RECURSIVE) {
|
||||
$childQueue[] = $child;
|
||||
} else {
|
||||
$size = -1;
|
||||
}
|
||||
} else if ($size !== -1) {
|
||||
$size += $data['size'];
|
||||
}
|
||||
} else if ($size !== -1) {
|
||||
$size += $data['size'];
|
||||
}
|
||||
}
|
||||
catch (\Doctrine\DBAL\DBALException $ex){
|
||||
// might happen if inserting duplicate while a scanning
|
||||
// process is running in parallel
|
||||
// log and ignore
|
||||
\OC_Log::write('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OC_Log::DEBUG);
|
||||
$exceptionOccurred = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -217,6 +227,14 @@ class Scanner extends BasicEmitter {
|
|||
$this->cache->remove($child);
|
||||
}
|
||||
\OC_DB::commit();
|
||||
if ($exceptionOccurred){
|
||||
// It might happen that the parallel scan process has already
|
||||
// inserted mimetypes but those weren't available yet inside the transaction
|
||||
// To make sure to have the updated mime types in such cases,
|
||||
// we reload them here
|
||||
$this->cache->loadMimetypes();
|
||||
}
|
||||
|
||||
foreach ($childQueue as $child) {
|
||||
$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
|
||||
if ($childSize === -1) {
|
||||
|
|
Loading…
Reference in New Issue