Add method to storage backends to get directory content with metadata

Currently you need to use `opendir` and then call `getMetadata` for
every file, which adds overhead because most storage backends already
get the metadata when doing the `opendir`.

While storagebackends can (and do) use caching to relief this problem,
this adds cache invalidation dificulties and only a limited number of
items are generally cached (to prevent memory usage exploding when
scanning large storages)

With this new methods storage backends can use the child metadata they
got from listing the folder to return metadata without having to keep
seperate caches.

Signed-off-by: Robin Appelman <robin@icewind.nl>
This commit is contained in:
Robin Appelman 2020-03-27 17:47:20 +01:00
parent b1a90da347
commit 9735b5d61f
No known key found for this signature in database
GPG Key ID: 42B69D8A64526EFB
14 changed files with 177 additions and 69 deletions

View File

@ -55,6 +55,7 @@ use OC\Cache\CappedMemoryCache;
use OC\Files\Filesystem; use OC\Files\Filesystem;
use OC\Files\Storage\Common; use OC\Files\Storage\Common;
use OCA\Files_External\Lib\Notify\SMBNotifyHandler; use OCA\Files_External\Lib\Notify\SMBNotifyHandler;
use OCP\Constants;
use OCP\Files\Notify\IChange; use OCP\Files\Notify\IChange;
use OCP\Files\Notify\IRenameChange; use OCP\Files\Notify\IRenameChange;
use OCP\Files\Storage\INotifyStorage; use OCP\Files\Storage\INotifyStorage;
@ -97,7 +98,7 @@ class SMB extends Common implements INotifyStorage {
if (isset($params['auth'])) { if (isset($params['auth'])) {
$auth = $params['auth']; $auth = $params['auth'];
} elseif (isset($params['user']) && isset($params['password']) && isset($params['share'])) { } elseif (isset($params['user']) && isset($params['password']) && isset($params['share'])) {
list($workgroup, $user) = $this->splitUser($params['user']); [$workgroup, $user] = $this->splitUser($params['user']);
$auth = new BasicAuth($user, $workgroup, $params['password']); $auth = new BasicAuth($user, $workgroup, $params['password']);
} else { } else {
throw new \Exception('Invalid configuration, no credentials provided'); throw new \Exception('Invalid configuration, no credentials provided');
@ -206,14 +207,15 @@ class SMB extends Common implements INotifyStorage {
* @return \Icewind\SMB\IFileInfo[] * @return \Icewind\SMB\IFileInfo[]
* @throws StorageNotAvailableException * @throws StorageNotAvailableException
*/ */
protected function getFolderContents($path) { protected function getFolderContents($path): iterable {
try { try {
$path = ltrim($this->buildPath($path), '/'); $path = ltrim($this->buildPath($path), '/');
$files = $this->share->dir($path); $files = $this->share->dir($path);
foreach ($files as $file) { foreach ($files as $file) {
$this->statCache[$path . '/' . $file->getName()] = $file; $this->statCache[$path . '/' . $file->getName()] = $file;
} }
return array_filter($files, function (IFileInfo $file) {
foreach ($files as $file) {
try { try {
// the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch // the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch
// so we trigger the below exceptions where applicable // so we trigger the below exceptions where applicable
@ -221,15 +223,15 @@ class SMB extends Common implements INotifyStorage {
if ($hide) { if ($hide) {
$this->logger->debug('hiding hidden file ' . $file->getName()); $this->logger->debug('hiding hidden file ' . $file->getName());
} }
return !$hide; if (!$hide) {
yield $file;
}
} catch (ForbiddenException $e) { } catch (ForbiddenException $e) {
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]); $this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]);
return false;
} catch (NotFoundException $e) { } catch (NotFoundException $e) {
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]); $this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]);
return false;
} }
}); }
} catch (ConnectException $e) { } catch (ConnectException $e) {
$this->logger->logException($e, ['message' => 'Error while getting folder content']); $this->logger->logException($e, ['message' => 'Error while getting folder content']);
throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e); throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e);
@ -508,6 +510,46 @@ class SMB extends Common implements INotifyStorage {
} }
} }
public function getMetaData($path) {
$fileInfo = $this->getFileInfo($path);
if (!$fileInfo) {
return null;
}
return $this->getMetaDataFromFileInfo($fileInfo);
}
private function getMetaDataFromFileInfo(IFileInfo $fileInfo) {
$permissions = Constants::PERMISSION_READ + Constants::PERMISSION_SHARE;
if (!$fileInfo->isReadOnly()) {
$permissions += Constants::PERMISSION_DELETE;
$permissions += Constants::PERMISSION_UPDATE;
if ($fileInfo->isDirectory()) {
$permissions += Constants::PERMISSION_CREATE;
}
}
$data = [];
if ($fileInfo->isDirectory()) {
$data['mimetype'] = 'httpd/unix-directory';
} else {
$data['mimetype'] = \OC::$server->getMimeTypeDetector()->detectPath($fileInfo->getPath());
}
$data['mtime'] = $fileInfo->getMTime();
if ($fileInfo->isDirectory()) {
$data['size'] = -1; //unknown
} else {
$data['size'] = $fileInfo->getSize();
}
$data['etag'] = $this->getETag($fileInfo->getPath());
$data['storage_mtime'] = $data['mtime'];
$data['permissions'] = $permissions;
$data['name'] = $fileInfo->getName();
return $data;
}
public function opendir($path) { public function opendir($path) {
try { try {
$files = $this->getFolderContents($path); $files = $this->getFolderContents($path);
@ -519,10 +561,17 @@ class SMB extends Common implements INotifyStorage {
$names = array_map(function ($info) { $names = array_map(function ($info) {
/** @var \Icewind\SMB\IFileInfo $info */ /** @var \Icewind\SMB\IFileInfo $info */
return $info->getName(); return $info->getName();
}, $files); }, iterator_to_array($files));
return IteratorDirectory::wrap($names); return IteratorDirectory::wrap($names);
} }
public function getDirectoryContent($directory): \Traversable {
$files = $this->getFolderContents($directory);
foreach ($files as $file) {
yield $this->getMetaDataFromFileInfo($file);
}
}
public function filetype($path) { public function filetype($path) {
try { try {
return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file'; return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file';

View File

@ -57,7 +57,7 @@ class Scanner extends \OC\Files\Cache\Scanner {
* @param bool $lock set to false to disable getting an additional read lock during scanning * @param bool $lock set to false to disable getting an additional read lock during scanning
* @return array an array of metadata of the scanned file * @return array an array of metadata of the scanned file
*/ */
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
try { try {
return parent::scanFile($file, $reuseExisting); return parent::scanFile($file, $reuseExisting);
} catch (ForbiddenException $e) { } catch (ForbiddenException $e) {

View File

@ -71,7 +71,7 @@ class Scanner extends \OC\Files\Cache\Scanner {
} }
} }
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
$sourceScanner = $this->getSourceScanner(); $sourceScanner = $this->getSourceScanner();
if ($sourceScanner instanceof NoopScanner) { if ($sourceScanner instanceof NoopScanner) {
return []; return [];

View File

@ -126,11 +126,11 @@ class Scanner extends BasicEmitter implements IScanner {
* @param int $parentId * @param int $parentId
* @param array|null|false $cacheData existing data in the cache for the file to be scanned * @param array|null|false $cacheData existing data in the cache for the file to be scanned
* @param bool $lock set to false to disable getting an additional read lock during scanning * @param bool $lock set to false to disable getting an additional read lock during scanning
* @param null $data the metadata for the file, as returned by the storage
* @return array an array of metadata of the scanned file * @return array an array of metadata of the scanned file
* @throws \OC\ServerNotAvailableException
* @throws \OCP\Lock\LockedException * @throws \OCP\Lock\LockedException
*/ */
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
if ($file !== '') { if ($file !== '') {
try { try {
$this->storage->verifyPath(dirname($file), basename($file)); $this->storage->verifyPath(dirname($file), basename($file));
@ -149,7 +149,7 @@ class Scanner extends BasicEmitter implements IScanner {
} }
try { try {
$data = $this->getData($file); $data = $data ?? $this->getData($file);
} catch (ForbiddenException $e) { } catch (ForbiddenException $e) {
if ($lock) { if ($lock) {
if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) { if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
@ -366,26 +366,6 @@ class Scanner extends BasicEmitter implements IScanner {
return $existingChildren; return $existingChildren;
} }
/**
* Get the children from the storage
*
* @param string $folder
* @return string[]
*/
protected function getNewChildren($folder) {
$children = [];
if ($dh = $this->storage->opendir($folder)) {
if (is_resource($dh)) {
while (($file = readdir($dh)) !== false) {
if (!Filesystem::isIgnoredDir($file)) {
$children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/');
}
}
}
}
return $children;
}
/** /**
* scan all the files and folders in a folder * scan all the files and folders in a folder
* *
@ -425,7 +405,7 @@ class Scanner extends BasicEmitter implements IScanner {
private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) { private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
// we put this in it's own function so it cleans up the memory before we start recursing // we put this in it's own function so it cleans up the memory before we start recursing
$existingChildren = $this->getExistingChildren($folderId); $existingChildren = $this->getExistingChildren($folderId);
$newChildren = $this->getNewChildren($path); $newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
if ($this->useTransactions) { if ($this->useTransactions) {
\OC::$server->getDatabaseConnection()->beginTransaction(); \OC::$server->getDatabaseConnection()->beginTransaction();
@ -433,11 +413,14 @@ class Scanner extends BasicEmitter implements IScanner {
$exceptionOccurred = false; $exceptionOccurred = false;
$childQueue = []; $childQueue = [];
foreach ($newChildren as $file) { $newChildNames = [];
foreach ($newChildren as $fileMeta) {
$file = $fileMeta['name'];
$newChildNames[] = $file;
$child = $path ? $path . '/' . $file : $file; $child = $path ? $path . '/' . $file : $file;
try { try {
$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false; $existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock); $data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
if ($data) { if ($data) {
if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) { if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
$childQueue[$child] = $data['fileid']; $childQueue[$child] = $data['fileid'];
@ -471,7 +454,7 @@ class Scanner extends BasicEmitter implements IScanner {
throw $e; throw $e;
} }
} }
$removedChildren = \array_diff(array_keys($existingChildren), $newChildren); $removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
foreach ($removedChildren as $childName) { foreach ($removedChildren as $childName) {
$child = $path ? $path . '/' . $childName : $childName; $child = $path ? $path . '/' . $childName : $childName;
$this->removeFromCache($child); $this->removeFromCache($child);

View File

@ -44,7 +44,7 @@ class NoopScanner extends Scanner {
* @param array|null $cacheData existing data in the cache for the file to be scanned * @param array|null $cacheData existing data in the cache for the file to be scanned
* @return array an array of metadata of the scanned file * @return array an array of metadata of the scanned file
*/ */
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) { public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
return []; return [];
} }

View File

@ -234,7 +234,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
} else { } else {
$source = $this->fopen($path1, 'r'); $source = $this->fopen($path1, 'r');
$target = $this->fopen($path2, 'w'); $target = $this->fopen($path2, 'w');
list(, $result) = \OC_Helper::streamCopy($source, $target); [, $result] = \OC_Helper::streamCopy($source, $target);
if (!$result) { if (!$result) {
\OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2"); \OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2");
} }
@ -626,7 +626,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
// are not the same as the original one.Once this is fixed we also // are not the same as the original one.Once this is fixed we also
// need to adjust the encryption wrapper. // need to adjust the encryption wrapper.
$target = $this->fopen($targetInternalPath, 'w'); $target = $this->fopen($targetInternalPath, 'w');
list(, $result) = \OC_Helper::streamCopy($source, $target); [, $result] = \OC_Helper::streamCopy($source, $target);
if ($result and $preserveMtime) { if ($result and $preserveMtime) {
$this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath)); $this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath));
} }
@ -719,6 +719,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
$data['etag'] = $this->getETag($path); $data['etag'] = $this->getETag($path);
$data['storage_mtime'] = $data['mtime']; $data['storage_mtime'] = $data['mtime'];
$data['permissions'] = $permissions; $data['permissions'] = $permissions;
$data['name'] = basename($path);
return $data; return $data;
} }
@ -867,4 +868,17 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
} }
return $count; return $count;
} }
public function getDirectoryContent($directory): \Traversable {
$dh = $this->opendir($directory);
if (is_resource($dh)) {
$basePath = rtrim($directory, '/');
while (($file = readdir($dh)) !== false) {
if (!Filesystem::isIgnoredDir($file)) {
$childPath = $basePath . '/' . trim($file, '/');
yield $this->getMetaData($childPath);
}
}
}
}
} }

View File

@ -196,6 +196,7 @@ class Local extends \OC\Files\Storage\Common {
$data['etag'] = $this->calculateEtag($path, $stat); $data['etag'] = $this->calculateEtag($path, $stat);
$data['storage_mtime'] = $data['mtime']; $data['storage_mtime'] = $data['mtime'];
$data['permissions'] = $permissions; $data['permissions'] = $permissions;
$data['name'] = basename($path);
return $data; return $data;
} }

View File

@ -119,4 +119,22 @@ interface Storage extends \OCP\Files\Storage {
* @throws \OCP\Lock\LockedException * @throws \OCP\Lock\LockedException
*/ */
public function changeLock($path, $type, ILockingProvider $provider); public function changeLock($path, $type, ILockingProvider $provider);
/**
* Get the contents of a directory with metadata
*
* @param string $directory
* @return \Traversable an iterator, containing file metadata
*
* The metadata array will contain the following fields
*
* - name
* - mimetype
* - mtime
* - size
* - etag
* - storage_mtime
* - permissions
*/
public function getDirectoryContent($directory): \Traversable;
} }

View File

@ -461,4 +461,15 @@ class Availability extends Wrapper {
$this->getStorageCache()->setAvailability(false, $delay); $this->getStorageCache()->setAvailability(false, $delay);
throw $e; throw $e;
} }
public function getDirectoryContent($directory): \Traversable {
$this->checkAvailability();
try {
return parent::getDirectoryContent($directory);
} catch (StorageNotAvailableException $e) {
$this->setUnavailable($e);
}
}
} }

View File

@ -534,4 +534,8 @@ class Encoding extends Wrapper {
public function getMetaData($path) { public function getMetaData($path) {
return $this->storage->getMetaData($this->findPathToUse($path)); return $this->storage->getMetaData($this->findPathToUse($path));
} }
public function getDirectoryContent($directory): \Traversable {
return $this->storage->getDirectoryContent($this->findPathToUse($directory));
}
} }

View File

@ -169,15 +169,7 @@ class Encryption extends Wrapper {
return $this->storage->filesize($path); return $this->storage->filesize($path);
} }
/** private function modifyMetaData(string $path, array $data): array {
* @param string $path
* @return array
*/
public function getMetaData($path) {
$data = $this->storage->getMetaData($path);
if (is_null($data)) {
return null;
}
$fullPath = $this->getFullPath($path); $fullPath = $this->getFullPath($path);
$info = $this->getCache()->get($path); $info = $this->getCache()->get($path);
@ -198,6 +190,25 @@ class Encryption extends Wrapper {
return $data; return $data;
} }
/**
* @param string $path
* @return array
*/
public function getMetaData($path) {
$data = $this->storage->getMetaData($path);
if (is_null($data)) {
return null;
}
return $this->modifyMetaData($path, $data);
}
public function getDirectoryContent($directory): \Traversable {
$parent = rtrim($directory, '/');
foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
yield $this->modifyMetaData($parent . '/' . $data['name'], $data);
}
}
/** /**
* see http://php.net/manual/en/function.file_get_contents.php * see http://php.net/manual/en/function.file_get_contents.php
* *
@ -743,7 +754,7 @@ class Encryption extends Wrapper {
try { try {
$source = $sourceStorage->fopen($sourceInternalPath, 'r'); $source = $sourceStorage->fopen($sourceInternalPath, 'r');
$target = $this->fopen($targetInternalPath, 'w'); $target = $this->fopen($targetInternalPath, 'w');
list(, $result) = \OC_Helper::streamCopy($source, $target); [, $result] = \OC_Helper::streamCopy($source, $target);
fclose($source); fclose($source);
fclose($target); fclose($target);
} catch (\Exception $e) { } catch (\Exception $e) {
@ -1023,7 +1034,7 @@ class Encryption extends Wrapper {
public function writeStream(string $path, $stream, int $size = null): int { public function writeStream(string $path, $stream, int $size = null): int {
// always fall back to fopen // always fall back to fopen
$target = $this->fopen($path, 'w'); $target = $this->fopen($path, 'w');
list($count, $result) = \OC_Helper::streamCopy($stream, $target); [$count, $result] = \OC_Helper::streamCopy($stream, $target);
fclose($target); fclose($target);
return $count; return $count;
} }

View File

@ -539,4 +539,8 @@ class Jail extends Wrapper {
return $count; return $count;
} }
} }
public function getDirectoryContent($directory): \Traversable {
return $this->getWrapperStorage()->getDirectoryContent($this->getJailedPath($directory));
}
} }

View File

@ -157,4 +157,13 @@ class PermissionsMask extends Wrapper {
} }
return parent::getScanner($path, $storage); return parent::getScanner($path, $storage);
} }
public function getDirectoryContent($directory): \Traversable {
foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
$data['scan_permissions'] = isset($data['scan_permissions']) ? $data['scan_permissions'] : $data['permissions'];
$data['permissions'] &= $this->mask;
yield $data;
}
}
} }

View File

@ -637,4 +637,8 @@ class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStrea
return $count; return $count;
} }
} }
public function getDirectoryContent($directory): \Traversable {
return $this->getWrapperStorage()->getDirectoryContent($directory);
}
} }