Merge pull request #20232 from nextcloud/storage-getdirectorycontent

Add method to storage backends to get directory content with metadata

apps/files_external/lib/Lib/Storage/SMB.php

@@ -55,6 +55,7 @@ use OC\Cache\CappedMemoryCache;
 use OC\Files\Filesystem;
 use OC\Files\Storage\Common;
 use OCA\Files_External\Lib\Notify\SMBNotifyHandler;
+use OCP\Constants;
 use OCP\Files\Notify\IChange;
 use OCP\Files\Notify\IRenameChange;
 use OCP\Files\Storage\INotifyStorage;
@@ -97,7 +98,7 @@ class SMB extends Common implements INotifyStorage {
 		if (isset($params['auth'])) {
 			$auth = $params['auth'];
 		} elseif (isset($params['user']) && isset($params['password']) && isset($params['share'])) {
-			list($workgroup, $user) = $this->splitUser($params['user']);
+			[$workgroup, $user] = $this->splitUser($params['user']);
 			$auth = new BasicAuth($user, $workgroup, $params['password']);
 		} else {
 			throw new \Exception('Invalid configuration, no credentials provided');
@@ -206,14 +207,15 @@ class SMB extends Common implements INotifyStorage {
 	 * @return \Icewind\SMB\IFileInfo[]
 	 * @throws StorageNotAvailableException
 	 */
-	protected function getFolderContents($path) {
+	protected function getFolderContents($path): iterable {
 		try {
 			$path = ltrim($this->buildPath($path), '/');
 			$files = $this->share->dir($path);
 			foreach ($files as $file) {
 				$this->statCache[$path . '/' . $file->getName()] = $file;
 			}
-			return array_filter($files, function (IFileInfo $file) {
+
+			foreach ($files as $file) {
 				try {
 					// the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch
 					// so we trigger the below exceptions where applicable
@@ -221,15 +223,15 @@ class SMB extends Common implements INotifyStorage {
 					if ($hide) {
 						$this->logger->debug('hiding hidden file ' . $file->getName());
 					}
-					return !$hide;
+					if (!$hide) {
+						yield $file;
+					}
 				} catch (ForbiddenException $e) {
 					$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]);
-					return false;
 				} catch (NotFoundException $e) {
 					$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]);
-					return false;
 				}
-			});
+			}
 		} catch (ConnectException $e) {
 			$this->logger->logException($e, ['message' => 'Error while getting folder content']);
 			throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e);
@@ -508,6 +510,46 @@ class SMB extends Common implements INotifyStorage {
 		}
 	}
 
+	public function getMetaData($path) {
+		$fileInfo = $this->getFileInfo($path);
+		if (!$fileInfo) {
+			return null;
+		}
+
+		return $this->getMetaDataFromFileInfo($fileInfo);
+	}
+
+	private function getMetaDataFromFileInfo(IFileInfo $fileInfo) {
+		$permissions = Constants::PERMISSION_READ + Constants::PERMISSION_SHARE;
+
+		if (!$fileInfo->isReadOnly()) {
+			$permissions += Constants::PERMISSION_DELETE;
+			$permissions += Constants::PERMISSION_UPDATE;
+			if ($fileInfo->isDirectory()) {
+				$permissions += Constants::PERMISSION_CREATE;
+			}
+		}
+
+		$data = [];
+		if ($fileInfo->isDirectory()) {
+			$data['mimetype'] = 'httpd/unix-directory';
+		} else {
+			$data['mimetype'] = \OC::$server->getMimeTypeDetector()->detectPath($fileInfo->getPath());
+		}
+		$data['mtime'] = $fileInfo->getMTime();
+		if ($fileInfo->isDirectory()) {
+			$data['size'] = -1; //unknown
+		} else {
+			$data['size'] = $fileInfo->getSize();
+		}
+		$data['etag'] = $this->getETag($fileInfo->getPath());
+		$data['storage_mtime'] = $data['mtime'];
+		$data['permissions'] = $permissions;
+		$data['name'] = $fileInfo->getName();
+
+		return $data;
+	}
+
 	public function opendir($path) {
 		try {
 			$files = $this->getFolderContents($path);
@@ -519,10 +561,17 @@ class SMB extends Common implements INotifyStorage {
 		$names = array_map(function ($info) {
 			/** @var \Icewind\SMB\IFileInfo $info */
 			return $info->getName();
-		}, $files);
+		}, iterator_to_array($files));
 		return IteratorDirectory::wrap($names);
 	}
 
+	public function getDirectoryContent($directory): \Traversable {
+		$files = $this->getFolderContents($directory);
+		foreach ($files as $file) {
+			yield $this->getMetaDataFromFileInfo($file);
+		}
+	}
+
 	public function filetype($path) {
 		try {
 			return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file';

apps/files_sharing/lib/External/Scanner.php

@@ -57,7 +57,7 @@ class Scanner extends \OC\Files\Cache\Scanner {
 	 * @param bool $lock set to false to disable getting an additional read lock during scanning
 	 * @return array an array of metadata of the scanned file
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		try {
 			return parent::scanFile($file, $reuseExisting);
 		} catch (ForbiddenException $e) {

apps/files_sharing/lib/Scanner.php

@@ -71,7 +71,7 @@ class Scanner extends \OC\Files\Cache\Scanner {
 		}
 	}
 
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		$sourceScanner = $this->getSourceScanner();
 		if ($sourceScanner instanceof NoopScanner) {
 			return [];

lib/private/Files/Cache/Scanner.php

@@ -126,11 +126,11 @@ class Scanner extends BasicEmitter implements IScanner {
 	 * @param int $parentId
 	 * @param array|null|false $cacheData existing data in the cache for the file to be scanned
 	 * @param bool $lock set to false to disable getting an additional read lock during scanning
+	 * @param null $data the metadata for the file, as returned by the storage
 	 * @return array an array of metadata of the scanned file
-	 * @throws \OC\ServerNotAvailableException
 	 * @throws \OCP\Lock\LockedException
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		if ($file !== '') {
 			try {
 				$this->storage->verifyPath(dirname($file), basename($file));
@@ -149,7 +149,7 @@ class Scanner extends BasicEmitter implements IScanner {
 			}
 
 			try {
-				$data = $this->getData($file);
+				$data = $data ?? $this->getData($file);
 			} catch (ForbiddenException $e) {
 				if ($lock) {
 					if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
@@ -366,26 +366,6 @@ class Scanner extends BasicEmitter implements IScanner {
 		return $existingChildren;
 	}
 
-	/**
-	 * Get the children from the storage
-	 *
-	 * @param string $folder
-	 * @return string[]
-	 */
-	protected function getNewChildren($folder) {
-		$children = [];
-		if ($dh = $this->storage->opendir($folder)) {
-			if (is_resource($dh)) {
-				while (($file = readdir($dh)) !== false) {
-					if (!Filesystem::isIgnoredDir($file)) {
-						$children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/');
-					}
-				}
-			}
-		}
-		return $children;
-	}
-
 	/**
 	 * scan all the files and folders in a folder
 	 *
@@ -425,7 +405,7 @@ class Scanner extends BasicEmitter implements IScanner {
 	private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
 		// we put this in it's own function so it cleans up the memory before we start recursing
 		$existingChildren = $this->getExistingChildren($folderId);
-		$newChildren = $this->getNewChildren($path);
+		$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
 
 		if ($this->useTransactions) {
 			\OC::$server->getDatabaseConnection()->beginTransaction();
@@ -433,11 +413,14 @@ class Scanner extends BasicEmitter implements IScanner {
 
 		$exceptionOccurred = false;
 		$childQueue = [];
-		foreach ($newChildren as $file) {
+		$newChildNames = [];
+		foreach ($newChildren as $fileMeta) {
+			$file = $fileMeta['name'];
+			$newChildNames[] = $file;
 			$child = $path ? $path . '/' . $file : $file;
 			try {
 				$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
-				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock);
+				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
 				if ($data) {
 					if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
 						$childQueue[$child] = $data['fileid'];
@@ -471,7 +454,7 @@ class Scanner extends BasicEmitter implements IScanner {
 				throw $e;
 			}
 		}
-		$removedChildren = \array_diff(array_keys($existingChildren), $newChildren);
+		$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
 		foreach ($removedChildren as $childName) {
 			$child = $path ? $path . '/' . $childName : $childName;
 			$this->removeFromCache($child);

lib/private/Files/ObjectStore/NoopScanner.php

@@ -44,7 +44,7 @@ class NoopScanner extends Scanner {
 	 * @param array|null $cacheData existing data in the cache for the file to be scanned
 	 * @return array an array of metadata of the scanned file
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		return [];
 	}
 

lib/private/Files/Storage/Common.php

@@ -234,7 +234,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 		} else {
 			$source = $this->fopen($path1, 'r');
 			$target = $this->fopen($path2, 'w');
-			list(, $result) = \OC_Helper::streamCopy($source, $target);
+			[, $result] = \OC_Helper::streamCopy($source, $target);
 			if (!$result) {
 				\OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2");
 			}
@@ -626,7 +626,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 			// are not the same as the original one.Once this is fixed we also
 			// need to adjust the encryption wrapper.
 			$target = $this->fopen($targetInternalPath, 'w');
-			list(, $result) = \OC_Helper::streamCopy($source, $target);
+			[, $result] = \OC_Helper::streamCopy($source, $target);
 			if ($result and $preserveMtime) {
 				$this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath));
 			}
@@ -719,6 +719,7 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 		$data['etag'] = $this->getETag($path);
 		$data['storage_mtime'] = $data['mtime'];
 		$data['permissions'] = $permissions;
+		$data['name'] = basename($path);
 
 		return $data;
 	}
@@ -867,4 +868,17 @@ abstract class Common implements Storage, ILockingStorage, IWriteStreamStorage {
 		}
 		return $count;
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		$dh = $this->opendir($directory);
+		if (is_resource($dh)) {
+			$basePath = rtrim($directory, '/');
+			while (($file = readdir($dh)) !== false) {
+				if (!Filesystem::isIgnoredDir($file)) {
+					$childPath = $basePath . '/' . trim($file, '/');
+					yield $this->getMetaData($childPath);
+				}
+			}
+		}
+	}
 }

lib/private/Files/Storage/Local.php

@@ -196,6 +196,7 @@ class Local extends \OC\Files\Storage\Common {
 		$data['etag'] = $this->calculateEtag($path, $stat);
 		$data['storage_mtime'] = $data['mtime'];
 		$data['permissions'] = $permissions;
+		$data['name'] = basename($path);
 
 		return $data;
 	}

lib/private/Files/Storage/Storage.php

@@ -119,4 +119,22 @@ interface Storage extends \OCP\Files\Storage {
 	 * @throws \OCP\Lock\LockedException
 	 */
 	public function changeLock($path, $type, ILockingProvider $provider);
+
+	/**
+	 * Get the contents of a directory with metadata
+	 *
+	 * @param string $directory
+	 * @return \Traversable an iterator, containing file metadata
+	 *
+	 * The metadata array will contain the following fields
+	 *
+	 * - name
+	 * - mimetype
+	 * - mtime
+	 * - size
+	 * - etag
+	 * - storage_mtime
+	 * - permissions
+	 */
+	public function getDirectoryContent($directory): \Traversable;
 }

lib/private/Files/Storage/Wrapper/Availability.php

@@ -461,4 +461,15 @@ class Availability extends Wrapper {
 		$this->getStorageCache()->setAvailability(false, $delay);
 		throw $e;
 	}
+
+
+
+	public function getDirectoryContent($directory): \Traversable {
+		$this->checkAvailability();
+		try {
+			return parent::getDirectoryContent($directory);
+		} catch (StorageNotAvailableException $e) {
+			$this->setUnavailable($e);
+		}
+	}
 }

lib/private/Files/Storage/Wrapper/Encoding.php

@@ -534,4 +534,8 @@ class Encoding extends Wrapper {
 	public function getMetaData($path) {
 		return $this->storage->getMetaData($this->findPathToUse($path));
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		return $this->storage->getDirectoryContent($this->findPathToUse($directory));
+	}
 }

lib/private/Files/Storage/Wrapper/Encryption.php

@@ -105,17 +105,17 @@ class Encryption extends Wrapper {
 	 * @param ArrayCache $arrayCache
 	 */
 	public function __construct(
-			$parameters,
-			IManager $encryptionManager = null,
-			Util $util = null,
-			ILogger $logger = null,
-			IFile $fileHelper = null,
-			$uid = null,
-			IStorage $keyStorage = null,
-			Update $update = null,
-			Manager $mountManager = null,
-			ArrayCache $arrayCache = null
-		) {
+		$parameters,
+		IManager $encryptionManager = null,
+		Util $util = null,
+		ILogger $logger = null,
+		IFile $fileHelper = null,
+		$uid = null,
+		IStorage $keyStorage = null,
+		Update $update = null,
+		Manager $mountManager = null,
+		ArrayCache $arrayCache = null
+	) {
 		$this->mountPoint = $parameters['mountPoint'];
 		$this->mount = $parameters['mount'];
 		$this->encryptionManager = $encryptionManager;
@@ -169,15 +169,7 @@ class Encryption extends Wrapper {
 		return $this->storage->filesize($path);
 	}
 
-	/**
-	 * @param string $path
-	 * @return array
-	 */
-	public function getMetaData($path) {
-		$data = $this->storage->getMetaData($path);
-		if (is_null($data)) {
-			return null;
-		}
+	private function modifyMetaData(string $path, array $data): array {
 		$fullPath = $this->getFullPath($path);
 		$info = $this->getCache()->get($path);
 
@@ -198,6 +190,25 @@ class Encryption extends Wrapper {
 		return $data;
 	}
 
+	/**
+	 * @param string $path
+	 * @return array
+	 */
+	public function getMetaData($path) {
+		$data = $this->storage->getMetaData($path);
+		if (is_null($data)) {
+			return null;
+		}
+		return $this->modifyMetaData($path, $data);
+	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		$parent = rtrim($directory, '/');
+		foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
+			yield $this->modifyMetaData($parent . '/' . $data['name'], $data);
+		}
+	}
+
 	/**
 	 * see http://php.net/manual/en/function.file_get_contents.php
 	 *
@@ -493,7 +504,7 @@ class Encryption extends Wrapper {
 				try {
 					$result = $this->fixUnencryptedSize($path, $size, $unencryptedSize);
 				} catch (\Exception $e) {
-					$this->logger->error('Couldn\'t re-calculate unencrypted size for '. $path);
+					$this->logger->error('Couldn\'t re-calculate unencrypted size for ' . $path);
 					$this->logger->logException($e);
 				}
 				unset($this->fixUnencryptedSizeOf[$this->getFullPath($path)]);
@@ -546,7 +557,7 @@ class Encryption extends Wrapper {
 		// next highest is end of chunks, one subtracted is last one
 		// we have to read the last chunk, we can't just calculate it (because of padding etc)
 
-		$lastChunkNr = ceil($size/ $blockSize)-1;
+		$lastChunkNr = ceil($size / $blockSize) - 1;
 		// calculate last chunk position
 		$lastChunkPos = ($lastChunkNr * $blockSize);
 		// try to fseek to the last chunk, if it fails we have to read the whole file
@@ -554,16 +565,16 @@ class Encryption extends Wrapper {
 			$newUnencryptedSize += $lastChunkNr * $unencryptedBlockSize;
 		}
 
-		$lastChunkContentEncrypted='';
+		$lastChunkContentEncrypted = '';
 		$count = $blockSize;
 
 		while ($count > 0) {
-			$data=fread($stream, $blockSize);
-			$count=strlen($data);
+			$data = fread($stream, $blockSize);
+			$count = strlen($data);
 			$lastChunkContentEncrypted .= $data;
 			if (strlen($lastChunkContentEncrypted) > $blockSize) {
 				$newUnencryptedSize += $unencryptedBlockSize;
-				$lastChunkContentEncrypted=substr($lastChunkContentEncrypted, $blockSize);
+				$lastChunkContentEncrypted = substr($lastChunkContentEncrypted, $blockSize);
 			}
 		}
 
@@ -743,7 +754,7 @@ class Encryption extends Wrapper {
 			try {
 				$source = $sourceStorage->fopen($sourceInternalPath, 'r');
 				$target = $this->fopen($targetInternalPath, 'w');
-				list(, $result) = \OC_Helper::streamCopy($source, $target);
+				[, $result] = \OC_Helper::streamCopy($source, $target);
 				fclose($source);
 				fclose($target);
 			} catch (\Exception $e) {
@@ -889,7 +900,7 @@ class Encryption extends Wrapper {
 				$header = substr($header, 0, $endAt + strlen(Util::HEADER_END));
 
 				// +1 to not start with an ':' which would result in empty element at the beginning
-				$exploded = explode(':', substr($header, strlen(Util::HEADER_START)+1));
+				$exploded = explode(':', substr($header, strlen(Util::HEADER_START) + 1));
 
 				$element = array_shift($exploded);
 				while ($element !== Util::HEADER_END) {
@@ -1023,7 +1034,7 @@ class Encryption extends Wrapper {
 	public function writeStream(string $path, $stream, int $size = null): int {
 		// always fall back to fopen
 		$target = $this->fopen($path, 'w');
-		list($count, $result) = \OC_Helper::streamCopy($stream, $target);
+		[$count, $result] = \OC_Helper::streamCopy($stream, $target);
 		fclose($target);
 		return $count;
 	}

lib/private/Files/Storage/Wrapper/Jail.php

@@ -539,4 +539,8 @@ class Jail extends Wrapper {
 			return $count;
 		}
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		return $this->getWrapperStorage()->getDirectoryContent($this->getJailedPath($directory));
+	}
 }

lib/private/Files/Storage/Wrapper/PermissionsMask.php

@@ -157,4 +157,13 @@ class PermissionsMask extends Wrapper {
 		}
 		return parent::getScanner($path, $storage);
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
+			$data['scan_permissions'] = isset($data['scan_permissions']) ? $data['scan_permissions'] : $data['permissions'];
+			$data['permissions'] &= $this->mask;
+
+			yield $data;
+		}
+	}
 }

lib/private/Files/Storage/Wrapper/Wrapper.php

@@ -637,4 +637,8 @@ class Wrapper implements \OC\Files\Storage\Storage, ILockingStorage, IWriteStrea
 			return $count;
 		}
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		return $this->getWrapperStorage()->getDirectoryContent($directory);
+	}
 }