From 4f71915b052bbe82d1c4fb94cdfea342a19e296c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Thu, 6 May 2021 18:26:42 +0200 Subject: [PATCH] First attempt to make multipartPart upload working MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- .../composer/composer/autoload_classmap.php | 1 + .../dav/composer/composer/autoload_static.php | 1 + apps/dav/lib/Connector/Sabre/Directory.php | 1 + apps/dav/lib/Connector/Sabre/Node.php | 8 + apps/dav/lib/Server.php | 2 + apps/dav/lib/Upload/ChunkingV2Plugin.php | 223 ++++++++++++++++++ apps/dav/lib/Upload/FutureFile.php | 4 + apps/dav/lib/Upload/UploadFile.php | 8 + apps/dav/lib/Upload/UploadFolder.php | 10 +- apps/dav/lib/Upload/UploadHome.php | 20 +- build/psalm-baseline.xml | 5 + lib/composer/composer/autoload_classmap.php | 2 + lib/composer/composer/autoload_static.php | 2 + .../Files/ObjectStore/ObjectStoreStorage.php | 114 ++++++++- lib/private/Files/ObjectStore/S3.php | 50 +++- lib/private/Files/View.php | 18 ++ .../IObjectStoreMultiPartUpload.php | 54 +++++ .../Files/Storage/IChunkedFileWrite.php | 71 ++++++ 18 files changed, 585 insertions(+), 9 deletions(-) create mode 100644 apps/dav/lib/Upload/ChunkingV2Plugin.php create mode 100644 lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php create mode 100644 lib/public/Files/Storage/IChunkedFileWrite.php diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index 5da3526c82..97edd8d601 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -260,6 +260,7 @@ return array( 'OCA\\DAV\\Traits\\PrincipalProxyTrait' => $baseDir . '/../lib/Traits/PrincipalProxyTrait.php', 'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php', 'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php', + 'OCA\\DAV\\Upload\\ChunkingV2Plugin' => $baseDir . '/../lib/Upload/ChunkingV2Plugin.php', 'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php', 'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php', 'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index 60162ba555..12272a11a1 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -275,6 +275,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Traits\\PrincipalProxyTrait' => __DIR__ . '/..' . '/../lib/Traits/PrincipalProxyTrait.php', 'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php', 'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php', + 'OCA\\DAV\\Upload\\ChunkingV2Plugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingV2Plugin.php', 'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php', 'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php', 'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php', diff --git a/apps/dav/lib/Connector/Sabre/Directory.php b/apps/dav/lib/Connector/Sabre/Directory.php index 98705c5779..d455df5fb9 100644 --- a/apps/dav/lib/Connector/Sabre/Directory.php +++ b/apps/dav/lib/Connector/Sabre/Directory.php @@ -38,6 +38,7 @@ use OC\Files\View; use OCA\DAV\Connector\Sabre\Exception\FileLocked; use OCA\DAV\Connector\Sabre\Exception\Forbidden; use OCA\DAV\Connector\Sabre\Exception\InvalidPath; +use OCA\DAV\Upload\FutureFile; use OCP\Files\FileInfo; use OCP\Files\ForbiddenException; use OCP\Files\InvalidPathException; diff --git a/apps/dav/lib/Connector/Sabre/Node.php b/apps/dav/lib/Connector/Sabre/Node.php index b713d0d485..92f0e12d70 100644 --- a/apps/dav/lib/Connector/Sabre/Node.php +++ b/apps/dav/lib/Connector/Sabre/Node.php @@ -248,6 +248,14 @@ abstract class Node implements \Sabre\DAV\INode { return $this->info->getId(); } + public function getInternalPath(): string { + return $this->info->getInternalPath(); + } + + public function getAbsoluteInternalPath(): string { + return $this->info->getPath(); + } + /** * @param string $user * @return int diff --git a/apps/dav/lib/Server.php b/apps/dav/lib/Server.php index ba5bcf935e..1b4d390016 100644 --- a/apps/dav/lib/Server.php +++ b/apps/dav/lib/Server.php @@ -66,6 +66,7 @@ use OCA\DAV\Files\LazySearchBackend; use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin; use OCA\DAV\SystemTag\SystemTagPlugin; use OCA\DAV\Upload\ChunkingPlugin; +use OCA\DAV\Upload\ChunkingV2Plugin; use OCP\EventDispatcher\IEventDispatcher; use OCP\IRequest; use OCP\SabrePluginEvent; @@ -203,6 +204,7 @@ class Server { )); $this->server->addPlugin(new CopyEtagHeaderPlugin()); + $this->server->addPlugin(new ChunkingV2Plugin()); $this->server->addPlugin(new ChunkingPlugin()); // allow setup of additional plugins diff --git a/apps/dav/lib/Upload/ChunkingV2Plugin.php b/apps/dav/lib/Upload/ChunkingV2Plugin.php new file mode 100644 index 0000000000..c964ad22f7 --- /dev/null +++ b/apps/dav/lib/Upload/ChunkingV2Plugin.php @@ -0,0 +1,223 @@ + + * + * @author Julius Härtl + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\DAV\Upload; + +use OC\Files\View; +use OCA\DAV\Connector\Sabre\Directory; +use OCP\Files\Storage\IChunkedFileWrite; +use OCP\Files\Storage\IStorage; +use OCP\Files\StorageInvalidException; +use Sabre\DAV\Exception\BadRequest; +use Sabre\DAV\Exception\NotFound; +use Sabre\DAV\Server; +use Sabre\DAV\ServerPlugin; +use Sabre\HTTP\RequestInterface; +use Sabre\HTTP\ResponseInterface; +use Sabre\Uri; + +class ChunkingV2Plugin extends ServerPlugin { + + /** @var Server */ + private $server; + /** @var UploadFolder */ + private $uploadFolder; + + private const TEMP_TARGET = '.target'; + + private const OBJECT_UPLOAD_TARGET = '{http://nextcloud.org/ns}upload-target'; + private const OBJECT_UPLOAD_CHUNKTOKEN = '{http://nextcloud.org/ns}upload-chunktoken'; + + private const DESTINATION_HEADER = 'X-Chunking-Destination'; + + /** + * @inheritdoc + */ + public function initialize(Server $server) { + $server->on('afterMethod:MKCOL', [$this, 'beforeMkcol']); + // 200 priority to call after the custom properties backend is registered + $server->on('beforeMethod:PUT', [$this, 'beforePut'], 200); + $server->on('beforeMethod:DELETE', [$this, 'beforeDelete'], 200); + $server->on('beforeMove', [$this, 'beforeMove'], 90); + + $this->server = $server; + } + + /** + * @param string $path + * @param bool $createIfNotExists + * @return FutureFile|UploadFile|\Sabre\DAV\ICollection|\Sabre\DAV\INode + */ + private function getTargetFile(string $path, bool $createIfNotExists = false) { + try { + $targetFile = $this->server->tree->getNodeForPath($path); + } catch (NotFound $e) { + if ($createIfNotExists) { + $this->uploadFolder->createFile(self::TEMP_TARGET); + } + $targetFile = $this->uploadFolder->getChild(self::TEMP_TARGET); + } + return $targetFile; + } + + public function beforeMkcol(RequestInterface $request, ResponseInterface $response): bool { + $this->uploadFolder = $this->server->tree->getNodeForPath($request->getPath()); + try { + $this->checkPrerequisites(); + $storage = $this->getStorage(); + } catch (StorageInvalidException | BadRequest $e) { + return true; + } + + $targetPath = $this->server->httpRequest->getHeader(self::DESTINATION_HEADER); + if (!$targetPath) { + return true; + } + + $targetFile = $this->getTargetFile($targetPath, true); + + $uploadId = $storage->beginChunkedFile($targetFile->getInternalPath()); + + // DAV properties on the UploadFolder are used in order to properly cleanup stale chunked file writes and to persist the target path + $this->server->updateProperties($request->getPath(), [ + self::OBJECT_UPLOAD_CHUNKTOKEN => $uploadId, + self::OBJECT_UPLOAD_TARGET => $targetPath, + ]); + + $response->setStatus(201); + return true; + } + + public function beforePut(RequestInterface $request, ResponseInterface $response): bool { + $this->uploadFolder = $this->server->tree->getNodeForPath(dirname($request->getPath())); + try { + $this->checkPrerequisites(); + $storage = $this->getStorage(); + } catch (StorageInvalidException | BadRequest $e) { + return true; + } + + $properties = $this->server->getProperties(dirname($request->getPath()) . '/', [ self::OBJECT_UPLOAD_CHUNKTOKEN, self::OBJECT_UPLOAD_TARGET ]); + $targetPath = $properties[self::OBJECT_UPLOAD_TARGET]; + $uploadId = $properties[self::OBJECT_UPLOAD_CHUNKTOKEN]; + $partId = (int)basename($request->getPath()); + + if (!($partId >= 1 && $partId <= 10000)) { + throw new BadRequest('Invalid chunk id'); + } + + $targetFile = $this->getTargetFile($targetPath); + $stream = $request->getBodyAsStream(); + $storage->putChunkedFilePart($targetFile->getInternalPath(), $uploadId, (string)$partId, $stream, (int)$request->getHeader('Content-Length')); + + $response->setStatus(201); + return false; + } + + public function beforeMove($sourcePath, $destination): bool { + $this->uploadFolder = $this->server->tree->getNodeForPath(dirname($sourcePath)); + try { + $this->checkPrerequisites(); + $this->getStorage(); + } catch (StorageInvalidException | BadRequest $e) { + return true; + } + $properties = $this->server->getProperties(dirname($sourcePath) . '/', [ self::OBJECT_UPLOAD_CHUNKTOKEN, self::OBJECT_UPLOAD_TARGET ]); + $targetPath = $properties[self::OBJECT_UPLOAD_TARGET]; + $uploadId = $properties[self::OBJECT_UPLOAD_CHUNKTOKEN]; + + $targetFile = $this->getTargetFile($targetPath); + + [$destinationDir, $destinationName] = Uri\split($destination); + /** @var Directory $destinationParent */ + $destinationParent = $this->server->tree->getNodeForPath($destinationDir); + $destinationExists = $destinationParent->childExists($destinationName); + + $rootView = new View(); + $rootView->writeChunkedFile($targetFile->getAbsoluteInternalPath(), $uploadId); + if (!$destinationExists) { + $destinationInView = $destinationParent->getFileInfo()->getPath() . '/' . $destinationName; + $rootView->rename($targetFile->getAbsoluteInternalPath(), $destinationInView); + } + + $sourceNode = $this->server->tree->getNodeForPath($sourcePath); + if ($sourceNode instanceof FutureFile) { + $sourceNode->delete(); + } + + $this->server->emit('afterMove', [$sourcePath, $destination]); + $this->server->emit('afterUnbind', [$sourcePath]); + $this->server->emit('afterBind', [$destination]); + + $response = $this->server->httpResponse; + $response->setHeader('Content-Length', '0'); + $response->setStatus($destinationExists ? 204 : 201); + return false; + } + + public function beforeDelete(RequestInterface $request, ResponseInterface $response) { + $this->uploadFolder = $this->server->tree->getNodeForPath($request->getPath()); + try { + if (!$this->uploadFolder instanceof UploadFolder) { + return true; + } + $storage = $this->getStorage(); + } catch (StorageInvalidException | BadRequest $e) { + return true; + } + + $properties = $this->server->getProperties($request->getPath() . '/', [ self::OBJECT_UPLOAD_CHUNKTOKEN, self::OBJECT_UPLOAD_TARGET ]); + $targetPath = $properties[self::OBJECT_UPLOAD_TARGET]; + $uploadId = $properties[self::OBJECT_UPLOAD_CHUNKTOKEN]; + if (!$targetPath || !$uploadId) { + return true; + } + $targetFile = $this->getTargetFile($targetPath); + $storage->cancelChunkedFile($targetFile->getInternalPath(), $uploadId); + return true; + } + + /** @throws BadRequest */ + private function checkPrerequisites(): void { + if (!$this->uploadFolder instanceof UploadFolder || !$this->server->httpRequest->getHeader(self::DESTINATION_HEADER)) { + throw new BadRequest('Chunking destination header not set'); + } + } + + /** + * @return IChunkedFileWrite + * @throws BadRequest + * @throws StorageInvalidException + */ + private function getStorage(): IStorage { + $this->checkPrerequisites(); + $storage = $this->uploadFolder->getStorage(); + if (!$storage->instanceOfStorage(IChunkedFileWrite::class)) { + throw new StorageInvalidException('Storage does not support chunked file write'); + } + /** @var IChunkedFileWrite $storage */ + return $storage; + } +} diff --git a/apps/dav/lib/Upload/FutureFile.php b/apps/dav/lib/Upload/FutureFile.php index ef7d7ac9fd..ce4e585bca 100644 --- a/apps/dav/lib/Upload/FutureFile.php +++ b/apps/dav/lib/Upload/FutureFile.php @@ -68,6 +68,10 @@ class FutureFile implements \Sabre\DAV\IFile { return AssemblyStream::wrap($nodes); } + public function getPath() { + return $this->root->getFileInfo()->getInternalPath() . '/.file'; + } + /** * @inheritdoc */ diff --git a/apps/dav/lib/Upload/UploadFile.php b/apps/dav/lib/Upload/UploadFile.php index dabddeb143..84d8f3d7eb 100644 --- a/apps/dav/lib/Upload/UploadFile.php +++ b/apps/dav/lib/Upload/UploadFile.php @@ -73,4 +73,12 @@ class UploadFile implements IFile { public function getLastModified() { return $this->file->getLastModified(); } + + public function getInternalPath(): string { + return $this->file->getInternalPath(); + } + + public function getAbsoluteInternalPath(): string { + return $this->file->getFileInfo()->getPath(); + } } diff --git a/apps/dav/lib/Upload/UploadFolder.php b/apps/dav/lib/Upload/UploadFolder.php index b983458b95..0ece63b7e0 100644 --- a/apps/dav/lib/Upload/UploadFolder.php +++ b/apps/dav/lib/Upload/UploadFolder.php @@ -26,6 +26,7 @@ namespace OCA\DAV\Upload; use OCA\DAV\Connector\Sabre\Directory; +use OCP\Files\Storage\IStorage; use Sabre\DAV\Exception\Forbidden; use Sabre\DAV\ICollection; @@ -35,10 +36,13 @@ class UploadFolder implements ICollection { private $node; /** @var CleanupService */ private $cleanupService; + /** @var IStorage */ + private $storage; - public function __construct(Directory $node, CleanupService $cleanupService) { + public function __construct(Directory $node, CleanupService $cleanupService, IStorage $storage) { $this->node = $node; $this->cleanupService = $cleanupService; + $this->storage = $storage; } public function createFile($name, $data = null) { @@ -95,4 +99,8 @@ class UploadFolder implements ICollection { public function getLastModified() { return $this->node->getLastModified(); } + + public function getStorage() { + return $this->storage; + } } diff --git a/apps/dav/lib/Upload/UploadHome.php b/apps/dav/lib/Upload/UploadHome.php index b25a2824d3..58ba204256 100644 --- a/apps/dav/lib/Upload/UploadHome.php +++ b/apps/dav/lib/Upload/UploadHome.php @@ -56,12 +56,12 @@ class UploadHome implements ICollection { } public function getChild($name): UploadFolder { - return new UploadFolder($this->impl()->getChild($name), $this->cleanupService); + return new UploadFolder($this->impl()->getChild($name), $this->cleanupService, $this->getStorage()); } public function getChildren(): array { return array_map(function ($node) { - return new UploadFolder($node, $this->cleanupService); + return new UploadFolder($node, $this->cleanupService, $this->getStorage()); }, $this->impl()->getChildren()); } @@ -90,14 +90,24 @@ class UploadHome implements ICollection { * @return Directory */ private function impl() { + $view = $this->getView(); + $rootInfo = $view->getFileInfo(''); + return new Directory($view, $rootInfo); + } + + private function getView() { $rootView = new View(); $user = \OC::$server->getUserSession()->getUser(); Filesystem::initMountPoints($user->getUID()); if (!$rootView->file_exists('/' . $user->getUID() . '/uploads')) { $rootView->mkdir('/' . $user->getUID() . '/uploads'); } - $view = new View('/' . $user->getUID() . '/uploads'); - $rootInfo = $view->getFileInfo(''); - return new Directory($view, $rootInfo); + return new View('/' . $user->getUID() . '/uploads'); + } + + private function getStorage() { + $view = $this->getView(); + $storage = $view->getFileInfo('')->getStorage(); + return $storage; } } diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 225d191f64..c89ce27785 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -965,6 +965,11 @@ null + + + Uri\split($destination) + + \Sabre\Uri\split($this->principalInfo['uri']) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 010c2a53a3..da9a4d2dc8 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -294,6 +294,7 @@ return array( 'OCP\\Files\\Notify\\INotifyHandler' => $baseDir . '/lib/public/Files/Notify/INotifyHandler.php', 'OCP\\Files\\Notify\\IRenameChange' => $baseDir . '/lib/public/Files/Notify/IRenameChange.php', 'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php', + 'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php', 'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php', 'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php', 'OCP\\Files\\Search\\ISearchComparison' => $baseDir . '/lib/public/Files/Search/ISearchComparison.php', @@ -311,6 +312,7 @@ return array( 'OCP\\Files\\StorageInvalidException' => $baseDir . '/lib/public/Files/StorageInvalidException.php', 'OCP\\Files\\StorageNotAvailableException' => $baseDir . '/lib/public/Files/StorageNotAvailableException.php', 'OCP\\Files\\StorageTimeoutException' => $baseDir . '/lib/public/Files/StorageTimeoutException.php', + 'OCP\\Files\\Storage\\IChunkedFileWrite' => $baseDir . '/lib/public/Files/Storage/IChunkedFileWrite.php', 'OCP\\Files\\Storage\\IDisableEncryptionStorage' => $baseDir . '/lib/public/Files/Storage/IDisableEncryptionStorage.php', 'OCP\\Files\\Storage\\ILockingStorage' => $baseDir . '/lib/public/Files/Storage/ILockingStorage.php', 'OCP\\Files\\Storage\\INotifyStorage' => $baseDir . '/lib/public/Files/Storage/INotifyStorage.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 1af3458d99..4f98df8d4e 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -323,6 +323,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OCP\\Files\\Notify\\INotifyHandler' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/INotifyHandler.php', 'OCP\\Files\\Notify\\IRenameChange' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/IRenameChange.php', 'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php', + 'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php', 'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php', 'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php', 'OCP\\Files\\Search\\ISearchComparison' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchComparison.php', @@ -340,6 +341,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OCP\\Files\\StorageInvalidException' => __DIR__ . '/../../..' . '/lib/public/Files/StorageInvalidException.php', 'OCP\\Files\\StorageNotAvailableException' => __DIR__ . '/../../..' . '/lib/public/Files/StorageNotAvailableException.php', 'OCP\\Files\\StorageTimeoutException' => __DIR__ . '/../../..' . '/lib/public/Files/StorageTimeoutException.php', + 'OCP\\Files\\Storage\\IChunkedFileWrite' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IChunkedFileWrite.php', 'OCP\\Files\\Storage\\IDisableEncryptionStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IDisableEncryptionStorage.php', 'OCP\\Files\\Storage\\ILockingStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/ILockingStorage.php', 'OCP\\Files\\Storage\\INotifyStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/INotifyStorage.php', diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index 5c792e59a3..aeca933870 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -30,19 +30,27 @@ namespace OC\Files\ObjectStore; +use Aws\S3\Exception\S3Exception; +use Aws\S3\Exception\S3MultipartUploadException; use Icewind\Streams\CallbackWrapper; use Icewind\Streams\CountWrapper; use Icewind\Streams\IteratorDirectory; use OC\Files\Cache\Cache; use OC\Files\Cache\CacheEntry; use OC\Files\Storage\PolyFill\CopyDirectory; +use OC\Memcache\ArrayCache; +use OC\Memcache\NullCache; use OCP\Files\Cache\ICacheEntry; use OCP\Files\FileInfo; +use OCP\Files\GenericFileException; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; +use OCP\Files\Storage\IChunkedFileWrite; use OCP\Files\Storage\IStorage; +use OCP\ICache; -class ObjectStoreStorage extends \OC\Files\Storage\Common { +class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite { use CopyDirectory; /** @@ -62,6 +70,9 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { private $logger; + /** @var ICache */ + private $uploadCache; + public function __construct($params) { if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) { $this->objectStore = $params['objectstore']; @@ -82,11 +93,11 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { } $this->logger = \OC::$server->getLogger(); + $this->uploadCache = \OC::$server->getMemCacheFactory()->createDistributed('objectstore'); } public function mkdir($path) { $path = $this->normalizePath($path); - if ($this->file_exists($path)) { return false; } @@ -601,4 +612,103 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common { throw $e; } } + + public function beginChunkedFile(string $targetPath): string { + $this->validateUploadCache(); + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $uploadId = $this->objectStore->initiateMultipartUpload($urn); + $this->uploadCache->set($this->getUploadCacheKey($urn, $uploadId, 'uploadId'), $uploadId); + return $uploadId; + } + + /** + * + * @throws GenericFileException + */ + public function putChunkedFilePart(string $targetPath, string $writeToken, string $chunkId, $data, $size = null): void { + $this->validateUploadCache(); + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $uploadId = $this->uploadCache->get($this->getUploadCacheKey($urn, $writeToken, 'uploadId')); + + $result = $this->objectStore->uploadMultipartPart($urn, $uploadId, (int)$chunkId, $data, $size); + + $parts = $this->uploadCache->get($this->getUploadCacheKey($urn, $uploadId, 'parts')); + if (!$parts) { + $parts = []; + } + $parts[$chunkId] = [ + 'PartNumber' => $chunkId, + 'ETag' => trim($result->get('ETag'), '"') + ]; + $this->uploadCache->set($this->getUploadCacheKey($urn, $uploadId, 'parts'), $parts); + } + + public function writeChunkedFile(string $targetPath, string $writeToken): int { + $this->validateUploadCache(); + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $uploadId = $this->uploadCache->get($this->getUploadCacheKey($urn, $writeToken, 'uploadId')); + $parts = $this->uploadCache->get($this->getUploadCacheKey($urn, $uploadId, 'parts')); + try { + $size = $this->objectStore->completeMultipartUpload($urn, $uploadId, array_values($parts)); + $stat = $this->stat($targetPath); + $mtime = time(); + if (is_array($stat)) { + $stat['size'] = $size; + $stat['mtime'] = $mtime; + $stat['mimetype'] = $this->getMimeType($targetPath); + $this->getCache()->update($stat['fileid'], $stat); + } + } catch (S3MultipartUploadException | S3Exception $e) { + $this->objectStore->abortMultipartUpload($urn, $uploadId); + $this->logger->logException($e, [ + 'app' => 'objectstore', + 'message' => 'Could not compete multipart upload ' . $urn. ' with uploadId ' . $uploadId + ]); + throw new GenericFileException('Could not write chunked file'); + } finally { + $this->clearCache($urn, $uploadId); + } + return $size; + } + + public function cancelChunkedFile(string $targetPath, string $writeToken): void { + $this->validateUploadCache(); + if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) { + throw new GenericFileException('Object store does not support multipart upload'); + } + $cacheEntry = $this->getCache()->get($targetPath); + $urn = $this->getURN($cacheEntry->getId()); + $uploadId = $this->uploadCache->get($this->getUploadCacheKey($urn, $writeToken, 'uploadId')); + $this->objectStore->abortMultipartUpload($urn, $uploadId); + $this->clearCache($urn, $uploadId); + } + + /** + * @throws GenericFileException + */ + private function validateUploadCache(): void { + if ($this->uploadCache instanceof NullCache || $this->uploadCache instanceof ArrayCache) { + throw new GenericFileException('ChunkedFileWrite not available: A cross-request persistent cache is required'); + } + } + + private function getUploadCacheKey($urn, $uploadId, $chunkId = null): string { + return $urn . '-' . $uploadId . '-' . ($chunkId ? $chunkId . '-' : ''); + } + + private function clearCache($urn, $uploadId): void { + $this->uploadCache->clear($this->getUploadCacheKey($urn, $uploadId)); + } } diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 3d1a658eb9..507ba710d2 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -24,9 +24,12 @@ namespace OC\Files\ObjectStore; +use Aws\Result; +use Exception; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; -class S3 implements IObjectStore { +class S3 implements IObjectStore, IObjectStoreMultiPartUpload { use S3ConnectionTrait; use S3ObjectTrait; @@ -41,4 +44,49 @@ class S3 implements IObjectStore { public function getStorageId() { return $this->id; } + + public function initiateMultipartUpload(string $urn): string { + $upload = $this->getConnection()->createMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ]); + $uploadId = $upload->get('UploadId'); + if ($uploadId === null) { + throw new Exception('No upload id returned'); + } + return (string)$uploadId; + } + + public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result { + return $this->getConnection()->uploadPart([ + 'Body' => $stream, + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'ContentLength' => $size, + 'PartNumber' => $partId, + 'UploadId' => $uploadId, + ]); + } + + public function completeMultipartUpload(string $urn, string $uploadId, array $result): int { + $this->getConnection()->completeMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId, + 'MultipartUpload' => ['Parts' => $result], + ]); + $stat = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ]); + return (int)$stat->get('ContentLength'); + } + + public function abortMultipartUpload($urn, $uploadId): void { + $this->getConnection()->abortMultipartUpload([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'UploadId' => $uploadId + ]); + } } diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index a59f735c10..1cba397bc3 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -62,7 +62,9 @@ use OCP\Files\InvalidPathException; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\ReservedWordException; +use OCP\Files\Storage\IChunkedFileWrite; use OCP\Files\Storage\IStorage; +use OCP\Files\StorageInvalidException; use OCP\ILogger; use OCP\IUser; use OCP\Lock\ILockingProvider; @@ -704,6 +706,22 @@ class View { } } + /** + * @param string $path + * @param string $chunkToken + * @return false|mixed|null + * @throws LockedException + * @throws StorageInvalidException + */ + public function writeChunkedFile(string $path, string $chunkToken) { + /** @var IStorage|null $storage */ + [$storage, ] = Filesystem::resolvePath($path); + if (!$storage || !$storage->instanceOfStorage(IChunkedFileWrite::class)) { + throw new StorageInvalidException('path is not a chunked file write storage'); + } + return $this->basicOperation('writeChunkedFile', $path, ['update', 'write'], $chunkToken); + } + /** * @param string $path * @return bool|mixed diff --git a/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php b/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php new file mode 100644 index 0000000000..cd94fdc184 --- /dev/null +++ b/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php @@ -0,0 +1,54 @@ + + * + * @author Julius Härtl + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +declare(strict_types=1); + + +namespace OCP\Files\ObjectStore; + +use Aws\Result; + +/** + * @since 22.0.0 + */ +interface IObjectStoreMultiPartUpload { + /** + * @since 22.0.0 + */ + public function initiateMultipartUpload(string $urn): string; + + /** + * @since 22.0.0 + */ + public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result; + + /** + * @since 22.0.0 + */ + public function completeMultipartUpload(string $urn, string $uploadId, array $result): int; + + /** + * @since 22.0.0 + */ + public function abortMultipartUpload(string $urn, string $uploadId): void; +} diff --git a/lib/public/Files/Storage/IChunkedFileWrite.php b/lib/public/Files/Storage/IChunkedFileWrite.php new file mode 100644 index 0000000000..10eeac2ab4 --- /dev/null +++ b/lib/public/Files/Storage/IChunkedFileWrite.php @@ -0,0 +1,71 @@ + + * + * @author Julius Härtl + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +declare(strict_types=1); + + +namespace OCP\Files\Storage; + +use OCP\Files\GenericFileException; + +/** + * @since 22.0.0 + */ +interface IChunkedFileWrite extends IStorage { + + /** + * @param string $targetPath Relative target path in the storage + * @return string writeToken to be used with the other methods to uniquely identify the file write operation + * @throws GenericFileException + * @since 22.0.0 + */ + public function beginChunkedFile(string $targetPath): string; + + /** + * @param string $targetPath + * @param string $writeToken + * @param string $chunkId + * @param resource $data + * @param int|null $size + * @throws GenericFileException + * @since 22.0.0 + */ + public function putChunkedFilePart(string $targetPath, string $writeToken, string $chunkId, $data, int $size = null): void; + + /** + * @param string $targetPath + * @param string $writeToken + * @return int + * @throws GenericFileException + * @since 22.0.0 + */ + public function writeChunkedFile(string $targetPath, string $writeToken): int; + + /** + * @param string $targetPath + * @param string $writeToken + * @throws GenericFileException + * @since 22.0.0 + */ + public function cancelChunkedFile(string $targetPath, string $writeToken): void; +}