Compare commits

...

18 Commits

Author SHA1 Message Date
Julius Härtl b45ba6a94b
Adapt multipart upload to use sse-kms parameters
Signed-off-by: Julius Härtl <jus@bitgrid.net>
2021-06-02 15:34:05 +02:00
Julius Härtl c62afe0128
Move dav property deletion to dedicated service to cleanup on background job
Signed-off-by: Julius Härtl <jus@bitgrid.net>
2021-06-02 15:34:05 +02:00
Julius Härtl 37c5f0a11b
First attempt to make multipartPart upload working
Signed-off-by: Julius Härtl <jus@bitgrid.net>
2021-06-02 15:34:05 +02:00
Bernd.Rederlechner@t-systems.com ea92ba6f02 Disable object uploader experiment
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-06-01 12:04:19 +00:00
Bernd.Rederlechner@t-systems.com 1b19e0ebbf Experiment with ObjectUploader
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-27 14:11:53 +00:00
Bernd.Rederlechner@t-systems.com 2805536f7d Correct bucket key headers
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-27 14:11:12 +00:00
Bernd.Rederlechner@t-systems.com 5809f7a7ac Reduce sending of kms key to minimum in requests
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-18 07:13:30 +00:00
Bernd.Rederlechner@t-systems.com f9c92477b9 Backport key handling fix from s3ssekms 2021-05-17 16:16:48 +00:00
Bernd.Rederlechner@t-systems.com 62555d7c0f Clean up with fixer
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-12 13:24:36 +00:00
Bernd.Rederlechner@t-systems.com b5dfb01dc5 Add lacking cleanup for some objects created.
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-12 13:24:36 +00:00
Bernd.Rederlechner@t-systems.com f2256a2c11 Add empty file test
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-12 13:24:36 +00:00
Bernd.Rederlechner@t-systems.com 7d3b5956e4 Change fragment handling strategy for empty files
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-12 13:24:36 +00:00
Bernd.Rederlechner@t-systems.com 828dde4453 Prohibit fragments for empty streams
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-12 13:24:36 +00:00
Bernd.Rederlechner@t-systems.com 667d589688 Remove redundant checking
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-12 13:24:36 +00:00
Bernd.Rederlechner@t-systems.com 4f4fc6350b Fix wrong class reference name to make test run
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-12 13:24:36 +00:00
Bernd.Rederlechner@t-systems.com 6b3f4ff761 Fix syntax problem
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-07 12:46:22 +00:00
Bernd.Rederlechner@t-systems.com f139d2bf9d Keep miemtype handling for writeObject
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-07 08:52:47 +00:00
tsdicloud cafb437e11 Add S3 SSE KMS key and bucketked encryption
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
2021-05-06 15:26:53 +00:00
28 changed files with 1149 additions and 259 deletions

View File

@ -248,6 +248,7 @@ return array(
'OCA\\DAV\\Search\\EventsSearchProvider' => $baseDir . '/../lib/Search/EventsSearchProvider.php',
'OCA\\DAV\\Search\\TasksSearchProvider' => $baseDir . '/../lib/Search/TasksSearchProvider.php',
'OCA\\DAV\\Server' => $baseDir . '/../lib/Server.php',
'OCA\\DAV\\Service\\CustomPropertiesService' => $baseDir . '/../lib/Service/CustomPropertiesService.php',
'OCA\\DAV\\Settings\\CalDAVSettings' => $baseDir . '/../lib/Settings/CalDAVSettings.php',
'OCA\\DAV\\Storage\\PublicOwnerWrapper' => $baseDir . '/../lib/Storage/PublicOwnerWrapper.php',
'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => $baseDir . '/../lib/SystemTag/SystemTagMappingNode.php',
@ -260,6 +261,7 @@ return array(
'OCA\\DAV\\Traits\\PrincipalProxyTrait' => $baseDir . '/../lib/Traits/PrincipalProxyTrait.php',
'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
'OCA\\DAV\\Upload\\ChunkingV2Plugin' => $baseDir . '/../lib/Upload/ChunkingV2Plugin.php',
'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',

View File

@ -263,6 +263,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\Search\\EventsSearchProvider' => __DIR__ . '/..' . '/../lib/Search/EventsSearchProvider.php',
'OCA\\DAV\\Search\\TasksSearchProvider' => __DIR__ . '/..' . '/../lib/Search/TasksSearchProvider.php',
'OCA\\DAV\\Server' => __DIR__ . '/..' . '/../lib/Server.php',
'OCA\\DAV\\Service\\CustomPropertiesService' => __DIR__ . '/..' . '/../lib/Service/CustomPropertiesService.php',
'OCA\\DAV\\Settings\\CalDAVSettings' => __DIR__ . '/..' . '/../lib/Settings/CalDAVSettings.php',
'OCA\\DAV\\Storage\\PublicOwnerWrapper' => __DIR__ . '/..' . '/../lib/Storage/PublicOwnerWrapper.php',
'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagMappingNode.php',
@ -275,6 +276,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\Traits\\PrincipalProxyTrait' => __DIR__ . '/..' . '/../lib/Traits/PrincipalProxyTrait.php',
'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
'OCA\\DAV\\Upload\\ChunkingV2Plugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingV2Plugin.php',
'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',

View File

@ -29,6 +29,7 @@ declare(strict_types=1);
namespace OCA\DAV\BackgroundJob;
use OC\User\NoUserException;
use OCA\DAV\Service\CustomPropertiesService;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\BackgroundJob\TimedJob;
@ -45,10 +46,14 @@ class UploadCleanup extends TimedJob {
/** @var IJobList */
private $jobList;
public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) {
/** @var CustomPropertiesService */
private $customPropertiesService;
public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList, CustomPropertiesService $customPropertiesService) {
parent::__construct($time);
$this->rootFolder = $rootFolder;
$this->jobList = $jobList;
$this->customPropertiesService = $customPropertiesService;
// Run once a day
$this->setInterval(60 * 60 * 24);
@ -72,6 +77,9 @@ class UploadCleanup extends TimedJob {
$files = $uploadFolder->getDirectoryListing();
$davPath = 'uploads/' . $uid . '/' . $uploadFolder->getName();
$this->customPropertiesService->delete($uid, $davPath);
// Remove if all files have an mtime of more than a day
$time = $this->time->getTime() - 60 * 60 * 24;

View File

@ -38,6 +38,7 @@ use OC\Files\View;
use OCA\DAV\Connector\Sabre\Exception\FileLocked;
use OCA\DAV\Connector\Sabre\Exception\Forbidden;
use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
use OCA\DAV\Upload\FutureFile;
use OCP\Files\FileInfo;
use OCP\Files\ForbiddenException;
use OCP\Files\InvalidPathException;

View File

@ -248,6 +248,14 @@ abstract class Node implements \Sabre\DAV\INode {
return $this->info->getId();
}
public function getInternalPath(): string {
return $this->info->getInternalPath();
}
public function getAbsoluteInternalPath(): string {
return $this->info->getPath();
}
/**
* @param string $user
* @return int

View File

@ -35,6 +35,7 @@ namespace OCA\DAV\Connector\Sabre;
use OC\Files\Node\Folder;
use OCA\DAV\AppInfo\PluginManager;
use OCA\DAV\Files\BrowserErrorPagePlugin;
use OCA\DAV\Service\CustomPropertiesService;
use OCP\Files\Mount\IMountManager;
use OCP\IConfig;
use OCP\IDBConnection;
@ -204,6 +205,7 @@ class ServerFactory {
new \OCA\DAV\DAV\CustomPropertiesBackend(
$objectTree,
$this->databaseConnection,
\OC::$server->get(CustomPropertiesService::class),
$this->userSession->getUser()
)
)

View File

@ -26,6 +26,7 @@
namespace OCA\DAV\DAV;
use OCA\DAV\Connector\Sabre\Node;
use OCA\DAV\Service\CustomPropertiesService;
use OCP\IDBConnection;
use OCP\IUser;
use Sabre\DAV\PropertyStorage\Backend\BackendInterface;
@ -63,6 +64,11 @@ class CustomPropertiesBackend implements BackendInterface {
*/
private $connection;
/**
* @var CustomPropertiesService
*/
private $customPropertiesService;
/**
* @var IUser
*/
@ -83,9 +89,11 @@ class CustomPropertiesBackend implements BackendInterface {
public function __construct(
Tree $tree,
IDBConnection $connection,
CustomPropertiesService $customPropertiesService,
IUser $user) {
$this->tree = $tree;
$this->connection = $connection;
$this->customPropertiesService = $customPropertiesService;
$this->user = $user;
}
@ -155,12 +163,7 @@ class CustomPropertiesBackend implements BackendInterface {
* @param string $path path of node for which to delete properties
*/
public function delete($path) {
$statement = $this->connection->prepare(
'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
);
$statement->execute([$this->user->getUID(), $this->formatPath($path)]);
$statement->closeCursor();
$this->customPropertiesService->delete($this->user->getUID(), $path);
unset($this->cache[$path]);
}

View File

@ -64,8 +64,10 @@ use OCA\DAV\Events\SabrePluginAuthInitEvent;
use OCA\DAV\Files\BrowserErrorPagePlugin;
use OCA\DAV\Files\LazySearchBackend;
use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
use OCA\DAV\Service\CustomPropertiesService;
use OCA\DAV\SystemTag\SystemTagPlugin;
use OCA\DAV\Upload\ChunkingPlugin;
use OCA\DAV\Upload\ChunkingV2Plugin;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\IRequest;
use OCP\SabrePluginEvent;
@ -203,6 +205,7 @@ class Server {
));
$this->server->addPlugin(new CopyEtagHeaderPlugin());
$this->server->addPlugin(new ChunkingV2Plugin());
$this->server->addPlugin(new ChunkingPlugin());
// allow setup of additional plugins
@ -248,6 +251,7 @@ class Server {
new CustomPropertiesBackend(
$this->server->tree,
\OC::$server->getDatabaseConnection(),
\OC::$server->get(CustomPropertiesService::class),
\OC::$server->getUserSession()->getUser()
)
)

View File

@ -0,0 +1,60 @@
<?php
/*
* @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
*
* @author Julius Härtl <jus@bitgrid.net>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
declare(strict_types=1);
namespace OCA\DAV\Service;
use OCP\IDBConnection;
class CustomPropertiesService {
/** @var IDBConnection */
private $connection;
public function __construct(IDBConnection $connection) {
$this->connection = $connection;
}
public function delete(string $userId, string $path): void {
$statement = $this->connection->prepare(
'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
);
$result = $statement->execute([$userId, $this->formatPath($path)]);
$result->closeCursor();
}
/**
* long paths are hashed to ensure they fit in the database
*
* @param string $path
* @return string
*/
private function formatPath(string $path): string {
if (strlen($path) > 250) {
return sha1($path);
}
return $path;
}
}

View File

@ -0,0 +1,223 @@
<?php
declare(strict_types=1);
/*
* @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
*
* @author Julius Härtl <jus@bitgrid.net>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Upload;
use OC\Files\View;
use OCA\DAV\Connector\Sabre\Directory;
use OCP\Files\Storage\IChunkedFileWrite;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageInvalidException;
use Sabre\DAV\Exception\BadRequest;
use Sabre\DAV\Exception\NotFound;
use Sabre\DAV\Server;
use Sabre\DAV\ServerPlugin;
use Sabre\HTTP\RequestInterface;
use Sabre\HTTP\ResponseInterface;
use Sabre\Uri;
class ChunkingV2Plugin extends ServerPlugin {
/** @var Server */
private $server;
/** @var UploadFolder */
private $uploadFolder;
private const TEMP_TARGET = '.target';
private const OBJECT_UPLOAD_TARGET = '{http://nextcloud.org/ns}upload-target';
private const OBJECT_UPLOAD_CHUNKTOKEN = '{http://nextcloud.org/ns}upload-chunktoken';
private const DESTINATION_HEADER = 'X-Chunking-Destination';
/**
* @inheritdoc
*/
public function initialize(Server $server) {
$server->on('afterMethod:MKCOL', [$this, 'beforeMkcol']);
// 200 priority to call after the custom properties backend is registered
$server->on('beforeMethod:PUT', [$this, 'beforePut'], 200);
$server->on('beforeMethod:DELETE', [$this, 'beforeDelete'], 200);
$server->on('beforeMove', [$this, 'beforeMove'], 90);
$this->server = $server;
}
/**
* @param string $path
* @param bool $createIfNotExists
* @return FutureFile|UploadFile|\Sabre\DAV\ICollection|\Sabre\DAV\INode
*/
private function getTargetFile(string $path, bool $createIfNotExists = false) {
try {
$targetFile = $this->server->tree->getNodeForPath($path);
} catch (NotFound $e) {
if ($createIfNotExists) {
$this->uploadFolder->createFile(self::TEMP_TARGET);
}
$targetFile = $this->uploadFolder->getChild(self::TEMP_TARGET);
}
return $targetFile;
}
public function beforeMkcol(RequestInterface $request, ResponseInterface $response): bool {
$this->uploadFolder = $this->server->tree->getNodeForPath($request->getPath());
try {
$this->checkPrerequisites();
$storage = $this->getStorage();
} catch (StorageInvalidException | BadRequest $e) {
return true;
}
$targetPath = $this->server->httpRequest->getHeader(self::DESTINATION_HEADER);
if (!$targetPath) {
return true;
}
$targetFile = $this->getTargetFile($targetPath, true);
$uploadId = $storage->beginChunkedFile($targetFile->getInternalPath());
// DAV properties on the UploadFolder are used in order to properly cleanup stale chunked file writes and to persist the target path
$this->server->updateProperties($request->getPath(), [
self::OBJECT_UPLOAD_CHUNKTOKEN => $uploadId,
self::OBJECT_UPLOAD_TARGET => $targetPath,
]);
$response->setStatus(201);
return true;
}
public function beforePut(RequestInterface $request, ResponseInterface $response): bool {
$this->uploadFolder = $this->server->tree->getNodeForPath(dirname($request->getPath()));
try {
$this->checkPrerequisites();
$storage = $this->getStorage();
} catch (StorageInvalidException | BadRequest $e) {
return true;
}
$properties = $this->server->getProperties(dirname($request->getPath()) . '/', [ self::OBJECT_UPLOAD_CHUNKTOKEN, self::OBJECT_UPLOAD_TARGET ]);
$targetPath = $properties[self::OBJECT_UPLOAD_TARGET];
$uploadId = $properties[self::OBJECT_UPLOAD_CHUNKTOKEN];
$partId = (int)basename($request->getPath());
if (!($partId >= 1 && $partId <= 10000)) {
throw new BadRequest('Invalid chunk id');
}
$targetFile = $this->getTargetFile($targetPath);
$stream = $request->getBodyAsStream();
$storage->putChunkedFilePart($targetFile->getInternalPath(), $uploadId, (string)$partId, $stream, (int)$request->getHeader('Content-Length'));
$response->setStatus(201);
return false;
}
public function beforeMove($sourcePath, $destination): bool {
$this->uploadFolder = $this->server->tree->getNodeForPath(dirname($sourcePath));
try {
$this->checkPrerequisites();
$this->getStorage();
} catch (StorageInvalidException | BadRequest $e) {
return true;
}
$properties = $this->server->getProperties(dirname($sourcePath) . '/', [ self::OBJECT_UPLOAD_CHUNKTOKEN, self::OBJECT_UPLOAD_TARGET ]);
$targetPath = $properties[self::OBJECT_UPLOAD_TARGET];
$uploadId = $properties[self::OBJECT_UPLOAD_CHUNKTOKEN];
$targetFile = $this->getTargetFile($targetPath);
[$destinationDir, $destinationName] = Uri\split($destination);
/** @var Directory $destinationParent */
$destinationParent = $this->server->tree->getNodeForPath($destinationDir);
$destinationExists = $destinationParent->childExists($destinationName);
$rootView = new View();
$rootView->writeChunkedFile($targetFile->getAbsoluteInternalPath(), $uploadId);
if (!$destinationExists) {
$destinationInView = $destinationParent->getFileInfo()->getPath() . '/' . $destinationName;
$rootView->rename($targetFile->getAbsoluteInternalPath(), $destinationInView);
}
$sourceNode = $this->server->tree->getNodeForPath($sourcePath);
if ($sourceNode instanceof FutureFile) {
$sourceNode->delete();
}
$this->server->emit('afterMove', [$sourcePath, $destination]);
$this->server->emit('afterUnbind', [$sourcePath]);
$this->server->emit('afterBind', [$destination]);
$response = $this->server->httpResponse;
$response->setHeader('Content-Length', '0');
$response->setStatus($destinationExists ? 204 : 201);
return false;
}
public function beforeDelete(RequestInterface $request, ResponseInterface $response) {
$this->uploadFolder = $this->server->tree->getNodeForPath($request->getPath());
try {
if (!$this->uploadFolder instanceof UploadFolder) {
return true;
}
$storage = $this->getStorage();
} catch (StorageInvalidException | BadRequest $e) {
return true;
}
$properties = $this->server->getProperties($request->getPath() . '/', [ self::OBJECT_UPLOAD_CHUNKTOKEN, self::OBJECT_UPLOAD_TARGET ]);
$targetPath = $properties[self::OBJECT_UPLOAD_TARGET];
$uploadId = $properties[self::OBJECT_UPLOAD_CHUNKTOKEN];
if (!$targetPath || !$uploadId) {
return true;
}
$targetFile = $this->getTargetFile($targetPath);
$storage->cancelChunkedFile($targetFile->getInternalPath(), $uploadId);
return true;
}
/** @throws BadRequest */
private function checkPrerequisites(): void {
if (!$this->uploadFolder instanceof UploadFolder || !$this->server->httpRequest->getHeader(self::DESTINATION_HEADER)) {
throw new BadRequest('Chunking destination header not set');
}
}
/**
* @return IChunkedFileWrite
* @throws BadRequest
* @throws StorageInvalidException
*/
private function getStorage(): IStorage {
$this->checkPrerequisites();
$storage = $this->uploadFolder->getStorage();
if (!$storage->instanceOfStorage(IChunkedFileWrite::class)) {
throw new StorageInvalidException('Storage does not support chunked file write');
}
/** @var IChunkedFileWrite $storage */
return $storage;
}
}

View File

@ -68,6 +68,10 @@ class FutureFile implements \Sabre\DAV\IFile {
return AssemblyStream::wrap($nodes);
}
public function getPath() {
return $this->root->getFileInfo()->getInternalPath() . '/.file';
}
/**
* @inheritdoc
*/

View File

@ -73,4 +73,12 @@ class UploadFile implements IFile {
public function getLastModified() {
return $this->file->getLastModified();
}
public function getInternalPath(): string {
return $this->file->getInternalPath();
}
public function getAbsoluteInternalPath(): string {
return $this->file->getFileInfo()->getPath();
}
}

View File

@ -26,6 +26,7 @@
namespace OCA\DAV\Upload;
use OCA\DAV\Connector\Sabre\Directory;
use OCP\Files\Storage\IStorage;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\ICollection;
@ -35,10 +36,13 @@ class UploadFolder implements ICollection {
private $node;
/** @var CleanupService */
private $cleanupService;
/** @var IStorage */
private $storage;
public function __construct(Directory $node, CleanupService $cleanupService) {
public function __construct(Directory $node, CleanupService $cleanupService, IStorage $storage) {
$this->node = $node;
$this->cleanupService = $cleanupService;
$this->storage = $storage;
}
public function createFile($name, $data = null) {
@ -95,4 +99,8 @@ class UploadFolder implements ICollection {
public function getLastModified() {
return $this->node->getLastModified();
}
public function getStorage() {
return $this->storage;
}
}

View File

@ -56,12 +56,12 @@ class UploadHome implements ICollection {
}
public function getChild($name): UploadFolder {
return new UploadFolder($this->impl()->getChild($name), $this->cleanupService);
return new UploadFolder($this->impl()->getChild($name), $this->cleanupService, $this->getStorage());
}
public function getChildren(): array {
return array_map(function ($node) {
return new UploadFolder($node, $this->cleanupService);
return new UploadFolder($node, $this->cleanupService, $this->getStorage());
}, $this->impl()->getChildren());
}
@ -90,14 +90,24 @@ class UploadHome implements ICollection {
* @return Directory
*/
private function impl() {
$view = $this->getView();
$rootInfo = $view->getFileInfo('');
return new Directory($view, $rootInfo);
}
private function getView() {
$rootView = new View();
$user = \OC::$server->getUserSession()->getUser();
Filesystem::initMountPoints($user->getUID());
if (!$rootView->file_exists('/' . $user->getUID() . '/uploads')) {
$rootView->mkdir('/' . $user->getUID() . '/uploads');
}
$view = new View('/' . $user->getUID() . '/uploads');
$rootInfo = $view->getFileInfo('');
return new Directory($view, $rootInfo);
return new View('/' . $user->getUID() . '/uploads');
}
private function getStorage() {
$view = $this->getView();
$storage = $view->getFileInfo('')->getStorage();
return $storage;
}
}

View File

@ -37,6 +37,7 @@ namespace OCA\DAV\Tests\unit\Connector\Sabre;
use OCA\DAV\Connector\Sabre\Directory;
use OCA\DAV\Connector\Sabre\File;
use OCA\DAV\Service\CustomPropertiesService;
use OCP\IUser;
use Sabre\DAV\Tree;
@ -88,6 +89,7 @@ class CustomPropertiesBackendTest extends \Test\TestCase {
$this->plugin = new \OCA\DAV\DAV\CustomPropertiesBackend(
$this->tree,
\OC::$server->getDatabaseConnection(),
$this->createMock(CustomPropertiesService::class),
$this->user
);
}

View File

@ -29,8 +29,10 @@
namespace OCA\DAV\Tests\DAV;
use OCA\DAV\DAV\CustomPropertiesBackend;
use OCA\DAV\Service\CustomPropertiesService;
use OCP\IDBConnection;
use OCP\IUser;
use PHPUnit\Framework\MockObject\MockObject;
use Sabre\DAV\PropFind;
use Sabre\DAV\PropPatch;
use Sabre\DAV\Tree;
@ -41,16 +43,19 @@ use Test\TestCase;
*/
class CustomPropertiesBackendTest extends TestCase {
/** @var Tree | \PHPUnit\Framework\MockObject\MockObject */
/** @var Tree | MockObject */
private $tree;
/** @var IDBConnection */
private $dbConnection;
/** @var IUser | \PHPUnit\Framework\MockObject\MockObject */
/** @var CustomPropertiesService */
private $customPropertiesService;
/** @var IUser | MockObject */
private $user;
/** @var CustomPropertiesBackend | \PHPUnit\Framework\MockObject\MockObject */
/** @var CustomPropertiesBackend */
private $backend;
protected function setUp(): void {
@ -62,10 +67,12 @@ class CustomPropertiesBackendTest extends TestCase {
->with()
->willReturn('dummy_user_42');
$this->dbConnection = \OC::$server->getDatabaseConnection();
$this->customPropertiesService = new CustomPropertiesService($this->dbConnection);
$this->backend = new CustomPropertiesBackend(
$this->tree,
$this->dbConnection,
$this->customPropertiesService,
$this->user
);
}
@ -123,9 +130,11 @@ class CustomPropertiesBackendTest extends TestCase {
public function testPropFindNoDbCalls() {
$db = $this->createMock(IDBConnection::class);
$service = new CustomPropertiesService($db);
$backend = new CustomPropertiesBackend(
$this->tree,
$db,
$service,
$this->user
);

View File

@ -970,6 +970,11 @@
<code>null</code>
</NullArgument>
</file>
<file src="apps/dav/lib/Upload/ChunkingV2Plugin.php">
<UndefinedFunction occurrences="1">
<code>Uri\split($destination)</code>
</UndefinedFunction>
</file>
<file src="apps/dav/lib/Upload/UploadHome.php">
<UndefinedFunction occurrences="1">
<code>\Sabre\Uri\split($this-&gt;principalInfo['uri'])</code>

View File

@ -294,6 +294,7 @@ return array(
'OCP\\Files\\Notify\\INotifyHandler' => $baseDir . '/lib/public/Files/Notify/INotifyHandler.php',
'OCP\\Files\\Notify\\IRenameChange' => $baseDir . '/lib/public/Files/Notify/IRenameChange.php',
'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php',
'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php',
'OCP\\Files\\Search\\ISearchComparison' => $baseDir . '/lib/public/Files/Search/ISearchComparison.php',
@ -311,6 +312,7 @@ return array(
'OCP\\Files\\StorageInvalidException' => $baseDir . '/lib/public/Files/StorageInvalidException.php',
'OCP\\Files\\StorageNotAvailableException' => $baseDir . '/lib/public/Files/StorageNotAvailableException.php',
'OCP\\Files\\StorageTimeoutException' => $baseDir . '/lib/public/Files/StorageTimeoutException.php',
'OCP\\Files\\Storage\\IChunkedFileWrite' => $baseDir . '/lib/public/Files/Storage/IChunkedFileWrite.php',
'OCP\\Files\\Storage\\IDisableEncryptionStorage' => $baseDir . '/lib/public/Files/Storage/IDisableEncryptionStorage.php',
'OCP\\Files\\Storage\\ILockingStorage' => $baseDir . '/lib/public/Files/Storage/ILockingStorage.php',
'OCP\\Files\\Storage\\INotifyStorage' => $baseDir . '/lib/public/Files/Storage/INotifyStorage.php',

View File

@ -323,6 +323,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OCP\\Files\\Notify\\INotifyHandler' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/INotifyHandler.php',
'OCP\\Files\\Notify\\IRenameChange' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/IRenameChange.php',
'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php',
'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php',
'OCP\\Files\\Search\\ISearchComparison' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchComparison.php',
@ -340,6 +341,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OCP\\Files\\StorageInvalidException' => __DIR__ . '/../../..' . '/lib/public/Files/StorageInvalidException.php',
'OCP\\Files\\StorageNotAvailableException' => __DIR__ . '/../../..' . '/lib/public/Files/StorageNotAvailableException.php',
'OCP\\Files\\StorageTimeoutException' => __DIR__ . '/../../..' . '/lib/public/Files/StorageTimeoutException.php',
'OCP\\Files\\Storage\\IChunkedFileWrite' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IChunkedFileWrite.php',
'OCP\\Files\\Storage\\IDisableEncryptionStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IDisableEncryptionStorage.php',
'OCP\\Files\\Storage\\ILockingStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/ILockingStorage.php',
'OCP\\Files\\Storage\\INotifyStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/INotifyStorage.php',

View File

@ -30,19 +30,27 @@
namespace OC\Files\ObjectStore;
use Aws\S3\Exception\S3Exception;
use Aws\S3\Exception\S3MultipartUploadException;
use Icewind\Streams\CallbackWrapper;
use Icewind\Streams\CountWrapper;
use Icewind\Streams\IteratorDirectory;
use OC\Files\Cache\Cache;
use OC\Files\Cache\CacheEntry;
use OC\Files\Storage\PolyFill\CopyDirectory;
use OC\Memcache\ArrayCache;
use OC\Memcache\NullCache;
use OCP\Files\Cache\ICacheEntry;
use OCP\Files\FileInfo;
use OCP\Files\GenericFileException;
use OCP\Files\NotFoundException;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
use OCP\Files\Storage\IChunkedFileWrite;
use OCP\Files\Storage\IStorage;
use OCP\ICache;
class ObjectStoreStorage extends \OC\Files\Storage\Common {
class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite {
use CopyDirectory;
/**
@ -62,6 +70,9 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
private $logger;
/** @var ICache */
private $uploadCache;
public function __construct($params) {
if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) {
$this->objectStore = $params['objectstore'];
@ -82,11 +93,11 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
}
$this->logger = \OC::$server->getLogger();
$this->uploadCache = \OC::$server->getMemCacheFactory()->createDistributed('objectstore');
}
public function mkdir($path) {
$path = $this->normalizePath($path);
if ($this->file_exists($path)) {
return false;
}
@ -598,4 +609,103 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
throw $e;
}
}
public function beginChunkedFile(string $targetPath): string {
$this->validateUploadCache();
if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
throw new GenericFileException('Object store does not support multipart upload');
}
$cacheEntry = $this->getCache()->get($targetPath);
$urn = $this->getURN($cacheEntry->getId());
$uploadId = $this->objectStore->initiateMultipartUpload($urn);
$this->uploadCache->set($this->getUploadCacheKey($urn, $uploadId, 'uploadId'), $uploadId);
return $uploadId;
}
/**
*
* @throws GenericFileException
*/
public function putChunkedFilePart(string $targetPath, string $writeToken, string $chunkId, $data, $size = null): void {
$this->validateUploadCache();
if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
throw new GenericFileException('Object store does not support multipart upload');
}
$cacheEntry = $this->getCache()->get($targetPath);
$urn = $this->getURN($cacheEntry->getId());
$uploadId = $this->uploadCache->get($this->getUploadCacheKey($urn, $writeToken, 'uploadId'));
$result = $this->objectStore->uploadMultipartPart($urn, $uploadId, (int)$chunkId, $data, $size);
$parts = $this->uploadCache->get($this->getUploadCacheKey($urn, $uploadId, 'parts'));
if (!$parts) {
$parts = [];
}
$parts[$chunkId] = [
'PartNumber' => $chunkId,
'ETag' => trim($result->get('ETag'), '"')
];
$this->uploadCache->set($this->getUploadCacheKey($urn, $uploadId, 'parts'), $parts);
}
public function writeChunkedFile(string $targetPath, string $writeToken): int {
$this->validateUploadCache();
if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
throw new GenericFileException('Object store does not support multipart upload');
}
$cacheEntry = $this->getCache()->get($targetPath);
$urn = $this->getURN($cacheEntry->getId());
$uploadId = $this->uploadCache->get($this->getUploadCacheKey($urn, $writeToken, 'uploadId'));
$parts = $this->uploadCache->get($this->getUploadCacheKey($urn, $uploadId, 'parts'));
try {
$size = $this->objectStore->completeMultipartUpload($urn, $uploadId, array_values($parts));
$stat = $this->stat($targetPath);
$mtime = time();
if (is_array($stat)) {
$stat['size'] = $size;
$stat['mtime'] = $mtime;
$stat['mimetype'] = $this->getMimeType($targetPath);
$this->getCache()->update($stat['fileid'], $stat);
}
} catch (S3MultipartUploadException | S3Exception $e) {
$this->objectStore->abortMultipartUpload($urn, $uploadId);
$this->logger->logException($e, [
'app' => 'objectstore',
'message' => 'Could not compete multipart upload ' . $urn. ' with uploadId ' . $uploadId
]);
throw new GenericFileException('Could not write chunked file');
} finally {
$this->clearCache($urn, $uploadId);
}
return $size;
}
public function cancelChunkedFile(string $targetPath, string $writeToken): void {
$this->validateUploadCache();
if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
throw new GenericFileException('Object store does not support multipart upload');
}
$cacheEntry = $this->getCache()->get($targetPath);
$urn = $this->getURN($cacheEntry->getId());
$uploadId = $this->uploadCache->get($this->getUploadCacheKey($urn, $writeToken, 'uploadId'));
$this->objectStore->abortMultipartUpload($urn, $uploadId);
$this->clearCache($urn, $uploadId);
}
/**
* @throws GenericFileException
*/
private function validateUploadCache(): void {
if ($this->uploadCache instanceof NullCache || $this->uploadCache instanceof ArrayCache) {
throw new GenericFileException('ChunkedFileWrite not available: A cross-request persistent cache is required');
}
}
private function getUploadCacheKey($urn, $uploadId, $chunkId = null): string {
return $urn . '-' . $uploadId . '-' . ($chunkId ? $chunkId . '-' : '');
}
private function clearCache($urn, $uploadId): void {
$this->uploadCache->clear($this->getUploadCacheKey($urn, $uploadId));
}
}

View File

@ -24,9 +24,12 @@
namespace OC\Files\ObjectStore;
use Aws\Result;
use Exception;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
class S3 implements IObjectStore {
class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
use S3ConnectionTrait;
use S3ObjectTrait;
@ -41,4 +44,49 @@ class S3 implements IObjectStore {
public function getStorageId() {
return $this->id;
}
public function initiateMultipartUpload(string $urn): string {
$upload = $this->getConnection()->createMultipartUpload([
'Bucket' => $this->bucket,
'Key' => $urn,
] + $this->getSseKmsPutParameters());
$uploadId = $upload->get('UploadId');
if ($uploadId === null) {
throw new Exception('No upload id returned');
}
return (string)$uploadId;
}
public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result {
return $this->getConnection()->uploadPart([
'Body' => $stream,
'Bucket' => $this->bucket,
'Key' => $urn,
'ContentLength' => $size,
'PartNumber' => $partId,
'UploadId' => $uploadId,
]);
}
public function completeMultipartUpload(string $urn, string $uploadId, array $result): int {
$this->getConnection()->completeMultipartUpload([
'Bucket' => $this->bucket,
'Key' => $urn,
'UploadId' => $uploadId,
'MultipartUpload' => ['Parts' => $result],
] + $this->getSseKmsPutParameters());
$stat = $this->getConnection()->headObject([
'Bucket' => $this->bucket,
'Key' => $urn,
]);
return (int)$stat->get('ContentLength');
}
public function abortMultipartUpload($urn, $uploadId): void {
$this->getConnection()->abortMultipartUpload([
'Bucket' => $this->bucket,
'Key' => $urn,
'UploadId' => $uploadId
]);
}
}

View File

@ -1,7 +1,6 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @author Arthur Schiwon <blizzz@arthur-schiwon.de>
* @author Christoph Wurst <christoph@winzerhof-wurst.at>
* @author Florent <florent@coppint.com>
@ -10,7 +9,6 @@
* @author Roeland Jago Douma <roeland@famdouma.nl>
* @author S. Cat <33800996+sparrowjack63@users.noreply.github.com>
* @author Stephen Cuppett <steve@cuppett.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
@ -25,15 +23,14 @@
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;
use Aws\ClientResolver;
use Aws\Credentials\CredentialProvider;
use Aws\Credentials\EcsCredentialProvider;
use Aws\Credentials\Credentials;
use Aws\Credentials\EcsCredentialProvider;
use Aws\Exception\CredentialsException;
use Aws\S3\Exception\S3Exception;
use Aws\S3\S3Client;
@ -41,6 +38,7 @@ use GuzzleHttp\Promise;
use GuzzleHttp\Promise\RejectedPromise;
use OCP\ILogger;
trait S3ConnectionTrait {
/** @var array */
protected $params;
@ -60,11 +58,17 @@ trait S3ConnectionTrait {
/** @var int */
protected $uploadPartSize;
/** @var string */
protected $sseKmsKeyId;
/** @var bool */
protected $sseUseBucketKey;
protected $test;
protected function parseParams($params) {
if (empty($params['bucket'])) {
throw new \Exception("Bucket has to be configured.");
throw new \Exception('Bucket has to be configured.');
}
$this->id = 'amazon::' . $params['bucket'];
@ -78,7 +82,14 @@ trait S3ConnectionTrait {
if (!isset($params['port']) || $params['port'] === '') {
$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
}
$params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists'];
$params['autocreate'] = !isset($params['autocreate']) ? false : $params['autocreate'];
// this avoid at least the hash lookups for each read/weite operation
if (isset($params['ssekmskeyid'])) {
$this->sseKmsKeyId = $params['ssekmskeyid'];
}
$this->sseUseBucketKey = (isset($params['sseusebucketkey'])) ? $params['sseusebucketkey'] : false;
$this->params = $params;
}
@ -87,9 +98,122 @@ trait S3ConnectionTrait {
}
/**
* Returns the connection
* Add the SSE KMS parameterdepending on the
* KMS encryption strategy (bucket, individual or
* no encryption) for object creations.
*
* @return array with encryption parameters
*/
public function getSseKmsPutParameters(): array {
if ($this->sseUseBucketKey) {
return [
'ServerSideEncryption' => 'aws:kms',
'BucketKeyEnabled' => true,
];
} elseif (!empty($this->sseKmsKeyId)) {
return [
'ServerSideEncryption' => 'aws:kms',
'BucketKeyEnabled' => false,
'SSEKMSKeyId' => $this->sseKmsKeyId,
];
} else {
return [];
}
}
/**
* Add the SSE KMS parameter depending on the
* KMS encryption strategy (bucket, individual or
* no encryption) for object read.
*
* @return array with encryption parameters
*/
public function getSseKmsGetParameters(): array {
if (($this->sseUseBucketKey) || !empty($this->sseKmsKeyId)) {
return [
'ServerSideEncryption' => 'aws:kms',
];
} else {
return [];
}
}
/**
* Create the required bucket
*
* @throws \Exception if bucket creation fails
*/
protected function createNewBucket() {
$logger = \OC::$server->getLogger();
try {
$logger->info('Bucket "'.$this->bucket.'" does not exist - creating it.', ['app' => 'objectstore']);
if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
throw new \Exception('The bucket will not be created because the name is not dns compatible, please correct it: '.$this->bucket);
}
$this->connection->createBucket(['Bucket' => $this->bucket]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->logException($e, [
'message' => 'Invalid remote storage.',
'level' => ILogger::DEBUG,
'app' => 'objectstore',
]);
throw new \Exception('Creation of bucket "'.$this->bucket.'" failed. '.$e->getMessage());
}
}
/**
* Check bucket key consistency or put bucket key if missing
* This operation only works for bucket owner or with
* s3:GetEncryptionConfiguration/s3:PutEncryptionConfiguration permission
*
* We recommend to use autocreate only on initial setup and
* use an S3:user only with object operation permission and no bucket operation permissions
* later with autocreate=false
*
* @throws \Exception if bucket key config is inconsistent or if putting the key fails
*/
protected function checkOrPutBucketKey() {
$logger = \OC::$server->getLogger();
try {
$encrypt_state = $this->connection->getBucketEncryption([
'Bucket' => $this->bucket,
]);
} catch (S3Exception $e) {
try {
$logger->info('Bucket key for "'.$this->bucket.'" is not set - adding it.', ['app' => 'objectstore']);
$this->connection->putBucketEncryption([
'Bucket' => $this->bucket,
'ServerSideEncryptionConfiguration' => [
'Rules' => [
[
'ApplyServerSideEncryptionByDefault' => [
'SSEAlgorithm' => 'aws:kms',
'KMSMasterKeyID' => $this->sseKmsKeyId,
],
],
],
],
]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->logException($e, [
'message' => 'Bucket key problem.',
'level' => ILogger::DEBUG,
'app' => 'objectstore',
]);
throw new \Exception('Putting configured bucket key to "'.$this->bucket.'" failed. '.$e->getMessage());
}
}
}
/**
* Returns the connection.
*
* @return S3Client connected client
*
* @throws \Exception if connection could not be made
*/
public function getConnection() {
@ -98,7 +222,7 @@ trait S3ConnectionTrait {
}
$scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https';
$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
$base_url = $scheme.'://'.$this->params['hostname'].':'.$this->params['port'].'/';
// Adding explicit credential provider to the beginning chain.
// Including environment variables and IAM instance profiles.
@ -121,6 +245,7 @@ trait S3ConnectionTrait {
'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false,
'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()),
'csm' => false,
// 'debug' => true, // to debug S3 communication
];
if (isset($this->params['proxy'])) {
$options['request.options'] = ['proxy' => $this->params['proxy']];
@ -132,27 +257,16 @@ trait S3ConnectionTrait {
if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
$logger = \OC::$server->getLogger();
$logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.',
['app' => 'objectstore']);
$logger->debug('Bucket "' . $this->bucket.'" This bucket name is not dns compatible, it may contain invalid characters.',
['app' => 'objectstore']);
}
if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) {
$logger = \OC::$server->getLogger();
try {
$logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket);
}
$this->connection->createBucket(['Bucket' => $this->bucket]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->logException($e, [
'message' => 'Invalid remote storage.',
'level' => ILogger::DEBUG,
'app' => 'objectstore',
]);
throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
}
if ($this->params['autocreate'] && !$this->connection->doesBucketExist($this->bucket)) {
$this->createNewBucket();
}
if ($this->params['autocreate'] && $this->sseUseBucketKey) {
$this->checkOrPutBucketKey();
}
// google cloud's s3 compatibility doesn't like the EncodingType parameter
@ -164,7 +278,7 @@ trait S3ConnectionTrait {
}
/**
* when running the tests wait to let the buckets catch up
* when running the tests wait to let the buckets catch up.
*/
private function testTimeout() {
if ($this->test) {
@ -183,9 +297,9 @@ trait S3ConnectionTrait {
}
/**
* This function creates a credential provider based on user parameter file
* This function creates a credential provider based on user parameter file.
*/
protected function paramCredentialProvider() : callable {
protected function paramCredentialProvider(): callable {
return function () {
$key = empty($this->params['key']) ? null : $this->params['key'];
$secret = empty($this->params['secret']) ? null : $this->params['secret'];
@ -197,6 +311,7 @@ trait S3ConnectionTrait {
}
$msg = 'Could not find parameters set for credentials in config file.';
return new RejectedPromise(new CredentialsException($msg));
};
}

View File

@ -1,13 +1,11 @@
<?php
/**
* @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl>
*
* @author Christoph Wurst <christoph@winzerhof-wurst.at>
* @author Florent <florent@coppint.com>
* @author Morris Jobke <hey@morrisjobke.de>
* @author Robin Appelman <robin@icewind.nl>
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
@ -22,7 +20,6 @@
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;
@ -31,75 +28,114 @@ use Aws\S3\Exception\S3MultipartUploadException;
use Aws\S3\MultipartUploader;
use Aws\S3\ObjectUploader;
use Aws\S3\S3Client;
use GuzzleHttp\Psr7\CachingStream;
use Icewind\Streams\CallbackWrapper;
use OC\Files\Stream\SeekableHttpStream;
trait S3ObjectTrait {
/**
* Returns the connection
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
*/
abstract protected function getConnection();
trait S3ObjectTrait
{
/**
* Returns the connection.
*
* @return S3Client connected client
*
* @throws \Exception if connection could not be made
*/
abstract protected function getConnection();
/**
* @param string $urn the unified resource name used to identify the object
* @return resource stream with the read data
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
public function readObject($urn) {
return SeekableHttpStream::open(function ($range) use ($urn) {
$command = $this->getConnection()->getCommand('GetObject', [
'Bucket' => $this->bucket,
'Key' => $urn,
'Range' => 'bytes=' . $range,
]);
$request = \Aws\serialize($command);
$headers = [];
foreach ($request->getHeaders() as $key => $values) {
foreach ($values as $value) {
$headers[] = "$key: $value";
}
}
$opts = [
'http' => [
'protocol_version' => 1.1,
'header' => $headers,
],
];
/* compute configured encryption headers for put operations */
abstract protected function getSseKmsPutParameters();
$context = stream_context_create($opts);
return fopen($request->getUri(), 'r', false, $context);
});
}
/* compute configured encryption headers for get operations */
abstract protected function getSseKmsGetParameters();
/**
* @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write
* @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
public function writeObject($urn, $stream, string $mimetype = null) {
$count = 0;
$countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) {
$count += $read;
});
/**
* @param string $urn the unified resource name used to identify the object
*
* @return resource stream with the read data
*
* @throws \Exception when something goes wrong, message will be logged
*
* @since 7.0.0
*/
public function readObject($urn)
{
return SeekableHttpStream::open(function ($range) use ($urn) {
$s3params = [
'Bucket' => $this->bucket,
'Key' => $urn,
'Range' => 'bytes='.$range,
] + $this->getSseKmsGetParameters();
$command = $this->getConnection()->getCommand('GetObject', $s3params);
$request = \Aws\serialize($command);
$headers = [];
foreach ($request->getHeaders() as $key => $values) {
foreach ($values as $value) {
$headers[] = "$key: $value";
}
}
$opts = [
'http' => [
'protocol_version' => 1.1,
'header' => $headers,
],
];
$uploader = new MultipartUploader($this->getConnection(), $countStream, [
'bucket' => $this->bucket,
'key' => $urn,
'part_size' => $this->uploadPartSize,
'params' => [
'ContentType' => $mimetype
]
]);
$context = stream_context_create($opts);
return fopen($request->getUri(), 'r', false, $context);
});
}
/**
* @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write
* @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
public function writeObject($urn, $stream, string $mimetype = null)
{
//$streamMeta = stream_get_meta_data($stream);
//if ($streamMeta['seekable']) {
// the ObjectUploader requires the stream seekable for objects <5MB
// to copute checksum before uploading
//$stream = new CachingStream($stream);
//}
$count = 0;
$countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) {
$count += $read;
});
$s3params = [
'bucket' => $this->bucket,
'key' => $urn,
'part_size' => $this->uploadPartSize,
'params' => [
'ContentType' => $mimetype
] + $this->getSseKmsPutParameters(),
];
// ObjectUplader version
//$s3params = [
// 'part_size' => $this->uploadPartSize,
// 'params' => [
// 'ContentType' => $mimetype
// ] + $this->getSseKmsPutParameters(),
//];
// maybe, we should also use ObjectUploader here in the future
// it does direct uploads for small files < 5MB and multipart otherwise
//$uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, $countStream, 'private', $s3params);
$uploader = new MultipartUploader($this->getConnection(), $countStream, $s3params);
try {
$uploader->upload();
} catch (S3MultipartUploadException $e) {
// if anything goes wrong with multipart, make sure that you don´t poison and
// slow down s3 bucket with fragment management
$this->getConnection()->abortMultipartUpload($uploader->getState()->getId());
try {
$uploader->upload();
} catch (S3MultipartUploadException $e) {
// This is an empty file so just touch it then
if ($count === 0 && feof($countStream)) {
$uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '');
@ -107,27 +143,46 @@ trait S3ObjectTrait {
} else {
throw $e;
}
}
}
}
//finally {
// this handles [S3] fclose(): supplied resource is not a valid stream resource #23373
// see https://stackoverflow.com/questions/11247507/fclose-18-is-not-a-valid-stream-resource/11247555
// which also recommends the solution
// if (is_resource($stream)) {
// fclose($stream);
// }
//}
}
/**
* @param string $urn the unified resource name used to identify the object
* @return void
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
public function deleteObject($urn) {
$this->getConnection()->deleteObject([
'Bucket' => $this->bucket,
'Key' => $urn,
]);
}
/**
* @param string $urn the unified resource name used to identify the object
*
* @return void
*
* @throws \Exception when something goes wrong, message will be logged
*
* @since 7.0.0
*/
public function deleteObject($urn)
{
$this->getConnection()->deleteObject([
'Bucket' => $this->bucket,
'Key' => $urn,
]);
}
public function objectExists($urn) {
return $this->getConnection()->doesObjectExist($this->bucket, $urn);
}
public function objectExists($urn)
{
return $this->getConnection()->doesObjectExist($this->bucket, $urn);
}
public function copyObject($from, $to) {
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to);
}
/**
* S3 copy command with SSE KMS key handling.
*/
public function copyObject($from, $to)
{
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [
'params' => $this->getSseKmsPutParameters(),
]);
}
}

View File

@ -62,7 +62,9 @@ use OCP\Files\InvalidPathException;
use OCP\Files\Mount\IMountPoint;
use OCP\Files\NotFoundException;
use OCP\Files\ReservedWordException;
use OCP\Files\Storage\IChunkedFileWrite;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageInvalidException;
use OCP\ILogger;
use OCP\IUser;
use OCP\Lock\ILockingProvider;
@ -704,6 +706,22 @@ class View {
}
}
/**
* @param string $path
* @param string $chunkToken
* @return false|mixed|null
* @throws LockedException
* @throws StorageInvalidException
*/
public function writeChunkedFile(string $path, string $chunkToken) {
/** @var IStorage|null $storage */
[$storage, ] = Filesystem::resolvePath($path);
if (!$storage || !$storage->instanceOfStorage(IChunkedFileWrite::class)) {
throw new StorageInvalidException('path is not a chunked file write storage');
}
return $this->basicOperation('writeChunkedFile', $path, ['update', 'write'], $chunkToken);
}
/**
* @param string $path
* @return bool|mixed

View File

@ -0,0 +1,54 @@
<?php
/*
* @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
*
* @author Julius Härtl <jus@bitgrid.net>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
declare(strict_types=1);
namespace OCP\Files\ObjectStore;
use Aws\Result;
/**
* @since 22.0.0
*/
interface IObjectStoreMultiPartUpload {
/**
* @since 22.0.0
*/
public function initiateMultipartUpload(string $urn): string;
/**
* @since 22.0.0
*/
public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result;
/**
* @since 22.0.0
*/
public function completeMultipartUpload(string $urn, string $uploadId, array $result): int;
/**
* @since 22.0.0
*/
public function abortMultipartUpload(string $urn, string $uploadId): void;
}

View File

@ -0,0 +1,71 @@
<?php
/*
* @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
*
* @author Julius Härtl <jus@bitgrid.net>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
declare(strict_types=1);
namespace OCP\Files\Storage;
use OCP\Files\GenericFileException;
/**
* @since 22.0.0
*/
interface IChunkedFileWrite extends IStorage {
/**
* @param string $targetPath Relative target path in the storage
* @return string writeToken to be used with the other methods to uniquely identify the file write operation
* @throws GenericFileException
* @since 22.0.0
*/
public function beginChunkedFile(string $targetPath): string;
/**
* @param string $targetPath
* @param string $writeToken
* @param string $chunkId
* @param resource $data
* @param int|null $size
* @throws GenericFileException
* @since 22.0.0
*/
public function putChunkedFilePart(string $targetPath, string $writeToken, string $chunkId, $data, int $size = null): void;
/**
* @param string $targetPath
* @param string $writeToken
* @return int
* @throws GenericFileException
* @since 22.0.0
*/
public function writeChunkedFile(string $targetPath, string $writeToken): int;
/**
* @param string $targetPath
* @param string $writeToken
* @throws GenericFileException
* @since 22.0.0
*/
public function cancelChunkedFile(string $targetPath, string $writeToken): void;
}

View File

@ -24,104 +24,115 @@ namespace Test\Files\ObjectStore;
use Test\TestCase;
abstract class ObjectStoreTest extends TestCase {
abstract class ObjectStoreTest extends TestCase
{
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
abstract protected function getInstance();
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
abstract protected function getInstance();
protected function stringToStream($data) {
$stream = fopen('php://temp', 'w+');
fwrite($stream, $data);
rewind($stream);
return $stream;
}
protected function stringToStream($data)
{
$stream = fopen('php://temp', 'w+');
fwrite($stream, $data);
rewind($stream);
return $stream;
}
public function testWriteRead() {
$stream = $this->stringToStream('foobar');
public function testWriteRead()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance = $this->getInstance();
$instance->writeObject('1', $stream);
$instance->writeObject('1', $stream);
$result = $instance->readObject('1');
$instance->deleteObject('1');
$result = $instance->readObject('1');
$instance->deleteObject('1');
$this->assertEquals('foobar', stream_get_contents($result));
}
$this->assertEquals('foobar', stream_get_contents($result));
}
public function testDelete() {
$stream = $this->stringToStream('foobar');
public function testDelete()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance = $this->getInstance();
$instance->writeObject('2', $stream);
$instance->writeObject('2', $stream);
$instance->deleteObject('2');
$instance->deleteObject('2');
try {
// to to read to verify that the object no longer exists
$instance->readObject('2');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
try {
// to to read to verify that the object no longer exists
$instance->readObject('2');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testReadNonExisting() {
$instance = $this->getInstance();
public function testReadNonExisting()
{
$instance = $this->getInstance();
try {
$instance->readObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
try {
$instance->readObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testDeleteNonExisting() {
$instance = $this->getInstance();
public function testDeleteNonExisting()
{
$instance = $this->getInstance();
try {
$instance->deleteObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
try {
$instance->deleteObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testExists() {
$stream = $this->stringToStream('foobar');
public function testExists()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$this->assertFalse($instance->objectExists('2'));
$instance = $this->getInstance();
$this->assertFalse($instance->objectExists('2'));
$instance->writeObject('2', $stream);
$instance->writeObject('2', $stream);
$this->assertTrue($instance->objectExists('2'));
$this->assertTrue($instance->objectExists('2'));
$instance->deleteObject('2');
$instance->deleteObject('2');
$this->assertFalse($instance->objectExists('2'));
}
$this->assertFalse($instance->objectExists('2'));
}
public function testCopy() {
$stream = $this->stringToStream('foobar');
public function testCopy()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance = $this->getInstance();
$instance->writeObject('source', $stream);
$instance->writeObject('source', $stream);
$this->assertFalse($instance->objectExists('target'));
$this->assertFalse($instance->objectExists('target'));
$instance->copyObject('source', 'target');
$instance->copyObject('source', 'target');
$this->assertTrue($instance->objectExists('target'));
$this->assertTrue($instance->objectExists('target'));
$this->assertEquals('foobar', stream_get_contents($instance->readObject('target')));
}
$this->assertEquals('foobar', stream_get_contents($instance->readObject('target')));
//$instance->deleteObject('source');
//$instance->deleteObject('target');
}
}

View File

@ -22,81 +22,126 @@
namespace Test\Files\ObjectStore;
use Icewind\Streams\Wrapper;
use Icewind\Streams\CallbackWrapper;
use OC\Files\ObjectStore\S3;
class MultiPartUploadS3 extends S3 {
public function writeObject($urn, $stream, string $mimetype = null) {
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
'mup_threshold' => 1,
]);
}
use Aws\S3\S3Client;
use Aws\S3\Exception\S3MultipartUploadException;
class MultiPartUploadS3 extends S3
{
public function writeObject($urn, $stream, string $mimetype = null)
{
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
'mup_threshold' => 1,
]);
}
}
class NonSeekableStream extends Wrapper {
public static function wrap($source) {
$context = stream_context_create([
'nonseek' => [
'source' => $source,
],
]);
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
}
class NonSeekableStream extends Wrapper
{
public static function wrap($source)
{
$context = stream_context_create([
'nonseek' => [
'source' => $source,
],
]);
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
}
public function dir_opendir($path, $options) {
return false;
}
public function dir_opendir($path, $options)
{
return false;
}
public function stream_open($path, $mode, $options, &$opened_path) {
$this->loadContext('nonseek');
return true;
}
public function stream_open($path, $mode, $options, &$opened_path)
{
$this->loadContext('nonseek');
return true;
}
public function stream_seek($offset, $whence = SEEK_SET) {
return false;
}
public function stream_seek($offset, $whence = SEEK_SET)
{
return false;
}
}
/**
* @group PRIMARY-s3
*/
class S3Test extends ObjectStoreTest {
protected function getInstance() {
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
class S3Test extends ObjectStoreTest
{
protected function getInstance()
{
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== '\\OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
return new S3($config['arguments']);
}
return new S3($config['arguments']);
}
public function testUploadNonSeekable() {
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
public function testUploadNonSeekable()
{
$s3 = $this->getInstance();
$s3 = $this->getInstance();
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
$result = $s3->readObject('multiparttest');
$result = $s3->readObject('multiparttest');
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
}
$s3->deleteObject('multiparttest');
}
public function testSeek() {
$data = file_get_contents(__FILE__);
public function testSeek()
{
$data = file_get_contents(__FILE__);
$instance = $this->getInstance();
$instance->writeObject('seek', $this->stringToStream($data));
$instance = $this->getInstance();
$instance->writeObject('seek', $this->stringToStream($data));
$read = $instance->readObject('seek');
$this->assertEquals(substr($data, 0, 100), fread($read, 100));
$read = $instance->readObject('seek');
$this->assertEquals(substr($data, 0, 100), fread($read, 100));
fseek($read, 10);
$this->assertEquals(substr($data, 10, 100), fread($read, 100));
fseek($read, 10);
$this->assertEquals(substr($data, 10, 100), fread($read, 100));
fseek($read, 100, SEEK_CUR);
$this->assertEquals(substr($data, 210, 100), fread($read, 100));
}
fseek($read, 100, SEEK_CUR);
$this->assertEquals(substr($data, 210, 100), fread($read, 100));
$instance->deleteObject('seek');
}
public function assertNoUpload($objectUrn)
{
$s3 = $this->getInstance();
$s3client = $s3->getConnection();
$uploads = $s3client->listMultipartUploads([
'Bucket' => $s3->getBucket(),
'Prefix' => $objectUrn,
]);
//fwrite(STDERR, print_r($uploads, TRUE));
$this->assertArrayNotHasKey('Uploads', $uploads);
}
public function testEmptyUpload()
{
//$this->expectException(S3MultipartUploadException::class);
$s3 = $this->getInstance();
// create an empty stream and check that it fits to the
// pre-conditions in writeObject for the empty case
$emptyStream = fopen("php://memory", "r");
fwrite($emptyStream, null);
$s3->writeObject('emptystream', $emptyStream);
// this method intendedly produces an S3Exception
$this->assertNoUpload('emptystream');
$this->assertTrue($s3->objectExists('emptystream'));
$s3->deleteObject('emptystream');
}
}