From e4e5e735db2ae1d73432a3ec1ebf1a6654f2eda8 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 18 Sep 2017 18:24:53 +0200 Subject: [PATCH 1/4] multipart upload for s3 object storage Signed-off-by: Robin Appelman --- .../Files/ObjectStore/S3ObjectTrait.php | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 3ba4da92b9..f7d9fb305c 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -21,6 +21,8 @@ namespace OC\Files\ObjectStore; +use Aws\Exception\MultipartUploadException; +use Aws\S3\MultipartUploader; use Aws\S3\S3Client; use Psr\Http\Message\StreamInterface; @@ -60,11 +62,26 @@ trait S3ObjectTrait { * @since 7.0.0 */ function writeObject($urn, $stream) { - $this->getConnection()->putObject([ - 'Bucket' => $this->bucket, - 'Key' => $urn, - 'Body' => $stream + $uploader = new MultipartUploader($this->getConnection(), $stream, [ + 'bucket' => $this->bucket, + 'key' => $urn, ]); + $tries = 0; + do { + try { + $result = $uploader->upload(); + } catch (MultipartUploadException $e) { + rewind($stream); + $tries++; + if ($tries < 5) { + $uploader = new MultipartUploader($this->getConnection(), $stream, [ + 'state' => $e->getState() + ]); + } else { + $this->getConnection()->abortMultipartUpload($e->getState()->getId()); + } + } + } while (!isset($result) && $tries < 5); } /** @@ -79,4 +96,4 @@ trait S3ObjectTrait { 'Key' => $urn ]); } -} \ No newline at end of file +} From 4ae46d887626630b675d132db3587d06de08d35d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 21 Sep 2017 14:06:59 +0200 Subject: [PATCH 2/4] only do multipart upload for large files Signed-off-by: Robin Appelman --- .../Files/ObjectStore/S3ObjectTrait.php | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index f7d9fb305c..6527614e8d 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -26,6 +26,8 @@ use Aws\S3\MultipartUploader; use Aws\S3\S3Client; use Psr\Http\Message\StreamInterface; +const S3_UPLOAD_PART_SIZE = 5368709120; + trait S3ObjectTrait { /** * Returns the connection @@ -62,17 +64,39 @@ trait S3ObjectTrait { * @since 7.0.0 */ function writeObject($urn, $stream) { + $stat = fstat($stream); + + if ($stat['size'] && $stat['size'] < S3_UPLOAD_PART_SIZE) { + $this->singlePartUpload($urn, $stream); + } else { + $this->multiPartUpload($urn, $stream); + } + + } + + private function singlePartUpload($urn, $stream) { + $this->getConnection()->putObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Body' => $stream + ]); + } + + private function multiPartUpload($urn, $stream) { $uploader = new MultipartUploader($this->getConnection(), $stream, [ 'bucket' => $this->bucket, 'key' => $urn, ]); + $tries = 0; + do { try { $result = $uploader->upload(); } catch (MultipartUploadException $e) { rewind($stream); $tries++; + if ($tries < 5) { $uploader = new MultipartUploader($this->getConnection(), $stream, [ 'state' => $e->getState() From 385d6f098c1466608d1a87312b208bbf22abeb0d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 21 Sep 2017 14:47:14 +0200 Subject: [PATCH 3/4] Add tests for multipart upload Signed-off-by: Robin Appelman --- .../Files/ObjectStore/S3ObjectTrait.php | 4 ++-- tests/lib/Files/ObjectStore/S3Test.php | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 6527614e8d..93204896c1 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -74,7 +74,7 @@ trait S3ObjectTrait { } - private function singlePartUpload($urn, $stream) { + protected function singlePartUpload($urn, $stream) { $this->getConnection()->putObject([ 'Bucket' => $this->bucket, 'Key' => $urn, @@ -82,7 +82,7 @@ trait S3ObjectTrait { ]); } - private function multiPartUpload($urn, $stream) { + protected function multiPartUpload($urn, $stream) { $uploader = new MultipartUploader($this->getConnection(), $stream, [ 'bucket' => $this->bucket, 'key' => $urn, diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php index b93e9beebd..14167656fb 100644 --- a/tests/lib/Files/ObjectStore/S3Test.php +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -23,19 +23,32 @@ namespace Test\Files\ObjectStore; use OC\Files\ObjectStore\S3; +class MultiPartUploadS3 extends S3 { + public function multiPartUpload($urn, $stream) { + parent::multiPartUpload($urn, $stream); + } +} + /** * @group PRIMARY-s3 */ class S3Test extends ObjectStoreTest { - /** - * @return \OCP\Files\ObjectStore\IObjectStore - */ protected function getInstance() { $config = \OC::$server->getConfig()->getSystemValue('objectstore'); if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') { $this->markTestSkipped('objectstore not configured for s3'); } - return new S3($config['arguments']); + return new MultiPartUploadS3($config['arguments']); + } + + public function testMultiPartUploader() { + $s3 = $this->getInstance(); + + $s3->multiPartUpload('multiparttest', fopen(__FILE__, 'r')); + + $result = $s3->readObject('multiparttest'); + + $this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result)); } } From e393b3553eb5ad867b34f3fdc029a1887bcd3980 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 11 Oct 2017 15:59:53 +0200 Subject: [PATCH 4/4] set s3 part size to 500mb Signed-off-by: Robin Appelman --- lib/private/Files/ObjectStore/S3ObjectTrait.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 93204896c1..2cbe20d980 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -26,7 +26,7 @@ use Aws\S3\MultipartUploader; use Aws\S3\S3Client; use Psr\Http\Message\StreamInterface; -const S3_UPLOAD_PART_SIZE = 5368709120; +const S3_UPLOAD_PART_SIZE = 524288000; // 500MB trait S3ObjectTrait { /** @@ -86,6 +86,7 @@ trait S3ObjectTrait { $uploader = new MultipartUploader($this->getConnection(), $stream, [ 'bucket' => $this->bucket, 'key' => $urn, + 'part_size' => S3_UPLOAD_PART_SIZE ]); $tries = 0; @@ -94,6 +95,7 @@ trait S3ObjectTrait { try { $result = $uploader->upload(); } catch (MultipartUploadException $e) { + \OC::$server->getLogger()->logException($e); rewind($stream); $tries++;