mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #13866 from nextcloud/s3-multipart-uploader 

always use multipart uploader for s3 uploads
This commit is contained in:
Roeland Jago Douma 2019-01-29 11:14:05 +01:00 committed by GitHub
parent 2682d672d8 1d322d3b37
commit 0fd208a83b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/private/Files/ObjectStore/S3ObjectTrait.php
View File

@ -23,6 +23,7 @@
namespace OC\Files\ObjectStore;
use Aws\S3\MultipartUploader;
use Aws\S3\S3Client;
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
@ -72,10 +73,12 @@ trait S3ObjectTrait {
* @since 7.0.0
*/
function writeObject($urn, $stream) {
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
'mup_threshold' => S3_UPLOAD_PART_SIZE,
$uploader = new MultipartUploader($this->getConnection(), $stream, [
'bucket' => $this->bucket,
'key' => $urn,
'part_size' => S3_UPLOAD_PART_SIZE
]);
$uploader->upload();
}
/**

31
tests/lib/Files/ObjectStore/S3Test.php
View File

@ -21,6 +21,7 @@
namespace Test\Files\ObjectStore;
use Icewind\Streams\Wrapper;
use OC\Files\ObjectStore\S3;
class MultiPartUploadS3 extends S3 {
@ -31,6 +32,30 @@ class MultiPartUploadS3 extends S3 {
}
}
class NonSeekableStream extends Wrapper {
public static function wrap($source) {
$context = stream_context_create(array(
'nonseek' => array(
'source' => $source
)
));
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
}
public function dir_opendir($path, $options) {
return false;
}
public function stream_open($path, $mode, $options, &$opened_path) {
$this->loadContext('nonseek');
return true;
}
public function stream_seek($offset, $whence = SEEK_SET) {
return false;
}
}
/**
* @group PRIMARY-s3
*/
@ -44,15 +69,15 @@ class S3Test extends ObjectStoreTest {
return new S3($config['arguments']);
}
public function testMultiPartUploader() {
public function testUploadNonSeekable() {
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
$s3 = new MultiPartUploadS3($config['arguments']);
$s3 = $this->getInstance();
$s3->writeObject('multiparttest', fopen(__FILE__, 'r'));
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
$result = $s3->readObject('multiparttest');