Merge pull request #13866 from nextcloud/s3-multipart-uploader
always use multipart uploader for s3 uploads
This commit is contained in:
commit
0fd208a83b
|
@ -23,6 +23,7 @@
|
|||
|
||||
namespace OC\Files\ObjectStore;
|
||||
|
||||
use Aws\S3\MultipartUploader;
|
||||
use Aws\S3\S3Client;
|
||||
|
||||
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
|
||||
|
@ -72,10 +73,12 @@ trait S3ObjectTrait {
|
|||
* @since 7.0.0
|
||||
*/
|
||||
function writeObject($urn, $stream) {
|
||||
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
|
||||
'mup_threshold' => S3_UPLOAD_PART_SIZE,
|
||||
$uploader = new MultipartUploader($this->getConnection(), $stream, [
|
||||
'bucket' => $this->bucket,
|
||||
'key' => $urn,
|
||||
'part_size' => S3_UPLOAD_PART_SIZE
|
||||
]);
|
||||
$uploader->upload();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -21,6 +21,7 @@
|
|||
|
||||
namespace Test\Files\ObjectStore;
|
||||
|
||||
use Icewind\Streams\Wrapper;
|
||||
use OC\Files\ObjectStore\S3;
|
||||
|
||||
class MultiPartUploadS3 extends S3 {
|
||||
|
@ -31,6 +32,30 @@ class MultiPartUploadS3 extends S3 {
|
|||
}
|
||||
}
|
||||
|
||||
class NonSeekableStream extends Wrapper {
|
||||
public static function wrap($source) {
|
||||
$context = stream_context_create(array(
|
||||
'nonseek' => array(
|
||||
'source' => $source
|
||||
)
|
||||
));
|
||||
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
|
||||
}
|
||||
|
||||
public function dir_opendir($path, $options) {
|
||||
return false;
|
||||
}
|
||||
|
||||
public function stream_open($path, $mode, $options, &$opened_path) {
|
||||
$this->loadContext('nonseek');
|
||||
return true;
|
||||
}
|
||||
|
||||
public function stream_seek($offset, $whence = SEEK_SET) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @group PRIMARY-s3
|
||||
*/
|
||||
|
@ -44,15 +69,15 @@ class S3Test extends ObjectStoreTest {
|
|||
return new S3($config['arguments']);
|
||||
}
|
||||
|
||||
public function testMultiPartUploader() {
|
||||
public function testUploadNonSeekable() {
|
||||
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
|
||||
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
|
||||
$this->markTestSkipped('objectstore not configured for s3');
|
||||
}
|
||||
|
||||
$s3 = new MultiPartUploadS3($config['arguments']);
|
||||
$s3 = $this->getInstance();
|
||||
|
||||
$s3->writeObject('multiparttest', fopen(__FILE__, 'r'));
|
||||
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
|
||||
|
||||
$result = $s3->readObject('multiparttest');
|
||||
|
||||
|
|
Loading…
Reference in New Issue