Merge pull request #13866 from nextcloud/s3-multipart-uploader
always use multipart uploader for s3 uploads
This commit is contained in:
commit
0fd208a83b
|
@ -23,6 +23,7 @@
|
||||||
|
|
||||||
namespace OC\Files\ObjectStore;
|
namespace OC\Files\ObjectStore;
|
||||||
|
|
||||||
|
use Aws\S3\MultipartUploader;
|
||||||
use Aws\S3\S3Client;
|
use Aws\S3\S3Client;
|
||||||
|
|
||||||
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
|
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
|
||||||
|
@ -72,10 +73,12 @@ trait S3ObjectTrait {
|
||||||
* @since 7.0.0
|
* @since 7.0.0
|
||||||
*/
|
*/
|
||||||
function writeObject($urn, $stream) {
|
function writeObject($urn, $stream) {
|
||||||
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
|
$uploader = new MultipartUploader($this->getConnection(), $stream, [
|
||||||
'mup_threshold' => S3_UPLOAD_PART_SIZE,
|
'bucket' => $this->bucket,
|
||||||
|
'key' => $urn,
|
||||||
'part_size' => S3_UPLOAD_PART_SIZE
|
'part_size' => S3_UPLOAD_PART_SIZE
|
||||||
]);
|
]);
|
||||||
|
$uploader->upload();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -21,6 +21,7 @@
|
||||||
|
|
||||||
namespace Test\Files\ObjectStore;
|
namespace Test\Files\ObjectStore;
|
||||||
|
|
||||||
|
use Icewind\Streams\Wrapper;
|
||||||
use OC\Files\ObjectStore\S3;
|
use OC\Files\ObjectStore\S3;
|
||||||
|
|
||||||
class MultiPartUploadS3 extends S3 {
|
class MultiPartUploadS3 extends S3 {
|
||||||
|
@ -31,6 +32,30 @@ class MultiPartUploadS3 extends S3 {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class NonSeekableStream extends Wrapper {
|
||||||
|
public static function wrap($source) {
|
||||||
|
$context = stream_context_create(array(
|
||||||
|
'nonseek' => array(
|
||||||
|
'source' => $source
|
||||||
|
)
|
||||||
|
));
|
||||||
|
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function dir_opendir($path, $options) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_open($path, $mode, $options, &$opened_path) {
|
||||||
|
$this->loadContext('nonseek');
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_seek($offset, $whence = SEEK_SET) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @group PRIMARY-s3
|
* @group PRIMARY-s3
|
||||||
*/
|
*/
|
||||||
|
@ -44,15 +69,15 @@ class S3Test extends ObjectStoreTest {
|
||||||
return new S3($config['arguments']);
|
return new S3($config['arguments']);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function testMultiPartUploader() {
|
public function testUploadNonSeekable() {
|
||||||
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
|
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
|
||||||
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
|
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
|
||||||
$this->markTestSkipped('objectstore not configured for s3');
|
$this->markTestSkipped('objectstore not configured for s3');
|
||||||
}
|
}
|
||||||
|
|
||||||
$s3 = new MultiPartUploadS3($config['arguments']);
|
$s3 = $this->getInstance();
|
||||||
|
|
||||||
$s3->writeObject('multiparttest', fopen(__FILE__, 'r'));
|
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
|
||||||
|
|
||||||
$result = $s3->readObject('multiparttest');
|
$result = $s3->readObject('multiparttest');
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue