Merge pull request #6602 from nextcloud/s3-multipart-upload
Add multipart upload for s3
This commit is contained in:
commit
1a99e0dab4
|
@ -21,9 +21,13 @@
|
|||
|
||||
namespace OC\Files\ObjectStore;
|
||||
|
||||
use Aws\Exception\MultipartUploadException;
|
||||
use Aws\S3\MultipartUploader;
|
||||
use Aws\S3\S3Client;
|
||||
use Psr\Http\Message\StreamInterface;
|
||||
|
||||
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
|
||||
|
||||
trait S3ObjectTrait {
|
||||
/**
|
||||
* Returns the connection
|
||||
|
@ -60,6 +64,17 @@ trait S3ObjectTrait {
|
|||
* @since 7.0.0
|
||||
*/
|
||||
function writeObject($urn, $stream) {
|
||||
$stat = fstat($stream);
|
||||
|
||||
if ($stat['size'] && $stat['size'] < S3_UPLOAD_PART_SIZE) {
|
||||
$this->singlePartUpload($urn, $stream);
|
||||
} else {
|
||||
$this->multiPartUpload($urn, $stream);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected function singlePartUpload($urn, $stream) {
|
||||
$this->getConnection()->putObject([
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $urn,
|
||||
|
@ -67,6 +82,34 @@ trait S3ObjectTrait {
|
|||
]);
|
||||
}
|
||||
|
||||
protected function multiPartUpload($urn, $stream) {
|
||||
$uploader = new MultipartUploader($this->getConnection(), $stream, [
|
||||
'bucket' => $this->bucket,
|
||||
'key' => $urn,
|
||||
'part_size' => S3_UPLOAD_PART_SIZE
|
||||
]);
|
||||
|
||||
$tries = 0;
|
||||
|
||||
do {
|
||||
try {
|
||||
$result = $uploader->upload();
|
||||
} catch (MultipartUploadException $e) {
|
||||
\OC::$server->getLogger()->logException($e);
|
||||
rewind($stream);
|
||||
$tries++;
|
||||
|
||||
if ($tries < 5) {
|
||||
$uploader = new MultipartUploader($this->getConnection(), $stream, [
|
||||
'state' => $e->getState()
|
||||
]);
|
||||
} else {
|
||||
$this->getConnection()->abortMultipartUpload($e->getState()->getId());
|
||||
}
|
||||
}
|
||||
} while (!isset($result) && $tries < 5);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $urn the unified resource name used to identify the object
|
||||
* @return void
|
||||
|
|
|
@ -23,19 +23,32 @@ namespace Test\Files\ObjectStore;
|
|||
|
||||
use OC\Files\ObjectStore\S3;
|
||||
|
||||
class MultiPartUploadS3 extends S3 {
|
||||
public function multiPartUpload($urn, $stream) {
|
||||
parent::multiPartUpload($urn, $stream);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @group PRIMARY-s3
|
||||
*/
|
||||
class S3Test extends ObjectStoreTest {
|
||||
/**
|
||||
* @return \OCP\Files\ObjectStore\IObjectStore
|
||||
*/
|
||||
protected function getInstance() {
|
||||
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
|
||||
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
|
||||
$this->markTestSkipped('objectstore not configured for s3');
|
||||
}
|
||||
|
||||
return new S3($config['arguments']);
|
||||
return new MultiPartUploadS3($config['arguments']);
|
||||
}
|
||||
|
||||
public function testMultiPartUploader() {
|
||||
$s3 = $this->getInstance();
|
||||
|
||||
$s3->multiPartUpload('multiparttest', fopen(__FILE__, 'r'));
|
||||
|
||||
$result = $s3->readObject('multiparttest');
|
||||
|
||||
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue