mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #6602 from nextcloud/s3-multipart-upload 

Add multipart upload for s3
This commit is contained in:
Robin Appelman 2017-10-11 16:10:58 +02:00 committed by GitHub
parent 647b185c2b e393b3553e
commit 1a99e0dab4
2 changed files with 61 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
lib/private/Files/ObjectStore/S3ObjectTrait.php
View File

@ -21,9 +21,13 @@
namespace OC\Files\ObjectStore;
use Aws\Exception\MultipartUploadException;
use Aws\S3\MultipartUploader;
use Aws\S3\S3Client;
use Psr\Http\Message\StreamInterface;
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
trait S3ObjectTrait {
/**
* Returns the connection
@ -60,6 +64,17 @@ trait S3ObjectTrait {
* @since 7.0.0
*/
function writeObject($urn, $stream) {
$stat = fstat($stream);
if ($stat['size'] && $stat['size'] < S3_UPLOAD_PART_SIZE) {
$this->singlePartUpload($urn, $stream);
} else {
$this->multiPartUpload($urn, $stream);
}
}
protected function singlePartUpload($urn, $stream) {
$this->getConnection()->putObject([
'Bucket' => $this->bucket,
'Key' => $urn,
@ -67,6 +82,34 @@ trait S3ObjectTrait {
]);
}
protected function multiPartUpload($urn, $stream) {
$uploader = new MultipartUploader($this->getConnection(), $stream, [
'bucket' => $this->bucket,
'key' => $urn,
'part_size' => S3_UPLOAD_PART_SIZE
]);
$tries = 0;
do {
try {
$result = $uploader->upload();
} catch (MultipartUploadException $e) {
\OC::$server->getLogger()->logException($e);
rewind($stream);
$tries++;
if ($tries < 5) {
$uploader = new MultipartUploader($this->getConnection(), $stream, [
'state' => $e->getState()
]);
} else {
$this->getConnection()->abortMultipartUpload($e->getState()->getId());
}
}
} while (!isset($result) && $tries < 5);
}
/**
* @param string $urn the unified resource name used to identify the object
* @return void
@ -79,4 +122,4 @@ trait S3ObjectTrait {
'Key' => $urn
]);
}
}
}

21
tests/lib/Files/ObjectStore/S3Test.php
View File

@ -23,19 +23,32 @@ namespace Test\Files\ObjectStore;
use OC\Files\ObjectStore\S3;
class MultiPartUploadS3 extends S3 {
public function multiPartUpload($urn, $stream) {
parent::multiPartUpload($urn, $stream);
}
}
/**
* @group PRIMARY-s3
*/
class S3Test extends ObjectStoreTest {
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
protected function getInstance() {
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
return new S3($config['arguments']);
return new MultiPartUploadS3($config['arguments']);
}
public function testMultiPartUploader() {
$s3 = $this->getInstance();
$s3->multiPartUpload('multiparttest', fopen(__FILE__, 'r'));
$result = $s3->readObject('multiparttest');
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
}
}