Clean up with fixer

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
This commit is contained in:
Bernd.Rederlechner@t-systems.com 2021-05-12 10:54:44 +00:00
parent b5dfb01dc5
commit 62555d7c0f
3 changed files with 297 additions and 270 deletions

View File

@ -31,138 +31,144 @@ use Aws\S3\S3Client;
use Icewind\Streams\CallbackWrapper; use Icewind\Streams\CallbackWrapper;
use OC\Files\Stream\SeekableHttpStream; use OC\Files\Stream\SeekableHttpStream;
trait S3ObjectTrait { trait S3ObjectTrait
/** {
* Returns the connection. /**
* * Returns the connection.
* @return S3Client connected client *
* * @return S3Client connected client
* @throws \Exception if connection could not be made *
*/ * @throws \Exception if connection could not be made
abstract protected function getConnection(); */
abstract protected function getConnection();
/* compute configured encryption headers for put operations */ /* compute configured encryption headers for put operations */
abstract protected function getSseKmsPutParameters(); abstract protected function getSseKmsPutParameters();
/* compute configured encryption headers for get operations */ /* compute configured encryption headers for get operations */
abstract protected function getSseKmsGetParameters(); abstract protected function getSseKmsGetParameters();
/** /**
* @param string $urn the unified resource name used to identify the object * @param string $urn the unified resource name used to identify the object
* *
* @return resource stream with the read data * @return resource stream with the read data
* *
* @throws \Exception when something goes wrong, message will be logged * @throws \Exception when something goes wrong, message will be logged
* *
* @since 7.0.0 * @since 7.0.0
*/ */
public function readObject($urn) { public function readObject($urn)
return SeekableHttpStream::open(function ($range) use ($urn) { {
$s3params = [ return SeekableHttpStream::open(function ($range) use ($urn) {
'Bucket' => $this->bucket, $s3params = [
'Key' => $urn, 'Bucket' => $this->bucket,
'Range' => 'bytes='.$range, 'Key' => $urn,
] + $this->getSseKmsGetParameters(); 'Range' => 'bytes='.$range,
$command = $this->getConnection()->getCommand('GetObject', $s3params); ] + $this->getSseKmsGetParameters();
$request = \Aws\serialize($command); $command = $this->getConnection()->getCommand('GetObject', $s3params);
$headers = []; $request = \Aws\serialize($command);
foreach ($request->getHeaders() as $key => $values) { $headers = [];
foreach ($values as $value) { foreach ($request->getHeaders() as $key => $values) {
$headers[] = "$key: $value"; foreach ($values as $value) {
} $headers[] = "$key: $value";
} }
$opts = [ }
'http' => [ $opts = [
'protocol_version' => 1.1, 'http' => [
'header' => $headers, 'protocol_version' => 1.1,
], 'header' => $headers,
]; ],
];
$context = stream_context_create($opts); $context = stream_context_create($opts);
return fopen($request->getUri(), 'r', false, $context); return fopen($request->getUri(), 'r', false, $context);
}); });
} }
/** /**
* @param string $urn the unified resource name used to identify the object * @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write * @param resource $stream stream with the data to write
* @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
* @throws \Exception when something goes wrong, message will be logged * @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0 * @since 7.0.0
*/ */
public function writeObject($urn, $stream, string $mimetype = null) { public function writeObject($urn, $stream, string $mimetype = null)
$count = 0; {
$countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) { $count = 0;
$count += $read; $countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) {
}); $count += $read;
});
$s3params = [ $s3params = [
'bucket' => $this->bucket, 'bucket' => $this->bucket,
'key' => $urn, 'key' => $urn,
'part_size' => $this->uploadPartSize, 'part_size' => $this->uploadPartSize,
'params' => [ 'params' => [
'ContentType' => $mimetype 'ContentType' => $mimetype
] + $this->getSseKmsPutParameters(), ] + $this->getSseKmsPutParameters(),
]; ];
// maybe, we should also use ObjectUploader here in the future // maybe, we should also use ObjectUploader here in the future
// it does direct uploads for small files < 5MB and multipart otherwise // it does direct uploads for small files < 5MB and multipart otherwise
// $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, $countStream, 'private', $s3params); // $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, $countStream, 'private', $s3params);
$uploader = new MultipartUploader($this->getConnection(), $countStream, $s3params); $uploader = new MultipartUploader($this->getConnection(), $countStream, $s3params);
try { try {
$uploader->upload(); $uploader->upload();
} catch (S3MultipartUploadException $e) { } catch (S3MultipartUploadException $e) {
// if anything goes wrong with multipart, make sure that you don´t poison s3 bucket with fragments // if anything goes wrong with multipart, make sure that you don´t poison s3 bucket with fragments
$this->getConnection()->abortMultipartUpload($uploader->getState()->getId()); $this->getConnection()->abortMultipartUpload($uploader->getState()->getId());
if ($count === 0 && feof($countStream)) { if ($count === 0 && feof($countStream)) {
// This is an empty file case, so just touch it // This is an empty file case, so just touch it
$s3params = [ $s3params = [
'params' => $this->getSseKmsPutParameters(), 'params' => $this->getSseKmsPutParameters(),
]; ];
$uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '', 'private', $s3params); $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '', 'private', $s3params);
$uploader->upload(); $uploader->upload();
} else { } else {
throw $e; throw $e;
} }
} finally { } finally {
// this handles [S3] fclose(): supplied resource is not a valid stream resource #23373 // this handles [S3] fclose(): supplied resource is not a valid stream resource #23373
// see https://stackoverflow.com/questions/11247507/fclose-18-is-not-a-valid-stream-resource/11247555 // see https://stackoverflow.com/questions/11247507/fclose-18-is-not-a-valid-stream-resource/11247555
// which also recommends the solution // which also recommends the solution
if (is_resource($countStream)) { if (is_resource($countStream)) {
fclose($countStream); fclose($countStream);
} }
} }
} }
/** /**
* @param string $urn the unified resource name used to identify the object * @param string $urn the unified resource name used to identify the object
* *
* @return void * @return void
* *
* @throws \Exception when something goes wrong, message will be logged * @throws \Exception when something goes wrong, message will be logged
* *
* @since 7.0.0 * @since 7.0.0
*/ */
public function deleteObject($urn) { public function deleteObject($urn)
$this->getConnection()->deleteObject([ {
'Bucket' => $this->bucket, $this->getConnection()->deleteObject([
'Key' => $urn, 'Bucket' => $this->bucket,
]); 'Key' => $urn,
} ]);
}
public function objectExists($urn) { public function objectExists($urn)
return $this->getConnection()->doesObjectExist($this->bucket, $urn); {
} return $this->getConnection()->doesObjectExist($this->bucket, $urn);
}
/**
* S3 copy command with SSE KMS key handling. /**
*/ * S3 copy command with SSE KMS key handling.
public function copyObject($from, $to) { */
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [ public function copyObject($from, $to)
'params' => $this->getSseKmsPutParameters(), {
]); $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [
} 'params' => $this->getSseKmsPutParameters(),
]);
}
} }

View File

@ -24,107 +24,115 @@ namespace Test\Files\ObjectStore;
use Test\TestCase; use Test\TestCase;
abstract class ObjectStoreTest extends TestCase { abstract class ObjectStoreTest extends TestCase
{
/** /**
* @return \OCP\Files\ObjectStore\IObjectStore * @return \OCP\Files\ObjectStore\IObjectStore
*/ */
abstract protected function getInstance(); abstract protected function getInstance();
protected function stringToStream($data) { protected function stringToStream($data)
$stream = fopen('php://temp', 'w+'); {
fwrite($stream, $data); $stream = fopen('php://temp', 'w+');
rewind($stream); fwrite($stream, $data);
return $stream; rewind($stream);
} return $stream;
}
public function testWriteRead() { public function testWriteRead()
$stream = $this->stringToStream('foobar'); {
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance(); $instance = $this->getInstance();
$instance->writeObject('1', $stream); $instance->writeObject('1', $stream);
$result = $instance->readObject('1'); $result = $instance->readObject('1');
$instance->deleteObject('1'); $instance->deleteObject('1');
$this->assertEquals('foobar', stream_get_contents($result)); $this->assertEquals('foobar', stream_get_contents($result));
} }
public function testDelete() { public function testDelete()
$stream = $this->stringToStream('foobar'); {
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance(); $instance = $this->getInstance();
$instance->writeObject('2', $stream); $instance->writeObject('2', $stream);
$instance->deleteObject('2'); $instance->deleteObject('2');
try { try {
// to to read to verify that the object no longer exists // to to read to verify that the object no longer exists
$instance->readObject('2'); $instance->readObject('2');
$this->fail(); $this->fail();
} catch (\Exception $e) { } catch (\Exception $e) {
// dummy assert to keep phpunit happy // dummy assert to keep phpunit happy
$this->assertEquals(1, 1); $this->assertEquals(1, 1);
} }
} }
public function testReadNonExisting() { public function testReadNonExisting()
$instance = $this->getInstance(); {
$instance = $this->getInstance();
try { try {
$instance->readObject('non-existing'); $instance->readObject('non-existing');
$this->fail(); $this->fail();
} catch (\Exception $e) { } catch (\Exception $e) {
// dummy assert to keep phpunit happy // dummy assert to keep phpunit happy
$this->assertEquals(1, 1); $this->assertEquals(1, 1);
} }
} }
public function testDeleteNonExisting() { public function testDeleteNonExisting()
$instance = $this->getInstance(); {
$instance = $this->getInstance();
try { try {
$instance->deleteObject('non-existing'); $instance->deleteObject('non-existing');
$this->fail(); $this->fail();
} catch (\Exception $e) { } catch (\Exception $e) {
// dummy assert to keep phpunit happy // dummy assert to keep phpunit happy
$this->assertEquals(1, 1); $this->assertEquals(1, 1);
} }
} }
public function testExists() { public function testExists()
$stream = $this->stringToStream('foobar'); {
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance(); $instance = $this->getInstance();
$this->assertFalse($instance->objectExists('2')); $this->assertFalse($instance->objectExists('2'));
$instance->writeObject('2', $stream); $instance->writeObject('2', $stream);
$this->assertTrue($instance->objectExists('2')); $this->assertTrue($instance->objectExists('2'));
$instance->deleteObject('2'); $instance->deleteObject('2');
$this->assertFalse($instance->objectExists('2')); $this->assertFalse($instance->objectExists('2'));
} }
public function testCopy() { public function testCopy()
$stream = $this->stringToStream('foobar'); {
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance(); $instance = $this->getInstance();
$instance->writeObject('source', $stream); $instance->writeObject('source', $stream);
$this->assertFalse($instance->objectExists('target')); $this->assertFalse($instance->objectExists('target'));
$instance->copyObject('source', 'target'); $instance->copyObject('source', 'target');
$this->assertTrue($instance->objectExists('target')); $this->assertTrue($instance->objectExists('target'));
$this->assertEquals('foobar', stream_get_contents($instance->readObject('target'))); $this->assertEquals('foobar', stream_get_contents($instance->readObject('target')));
$instance->deleteObject('source'); $instance->deleteObject('source');
$instance->deleteObject('target'); $instance->deleteObject('target');
} }
} }

View File

@ -28,107 +28,120 @@ use OC\Files\ObjectStore\S3;
use Aws\S3\S3Client; use Aws\S3\S3Client;
use Aws\S3\Exception\S3MultipartUploadException; use Aws\S3\Exception\S3MultipartUploadException;
class MultiPartUploadS3 extends S3 { class MultiPartUploadS3 extends S3
public function writeObject($urn, $stream, string $mimetype = null) { {
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [ public function writeObject($urn, $stream, string $mimetype = null)
'mup_threshold' => 1, {
]); $this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
} 'mup_threshold' => 1,
]);
}
} }
class NonSeekableStream extends Wrapper { class NonSeekableStream extends Wrapper
public static function wrap($source) { {
$context = stream_context_create([ public static function wrap($source)
'nonseek' => [ {
'source' => $source, $context = stream_context_create([
], 'nonseek' => [
]); 'source' => $source,
return Wrapper::wrapSource($source, $context, 'nonseek', self::class); ],
} ]);
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
}
public function dir_opendir($path, $options) { public function dir_opendir($path, $options)
return false; {
} return false;
}
public function stream_open($path, $mode, $options, &$opened_path) { public function stream_open($path, $mode, $options, &$opened_path)
$this->loadContext('nonseek'); {
return true; $this->loadContext('nonseek');
} return true;
}
public function stream_seek($offset, $whence = SEEK_SET) { public function stream_seek($offset, $whence = SEEK_SET)
return false; {
} return false;
}
} }
/** /**
* @group PRIMARY-s3 * @group PRIMARY-s3
*/ */
class S3Test extends ObjectStoreTest { class S3Test extends ObjectStoreTest
protected function getInstance() { {
$config = \OC::$server->getConfig()->getSystemValue('objectstore'); protected function getInstance()
if (!is_array($config) || $config['class'] !== '\\OC\\Files\\ObjectStore\\S3') { {
$this->markTestSkipped('objectstore not configured for s3'); $config = \OC::$server->getConfig()->getSystemValue('objectstore');
} if (!is_array($config) || $config['class'] !== '\\OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
return new S3($config['arguments']); return new S3($config['arguments']);
} }
public function testUploadNonSeekable() { public function testUploadNonSeekable()
$s3 = $this->getInstance(); {
$s3 = $this->getInstance();
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r'))); $s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
$result = $s3->readObject('multiparttest'); $result = $s3->readObject('multiparttest');
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result)); $this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
$s3->deleteObject('multiparttest'); $s3->deleteObject('multiparttest');
} }
public function testSeek() { public function testSeek()
$data = file_get_contents(__FILE__); {
$data = file_get_contents(__FILE__);
$instance = $this->getInstance(); $instance = $this->getInstance();
$instance->writeObject('seek', $this->stringToStream($data)); $instance->writeObject('seek', $this->stringToStream($data));
$read = $instance->readObject('seek'); $read = $instance->readObject('seek');
$this->assertEquals(substr($data, 0, 100), fread($read, 100)); $this->assertEquals(substr($data, 0, 100), fread($read, 100));
fseek($read, 10); fseek($read, 10);
$this->assertEquals(substr($data, 10, 100), fread($read, 100)); $this->assertEquals(substr($data, 10, 100), fread($read, 100));
fseek($read, 100, SEEK_CUR); fseek($read, 100, SEEK_CUR);
$this->assertEquals(substr($data, 210, 100), fread($read, 100)); $this->assertEquals(substr($data, 210, 100), fread($read, 100));
$instance->deleteObject('seek'); $instance->deleteObject('seek');
} }
function assertNoUpload($objectUrn) { public function assertNoUpload($objectUrn)
$s3 = $this->getInstance(); {
$s3client = $s3->getConnection(); $s3 = $this->getInstance();
$uploads = $s3client->listMultipartUploads([ $s3client = $s3->getConnection();
'Bucket' => $s3->getBucket(), $uploads = $s3client->listMultipartUploads([
'Prefix' => $objectUrn, 'Bucket' => $s3->getBucket(),
]); 'Prefix' => $objectUrn,
//fwrite(STDERR, print_r($uploads, TRUE)); ]);
$this->assertArrayNotHasKey('Uploads', $uploads); //fwrite(STDERR, print_r($uploads, TRUE));
} $this->assertArrayNotHasKey('Uploads', $uploads);
}
public function testEmptyUpload() { public function testEmptyUpload()
{
//$this->expectException(S3MultipartUploadException::class); //$this->expectException(S3MultipartUploadException::class);
$s3 = $this->getInstance(); $s3 = $this->getInstance();
// create an empty stream and check that it fits to the // create an empty stream and check that it fits to the
// pre-conditions in writeObject for the empty case // pre-conditions in writeObject for the empty case
$emptyStream = fopen("php://memory", "r"); $emptyStream = fopen("php://memory", "r");
fwrite($emptyStream, NULL); fwrite($emptyStream, null);
$s3->writeObject('emptystream', $emptyStream); $s3->writeObject('emptystream', $emptyStream);
// this method intendedly produces an S3Exception // this method intendedly produces an S3Exception
$this->assertNoUpload('emptystream'); $this->assertNoUpload('emptystream');
$this->assertTrue($s3->objectExists('emptystream')); $this->assertTrue($s3->objectExists('emptystream'));
$s3->deleteObject('emptystream'); $s3->deleteObject('emptystream');
} }
} }