Clean up with fixer

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
This commit is contained in:
Bernd.Rederlechner@t-systems.com 2021-05-12 10:54:44 +00:00
parent b5dfb01dc5
commit 62555d7c0f
3 changed files with 297 additions and 270 deletions

View File

@ -31,138 +31,144 @@ use Aws\S3\S3Client;
use Icewind\Streams\CallbackWrapper;
use OC\Files\Stream\SeekableHttpStream;
trait S3ObjectTrait {
/**
* Returns the connection.
*
* @return S3Client connected client
*
* @throws \Exception if connection could not be made
*/
abstract protected function getConnection();
trait S3ObjectTrait
{
/**
* Returns the connection.
*
* @return S3Client connected client
*
* @throws \Exception if connection could not be made
*/
abstract protected function getConnection();
/* compute configured encryption headers for put operations */
abstract protected function getSseKmsPutParameters();
/* compute configured encryption headers for put operations */
abstract protected function getSseKmsPutParameters();
/* compute configured encryption headers for get operations */
abstract protected function getSseKmsGetParameters();
/* compute configured encryption headers for get operations */
abstract protected function getSseKmsGetParameters();
/**
* @param string $urn the unified resource name used to identify the object
*
* @return resource stream with the read data
*
* @throws \Exception when something goes wrong, message will be logged
*
* @since 7.0.0
*/
public function readObject($urn) {
return SeekableHttpStream::open(function ($range) use ($urn) {
$s3params = [
'Bucket' => $this->bucket,
'Key' => $urn,
'Range' => 'bytes='.$range,
] + $this->getSseKmsGetParameters();
$command = $this->getConnection()->getCommand('GetObject', $s3params);
$request = \Aws\serialize($command);
$headers = [];
foreach ($request->getHeaders() as $key => $values) {
foreach ($values as $value) {
$headers[] = "$key: $value";
}
}
$opts = [
'http' => [
'protocol_version' => 1.1,
'header' => $headers,
],
];
/**
* @param string $urn the unified resource name used to identify the object
*
* @return resource stream with the read data
*
* @throws \Exception when something goes wrong, message will be logged
*
* @since 7.0.0
*/
public function readObject($urn)
{
return SeekableHttpStream::open(function ($range) use ($urn) {
$s3params = [
'Bucket' => $this->bucket,
'Key' => $urn,
'Range' => 'bytes='.$range,
] + $this->getSseKmsGetParameters();
$command = $this->getConnection()->getCommand('GetObject', $s3params);
$request = \Aws\serialize($command);
$headers = [];
foreach ($request->getHeaders() as $key => $values) {
foreach ($values as $value) {
$headers[] = "$key: $value";
}
}
$opts = [
'http' => [
'protocol_version' => 1.1,
'header' => $headers,
],
];
$context = stream_context_create($opts);
$context = stream_context_create($opts);
return fopen($request->getUri(), 'r', false, $context);
});
}
return fopen($request->getUri(), 'r', false, $context);
});
}
/**
* @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write
* @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
public function writeObject($urn, $stream, string $mimetype = null) {
$count = 0;
$countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) {
$count += $read;
});
/**
* @param string $urn the unified resource name used to identify the object
* @param resource $stream stream with the data to write
* @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
* @throws \Exception when something goes wrong, message will be logged
* @since 7.0.0
*/
public function writeObject($urn, $stream, string $mimetype = null)
{
$count = 0;
$countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) {
$count += $read;
});
$s3params = [
'bucket' => $this->bucket,
'key' => $urn,
'part_size' => $this->uploadPartSize,
'params' => [
'ContentType' => $mimetype
] + $this->getSseKmsPutParameters(),
];
$s3params = [
'bucket' => $this->bucket,
'key' => $urn,
'part_size' => $this->uploadPartSize,
'params' => [
'ContentType' => $mimetype
] + $this->getSseKmsPutParameters(),
];
// maybe, we should also use ObjectUploader here in the future
// it does direct uploads for small files < 5MB and multipart otherwise
// $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, $countStream, 'private', $s3params);
$uploader = new MultipartUploader($this->getConnection(), $countStream, $s3params);
// maybe, we should also use ObjectUploader here in the future
// it does direct uploads for small files < 5MB and multipart otherwise
// $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, $countStream, 'private', $s3params);
$uploader = new MultipartUploader($this->getConnection(), $countStream, $s3params);
try {
$uploader->upload();
} catch (S3MultipartUploadException $e) {
// if anything goes wrong with multipart, make sure that you don´t poison s3 bucket with fragments
$this->getConnection()->abortMultipartUpload($uploader->getState()->getId());
if ($count === 0 && feof($countStream)) {
// This is an empty file case, so just touch it
$s3params = [
'params' => $this->getSseKmsPutParameters(),
];
$uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '', 'private', $s3params);
$uploader->upload();
} else {
throw $e;
}
} finally {
// this handles [S3] fclose(): supplied resource is not a valid stream resource #23373
// see https://stackoverflow.com/questions/11247507/fclose-18-is-not-a-valid-stream-resource/11247555
// which also recommends the solution
if (is_resource($countStream)) {
fclose($countStream);
}
}
}
try {
$uploader->upload();
} catch (S3MultipartUploadException $e) {
// if anything goes wrong with multipart, make sure that you don´t poison s3 bucket with fragments
$this->getConnection()->abortMultipartUpload($uploader->getState()->getId());
if ($count === 0 && feof($countStream)) {
// This is an empty file case, so just touch it
$s3params = [
'params' => $this->getSseKmsPutParameters(),
];
$uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '', 'private', $s3params);
$uploader->upload();
} else {
throw $e;
}
} finally {
// this handles [S3] fclose(): supplied resource is not a valid stream resource #23373
// see https://stackoverflow.com/questions/11247507/fclose-18-is-not-a-valid-stream-resource/11247555
// which also recommends the solution
if (is_resource($countStream)) {
fclose($countStream);
}
}
}
/**
* @param string $urn the unified resource name used to identify the object
*
* @return void
*
* @throws \Exception when something goes wrong, message will be logged
*
* @since 7.0.0
*/
public function deleteObject($urn) {
$this->getConnection()->deleteObject([
'Bucket' => $this->bucket,
'Key' => $urn,
]);
}
/**
* @param string $urn the unified resource name used to identify the object
*
* @return void
*
* @throws \Exception when something goes wrong, message will be logged
*
* @since 7.0.0
*/
public function deleteObject($urn)
{
$this->getConnection()->deleteObject([
'Bucket' => $this->bucket,
'Key' => $urn,
]);
}
public function objectExists($urn) {
return $this->getConnection()->doesObjectExist($this->bucket, $urn);
}
/**
* S3 copy command with SSE KMS key handling.
*/
public function copyObject($from, $to) {
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [
'params' => $this->getSseKmsPutParameters(),
]);
}
public function objectExists($urn)
{
return $this->getConnection()->doesObjectExist($this->bucket, $urn);
}
/**
* S3 copy command with SSE KMS key handling.
*/
public function copyObject($from, $to)
{
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [
'params' => $this->getSseKmsPutParameters(),
]);
}
}

View File

@ -24,107 +24,115 @@ namespace Test\Files\ObjectStore;
use Test\TestCase;
abstract class ObjectStoreTest extends TestCase {
abstract class ObjectStoreTest extends TestCase
{
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
abstract protected function getInstance();
/**
* @return \OCP\Files\ObjectStore\IObjectStore
*/
abstract protected function getInstance();
protected function stringToStream($data) {
$stream = fopen('php://temp', 'w+');
fwrite($stream, $data);
rewind($stream);
return $stream;
}
protected function stringToStream($data)
{
$stream = fopen('php://temp', 'w+');
fwrite($stream, $data);
rewind($stream);
return $stream;
}
public function testWriteRead() {
$stream = $this->stringToStream('foobar');
public function testWriteRead()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance = $this->getInstance();
$instance->writeObject('1', $stream);
$instance->writeObject('1', $stream);
$result = $instance->readObject('1');
$instance->deleteObject('1');
$result = $instance->readObject('1');
$instance->deleteObject('1');
$this->assertEquals('foobar', stream_get_contents($result));
}
$this->assertEquals('foobar', stream_get_contents($result));
}
public function testDelete() {
$stream = $this->stringToStream('foobar');
public function testDelete()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance = $this->getInstance();
$instance->writeObject('2', $stream);
$instance->writeObject('2', $stream);
$instance->deleteObject('2');
$instance->deleteObject('2');
try {
// to to read to verify that the object no longer exists
$instance->readObject('2');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
try {
// to to read to verify that the object no longer exists
$instance->readObject('2');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testReadNonExisting() {
$instance = $this->getInstance();
public function testReadNonExisting()
{
$instance = $this->getInstance();
try {
$instance->readObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
try {
$instance->readObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testDeleteNonExisting() {
$instance = $this->getInstance();
public function testDeleteNonExisting()
{
$instance = $this->getInstance();
try {
$instance->deleteObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
try {
$instance->deleteObject('non-existing');
$this->fail();
} catch (\Exception $e) {
// dummy assert to keep phpunit happy
$this->assertEquals(1, 1);
}
}
public function testExists() {
$stream = $this->stringToStream('foobar');
public function testExists()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$this->assertFalse($instance->objectExists('2'));
$instance = $this->getInstance();
$this->assertFalse($instance->objectExists('2'));
$instance->writeObject('2', $stream);
$instance->writeObject('2', $stream);
$this->assertTrue($instance->objectExists('2'));
$this->assertTrue($instance->objectExists('2'));
$instance->deleteObject('2');
$instance->deleteObject('2');
$this->assertFalse($instance->objectExists('2'));
}
$this->assertFalse($instance->objectExists('2'));
}
public function testCopy() {
$stream = $this->stringToStream('foobar');
public function testCopy()
{
$stream = $this->stringToStream('foobar');
$instance = $this->getInstance();
$instance = $this->getInstance();
$instance->writeObject('source', $stream);
$instance->writeObject('source', $stream);
$this->assertFalse($instance->objectExists('target'));
$this->assertFalse($instance->objectExists('target'));
$instance->copyObject('source', 'target');
$instance->copyObject('source', 'target');
$this->assertTrue($instance->objectExists('target'));
$this->assertTrue($instance->objectExists('target'));
$this->assertEquals('foobar', stream_get_contents($instance->readObject('target')));
$this->assertEquals('foobar', stream_get_contents($instance->readObject('target')));
$instance->deleteObject('source');
$instance->deleteObject('target');
}
$instance->deleteObject('source');
$instance->deleteObject('target');
}
}

View File

@ -28,107 +28,120 @@ use OC\Files\ObjectStore\S3;
use Aws\S3\S3Client;
use Aws\S3\Exception\S3MultipartUploadException;
class MultiPartUploadS3 extends S3 {
public function writeObject($urn, $stream, string $mimetype = null) {
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
'mup_threshold' => 1,
]);
}
class MultiPartUploadS3 extends S3
{
public function writeObject($urn, $stream, string $mimetype = null)
{
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
'mup_threshold' => 1,
]);
}
}
class NonSeekableStream extends Wrapper {
public static function wrap($source) {
$context = stream_context_create([
'nonseek' => [
'source' => $source,
],
]);
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
}
class NonSeekableStream extends Wrapper
{
public static function wrap($source)
{
$context = stream_context_create([
'nonseek' => [
'source' => $source,
],
]);
return Wrapper::wrapSource($source, $context, 'nonseek', self::class);
}
public function dir_opendir($path, $options) {
return false;
}
public function dir_opendir($path, $options)
{
return false;
}
public function stream_open($path, $mode, $options, &$opened_path) {
$this->loadContext('nonseek');
return true;
}
public function stream_open($path, $mode, $options, &$opened_path)
{
$this->loadContext('nonseek');
return true;
}
public function stream_seek($offset, $whence = SEEK_SET) {
return false;
}
public function stream_seek($offset, $whence = SEEK_SET)
{
return false;
}
}
/**
* @group PRIMARY-s3
*/
class S3Test extends ObjectStoreTest {
protected function getInstance() {
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== '\\OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
class S3Test extends ObjectStoreTest
{
protected function getInstance()
{
$config = \OC::$server->getConfig()->getSystemValue('objectstore');
if (!is_array($config) || $config['class'] !== '\\OC\\Files\\ObjectStore\\S3') {
$this->markTestSkipped('objectstore not configured for s3');
}
return new S3($config['arguments']);
}
return new S3($config['arguments']);
}
public function testUploadNonSeekable() {
$s3 = $this->getInstance();
public function testUploadNonSeekable()
{
$s3 = $this->getInstance();
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
$s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));
$result = $s3->readObject('multiparttest');
$result = $s3->readObject('multiparttest');
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
$s3->deleteObject('multiparttest');
}
$s3->deleteObject('multiparttest');
}
public function testSeek() {
$data = file_get_contents(__FILE__);
public function testSeek()
{
$data = file_get_contents(__FILE__);
$instance = $this->getInstance();
$instance->writeObject('seek', $this->stringToStream($data));
$instance = $this->getInstance();
$instance->writeObject('seek', $this->stringToStream($data));
$read = $instance->readObject('seek');
$this->assertEquals(substr($data, 0, 100), fread($read, 100));
$read = $instance->readObject('seek');
$this->assertEquals(substr($data, 0, 100), fread($read, 100));
fseek($read, 10);
$this->assertEquals(substr($data, 10, 100), fread($read, 100));
fseek($read, 10);
$this->assertEquals(substr($data, 10, 100), fread($read, 100));
fseek($read, 100, SEEK_CUR);
$this->assertEquals(substr($data, 210, 100), fread($read, 100));
fseek($read, 100, SEEK_CUR);
$this->assertEquals(substr($data, 210, 100), fread($read, 100));
$instance->deleteObject('seek');
}
$instance->deleteObject('seek');
}
function assertNoUpload($objectUrn) {
$s3 = $this->getInstance();
$s3client = $s3->getConnection();
$uploads = $s3client->listMultipartUploads([
'Bucket' => $s3->getBucket(),
'Prefix' => $objectUrn,
]);
//fwrite(STDERR, print_r($uploads, TRUE));
$this->assertArrayNotHasKey('Uploads', $uploads);
}
public function assertNoUpload($objectUrn)
{
$s3 = $this->getInstance();
$s3client = $s3->getConnection();
$uploads = $s3client->listMultipartUploads([
'Bucket' => $s3->getBucket(),
'Prefix' => $objectUrn,
]);
//fwrite(STDERR, print_r($uploads, TRUE));
$this->assertArrayNotHasKey('Uploads', $uploads);
}
public function testEmptyUpload() {
public function testEmptyUpload()
{
//$this->expectException(S3MultipartUploadException::class);
$s3 = $this->getInstance();
$s3 = $this->getInstance();
// create an empty stream and check that it fits to the
// pre-conditions in writeObject for the empty case
$emptyStream = fopen("php://memory", "r");
fwrite($emptyStream, NULL);
// create an empty stream and check that it fits to the
// pre-conditions in writeObject for the empty case
$emptyStream = fopen("php://memory", "r");
fwrite($emptyStream, null);
$s3->writeObject('emptystream', $emptyStream);
$s3->writeObject('emptystream', $emptyStream);
// this method intendedly produces an S3Exception
$this->assertNoUpload('emptystream');
$this->assertTrue($s3->objectExists('emptystream'));
// this method intendedly produces an S3Exception
$this->assertNoUpload('emptystream');
$this->assertTrue($s3->objectExists('emptystream'));
$s3->deleteObject('emptystream');
}
$s3->deleteObject('emptystream');
}
}