diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 9a1f5befb4..8954b883e3 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1025,6 +1025,7 @@ return array( 'OC\\Files\\Storage\\Wrapper\\Wrapper' => $baseDir . '/lib/private/Files/Storage/Wrapper/Wrapper.php', 'OC\\Files\\Stream\\Encryption' => $baseDir . '/lib/private/Files/Stream/Encryption.php', 'OC\\Files\\Stream\\Quota' => $baseDir . '/lib/private/Files/Stream/Quota.php', + 'OC\\Files\\Stream\\SeekableHttpStream' => $baseDir . '/lib/private/Files/Stream/SeekableHttpStream.php', 'OC\\Files\\Type\\Detection' => $baseDir . '/lib/private/Files/Type/Detection.php', 'OC\\Files\\Type\\Loader' => $baseDir . '/lib/private/Files/Type/Loader.php', 'OC\\Files\\Type\\TemplateManager' => $baseDir . '/lib/private/Files/Type/TemplateManager.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 318888bd43..136562ff75 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1054,6 +1054,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Storage\\Wrapper\\Wrapper' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Wrapper.php', 'OC\\Files\\Stream\\Encryption' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Encryption.php', 'OC\\Files\\Stream\\Quota' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Quota.php', + 'OC\\Files\\Stream\\SeekableHttpStream' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/SeekableHttpStream.php', 'OC\\Files\\Type\\Detection' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Detection.php', 'OC\\Files\\Type\\Loader' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Loader.php', 'OC\\Files\\Type\\TemplateManager' => __DIR__ . '/../../..' . '/lib/private/Files/Type/TemplateManager.php', diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 6188357521..0939dd2a23 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -30,6 +30,7 @@ use Aws\S3\MultipartUploader; use Aws\S3\ObjectUploader; use Aws\S3\S3Client; use Icewind\Streams\CallbackWrapper; +use OC\Files\Stream\SeekableHttpStream; const S3_UPLOAD_PART_SIZE = 524288000; // 500MB @@ -49,27 +50,29 @@ trait S3ObjectTrait { * @since 7.0.0 */ function readObject($urn) { - $client = $this->getConnection(); - $command = $client->getCommand('GetObject', [ - 'Bucket' => $this->bucket, - 'Key' => $urn - ]); - $request = \Aws\serialize($command); - $headers = []; - foreach ($request->getHeaders() as $key => $values) { - foreach ($values as $value) { - $headers[] = "$key: $value"; + return SeekableHttpStream::open(function ($range) use ($urn) { + $command = $this->getConnection()->getCommand('GetObject', [ + 'Bucket' => $this->bucket, + 'Key' => $urn, + 'Range' => 'bytes=' . $range, + ]); + $request = \Aws\serialize($command); + $headers = []; + foreach ($request->getHeaders() as $key => $values) { + foreach ($values as $value) { + $headers[] = "$key: $value"; + } } - } - $opts = [ - 'http' => [ - 'protocol_version' => 1.1, - 'header' => $headers - ] - ]; + $opts = [ + 'http' => [ + 'protocol_version' => 1.1, + 'header' => $headers, + ], + ]; - $context = stream_context_create($opts); - return fopen($request->getUri(), 'r', false, $context); + $context = stream_context_create($opts); + return fopen($request->getUri(), 'r', false, $context); + }); } /** @@ -87,7 +90,7 @@ trait S3ObjectTrait { $uploader = new MultipartUploader($this->getConnection(), $countStream, [ 'bucket' => $this->bucket, 'key' => $urn, - 'part_size' => S3_UPLOAD_PART_SIZE + 'part_size' => S3_UPLOAD_PART_SIZE, ]); try { @@ -114,7 +117,7 @@ trait S3ObjectTrait { function deleteObject($urn) { $this->getConnection()->deleteObject([ 'Bucket' => $this->bucket, - 'Key' => $urn + 'Key' => $urn, ]); } diff --git a/lib/private/Files/Stream/SeekableHttpStream.php b/lib/private/Files/Stream/SeekableHttpStream.php new file mode 100644 index 0000000000..fdcd9ea8cf --- /dev/null +++ b/lib/private/Files/Stream/SeekableHttpStream.php @@ -0,0 +1,182 @@ +. + * + */ + +namespace OC\Files\Stream; + +use Icewind\Streams\File; + +/** + * A stream wrapper that uses http range requests to provide a seekable stream for http reading + */ +class SeekableHttpStream implements File { + private const PROTOCOL = 'httpseek'; + + private static $registered = false; + + /** + * Registers the stream wrapper using the `httpseek://` url scheme + * $return void + */ + private static function registerIfNeeded() { + if (!self::$registered) { + stream_wrapper_register( + self::PROTOCOL, + self::class + ); + self::$registered = true; + } + } + + /** + * Open a readonly-seekable http stream + * + * The provided callback will be called with byte range and should return an http stream for the requested range + * + * @param callable $callback + * @return false|resource + */ + public static function open(callable $callback) { + $context = stream_context_create([ + SeekableHttpStream::PROTOCOL => [ + 'callback' => $callback + ], + ]); + + SeekableHttpStream::registerIfNeeded(); + return fopen(SeekableHttpStream::PROTOCOL . '://', 'r', false, $context); + } + + /** @var resource */ + public $context; + + /** @var callable */ + private $openCallback; + + /** @var resource */ + private $current; + /** @var int */ + private $offset = 0; + + private function reconnect(int $start) { + $range = $start . '-'; + if ($this->current != null) { + fclose($this->current); + } + + $this->current = ($this->openCallback)($range); + + if ($this->current === false) { + return false; + } + + $responseHead = stream_get_meta_data($this->current)['wrapper_data']; + $rangeHeaders = array_values(array_filter($responseHead, function ($v) { + return preg_match('#^content-range:#i', $v) === 1; + })); + if (!$rangeHeaders) { + return false; + } + $contentRange = $rangeHeaders[0]; + + $content = trim(explode(':', $contentRange)[1]); + $range = trim(explode(' ', $content)[1]); + $begin = intval(explode('-', $range)[0]); + + if ($begin !== $start) { + return false; + } + + $this->offset = $begin; + + return true; + } + + public function stream_open($path, $mode, $options, &$opened_path) { + $options = stream_context_get_options($this->context)[self::PROTOCOL]; + $this->openCallback = $options['callback']; + + return $this->reconnect(0); + } + + public function stream_read($count) { + if (!$this->current) { + return false; + } + $ret = fread($this->current, $count); + $this->offset += strlen($ret); + return $ret; + } + + public function stream_seek($offset, $whence = SEEK_SET) { + switch ($whence) { + case SEEK_SET: + if ($offset === $this->offset) { + return true; + } + return $this->reconnect($offset); + case SEEK_CUR: + if ($offset === 0) { + return true; + } + return $this->reconnect($this->offset + $offset); + case SEEK_END: + return false; + } + return false; + } + + public function stream_tell() { + return $this->offset; + } + + public function stream_stat() { + return fstat($this->current); + } + + public function stream_eof() { + return feof($this->current); + } + + public function stream_close() { + fclose($this->current); + } + + public function stream_write($data) { + return false; + } + + public function stream_set_option($option, $arg1, $arg2) { + return false; + } + + public function stream_truncate($size) { + return false; + } + + public function stream_lock($operation) { + return false; + } + + public function stream_flush() { + return; //noop because readonly stream + } +} diff --git a/tests/lib/Files/ObjectStore/ObjectStoreTest.php b/tests/lib/Files/ObjectStore/ObjectStoreTest.php index 1383c0149a..67c41eb7cc 100644 --- a/tests/lib/Files/ObjectStore/ObjectStoreTest.php +++ b/tests/lib/Files/ObjectStore/ObjectStoreTest.php @@ -31,7 +31,7 @@ abstract class ObjectStoreTest extends TestCase { */ abstract protected function getInstance(); - private function stringToStream($data) { + protected function stringToStream($data) { $stream = fopen('php://temp', 'w+'); fwrite($stream, $data); rewind($stream); diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php index b56978f4fe..525e020445 100644 --- a/tests/lib/Files/ObjectStore/S3Test.php +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -27,7 +27,7 @@ use OC\Files\ObjectStore\S3; class MultiPartUploadS3 extends S3 { function writeObject($urn, $stream) { $this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [ - 'mup_threshold' => 1 + 'mup_threshold' => 1, ]); } } @@ -36,8 +36,8 @@ class NonSeekableStream extends Wrapper { public static function wrap($source) { $context = stream_context_create([ 'nonseek' => [ - 'source' => $source - ] + 'source' => $source, + ], ]); return Wrapper::wrapSource($source, $context, 'nonseek', self::class); } @@ -83,4 +83,20 @@ class S3Test extends ObjectStoreTest { $this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result)); } + + public function testSeek() { + $data = file_get_contents(__FILE__); + + $instance = $this->getInstance(); + $instance->writeObject('seek', $this->stringToStream($data)); + + $read = $instance->readObject('seek'); + $this->assertEquals(substr($data, 0, 100), fread($read, 100)); + + fseek($read, 10); + $this->assertEquals(substr($data, 10, 100), fread($read, 100)); + + fseek($read, 100, SEEK_CUR); + $this->assertEquals(substr($data, 210, 100), fread($read, 100)); + } }