Initial implementation of the new chunked upload - as specified in https://dragotin.wordpress.com/2015/06/22/owncloud-chunking-ng/

This commit is contained in:
Thomas Müller 2015-10-28 14:52:45 +01:00
parent 276b8a5831
commit 72f5c539e8
10 changed files with 712 additions and 1 deletions

View File

@ -0,0 +1,76 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2016, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
require '../../../../3rdparty/autoload.php';
if ($argc !== 6) {
echo "Invalid number of arguments" . PHP_EOL;
exit;
}
/**
* @param \Sabre\DAV\Client $client
* @param $uploadUrl
* @return mixed
*/
function request($client, $method, $uploadUrl, $data = null, $headers = []) {
echo "$method $uploadUrl ... ";
$t0 = microtime(true);
$result = $client->request($method, $uploadUrl, $data, $headers);
$t1 = microtime(true);
echo $result['statusCode'] . " - " . ($t1 - $t0) . ' seconds' . PHP_EOL;
if (!in_array($result['statusCode'], [200, 201])) {
echo $result['body'] . PHP_EOL;
}
return $result;
}
$baseUri = $argv[1];
$userName = $argv[2];
$password = $argv[3];
$file = $argv[4];
$chunkSize = $argv[5] * 1024 * 1024;
$client = new \Sabre\DAV\Client([
'baseUri' => $baseUri,
'userName' => $userName,
'password' => $password
]);
$transfer = uniqid('transfer', true);
$uploadUrl = "$baseUri/uploads/$userName/$transfer";
request($client, 'MKCOL', $uploadUrl);
$size = filesize($file);
$stream = fopen($file, 'r');
$index = 0;
while(!feof($stream)) {
request($client, 'PUT', "$uploadUrl/$index", fread($stream, $chunkSize));
$index++;
}
$destination = pathinfo($file, PATHINFO_BASENAME);
//echo "Moving $uploadUrl/.file to it's final destination $baseUri/files/$userName/$destination" . PHP_EOL;
request($client, 'MOVE', "$uploadUrl/.file", null, [
'Destination' => "$baseUri/files/$userName/$destination"
]);

View File

@ -143,7 +143,7 @@ class File extends Node implements IFile {
// if content length is sent by client:
// double check if the file was fully received
// compare expected and actual size
if (isset($_SERVER['CONTENT_LENGTH']) && $_SERVER['REQUEST_METHOD'] !== 'LOCK') {
if (isset($_SERVER['CONTENT_LENGTH']) && $_SERVER['REQUEST_METHOD'] === 'PUT') {
$expected = $_SERVER['CONTENT_LENGTH'];
if ($count != $expected) {
throw new BadRequest('expected filesize ' . $expected . ' got ' . $count);

View File

@ -89,6 +89,9 @@ class RootCollection extends SimpleCollection {
$systemAddressBookRoot = new AddressBookRoot(new SystemPrincipalBackend(), $systemCardDavBackend, 'principals/system');
$systemAddressBookRoot->disableListing = $disableListing;
$uploadCollection = new Upload\RootCollection($userPrincipalBackend, 'principals/users');
$uploadCollection->disableListing = $disableListing;
$children = [
new SimpleCollection('principals', [
$userPrincipals,
@ -102,6 +105,7 @@ class RootCollection extends SimpleCollection {
$systemTagCollection,
$systemTagRelationsCollection,
$commentsCollection,
$uploadCollection,
];
parent::__construct('root', $children);

View File

@ -0,0 +1,234 @@
<?php
namespace OCA\DAV\Upload;
use Sabre\DAV\IFile;
/**
* Class AssemblyStream
*
* The assembly stream is a virtual stream that wraps multiple chunks.
* Reading from the stream transparently accessed the underlying chunks and
* give a representation as if they were already merged together.
*
* @package OCA\DAV\Upload
*/
class AssemblyStream implements \Icewind\Streams\File {
/** @var resource */
private $context;
/** @var IFile[] */
private $nodes;
/** @var int */
private $pos = 0;
/** @var array */
private $sortedNodes;
/** @var int */
private $size;
/**
* @param string $path
* @param string $mode
* @param int $options
* @param string &$opened_path
* @return bool
*/
public function stream_open($path, $mode, $options, &$opened_path) {
$this->loadContext('assembly');
// sort the nodes
$nodes = $this->nodes;
// http://stackoverflow.com/a/10985500
@usort($nodes, function(IFile $a, IFile $b) {
return strcmp($a->getName(), $b->getName());
});
$this->nodes = $nodes;
// build additional information
$this->sortedNodes = [];
$start = 0;
foreach($this->nodes as $node) {
$size = $node->getSize();
$name = $node->getName();
$this->sortedNodes[$name] = ['node' => $node, 'start' => $start, 'end' => $start + $size];
$start += $size;
$this->size = $start;
}
return true;
}
/**
* @param string $offset
* @param int $whence
* @return bool
*/
public function stream_seek($offset, $whence = SEEK_SET) {
return false;
}
/**
* @return int
*/
public function stream_tell() {
return $this->pos;
}
/**
* @param int $count
* @return string
*/
public function stream_read($count) {
list($node, $posInNode) = $this->getNodeForPosition($this->pos);
if (is_null($node)) {
return null;
}
$stream = $this->getStream($node);
fseek($stream, $posInNode);
$data = fread($stream, $count);
$read = strlen($data);
// update position
$this->pos += $read;
return $data;
}
/**
* @param string $data
* @return int
*/
public function stream_write($data) {
return false;
}
/**
* @param int $option
* @param int $arg1
* @param int $arg2
* @return bool
*/
public function stream_set_option($option, $arg1, $arg2) {
return false;
}
/**
* @param int $size
* @return bool
*/
public function stream_truncate($size) {
return false;
}
/**
* @return array
*/
public function stream_stat() {
return [];
}
/**
* @param int $operation
* @return bool
*/
public function stream_lock($operation) {
return false;
}
/**
* @return bool
*/
public function stream_flush() {
return false;
}
/**
* @return bool
*/
public function stream_eof() {
return $this->pos >= $this->size;
}
/**
* @return bool
*/
public function stream_close() {
return true;
}
/**
* Load the source from the stream context and return the context options
*
* @param string $name
* @return array
* @throws \Exception
*/
protected function loadContext($name) {
$context = stream_context_get_options($this->context);
if (isset($context[$name])) {
$context = $context[$name];
} else {
throw new \BadMethodCallException('Invalid context, "' . $name . '" options not set');
}
if (isset($context['nodes']) and is_array($context['nodes'])) {
$this->nodes = $context['nodes'];
} else {
throw new \BadMethodCallException('Invalid context, nodes not set');
}
return $context;
}
/**
* @param IFile[] $nodes
* @return resource
*
* @throws \BadMethodCallException
*/
public static function wrap(array $nodes) {
$context = stream_context_create([
'assembly' => [
'nodes' => $nodes]
]);
stream_wrapper_register('assembly', '\OCA\DAV\Upload\AssemblyStream');
try {
$wrapped = fopen('assembly://', 'r', null, $context);
} catch (\BadMethodCallException $e) {
stream_wrapper_unregister('assembly');
throw $e;
}
stream_wrapper_unregister('assembly');
return $wrapped;
}
/**
* @param $pos
* @return IFile | null
*/
private function getNodeForPosition($pos) {
foreach($this->sortedNodes as $node) {
if ($pos >= $node['start'] && $pos < $node['end']) {
return [$node['node'], $pos - $node['start']];
}
}
return null;
}
/**
* @param IFile $node
* @return resource
*/
private function getStream(IFile $node) {
$data = $node->get();
if (is_resource($data)) {
return $data;
}
return fopen('data://text/plain,' . $data,'r');
}
}

View File

@ -0,0 +1,103 @@
<?php
namespace OCA\DAV\Upload;
use OCA\DAV\Connector\Sabre\Directory;
use OCA\DAV\Upload\AssemblyStream;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\IFile;
/**
* Class FutureFile
*
* The FutureFile is a SabreDav IFile which connects the chunked upload directory
* with the AssemblyStream, who does the final assembly job
*
* @package OCA\DAV\Upload
*/
class FutureFile implements \Sabre\DAV\IFile {
/** @var Directory */
private $root;
/** @var string */
private $name;
/**
* @param Directory $root
* @param string $name
*/
function __construct(Directory $root, $name) {
$this->root = $root;
$this->name = $name;
}
/**
* @inheritdoc
*/
function put($data) {
throw new Forbidden('Permission denied to put into this file');
}
/**
* @inheritdoc
*/
function get() {
$nodes = $this->root->getChildren();
return AssemblyStream::wrap($nodes);
}
/**
* @inheritdoc
*/
function getContentType() {
return 'application/octet-stream';
}
/**
* @inheritdoc
*/
function getETag() {
return $this->root->getETag();
}
/**
* @inheritdoc
*/
function getSize() {
$children = $this->root->getChildren();
$sizes = array_map(function($node) {
/** @var IFile $node */
return $node->getSize();
}, $children);
return array_sum($sizes);
}
/**
* @inheritdoc
*/
function delete() {
$this->root->delete();
}
/**
* @inheritdoc
*/
function getName() {
return $this->name;
}
/**
* @inheritdoc
*/
function setName($name) {
throw new Forbidden('Permission denied to rename this file');
}
/**
* @inheritdoc
*/
function getLastModified() {
return $this->root->getLastModified();
}
}

View File

@ -0,0 +1,23 @@
<?php
namespace OCA\DAV\Upload;
use Sabre\DAVACL\AbstractPrincipalCollection;
class RootCollection extends AbstractPrincipalCollection {
/**
* @inheritdoc
*/
function getChildForPrincipal(array $principalInfo) {
return new UploadHome($principalInfo);
}
/**
* @inheritdoc
*/
function getName() {
return 'uploads';
}
}

View File

@ -0,0 +1,61 @@
<?php
namespace OCA\DAV\Upload;
use OCA\DAV\Connector\Sabre\Directory;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\ICollection;
class UploadFolder implements ICollection {
private $node;
function __construct(Directory $node) {
$this->node = $node;
}
function createFile($name, $data = null) {
// TODO: verify name - should be a simple number
$this->node->createFile($name, $data);
}
function createDirectory($name) {
throw new Forbidden('Permission denied to create file (filename ' . $name . ')');
}
function getChild($name) {
if ($name === '.file') {
return new FutureFile($this->node, '.file');
}
return $this->node->getChild($name);
}
function getChildren() {
$children = $this->node->getChildren();
$children[] = new FutureFile($this->node, '.file');
return $children;
}
function childExists($name) {
if ($name === '.file') {
return true;
}
return $this->node->childExists($name);
}
function delete() {
$this->node->delete();
}
function getName() {
return $this->node->getName();
}
function setName($name) {
throw new Forbidden('Permission denied to rename this folder');
}
function getLastModified() {
return $this->node->getLastModified();
}
}

View File

@ -0,0 +1,74 @@
<?php
namespace OCA\DAV\Upload;
use OC\Files\Filesystem;
use OC\Files\View;
use OCA\DAV\Connector\Sabre\Directory;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\ICollection;
class UploadHome implements ICollection {
/**
* FilesHome constructor.
*
* @param array $principalInfo
*/
public function __construct($principalInfo) {
$this->principalInfo = $principalInfo;
}
function createFile($name, $data = null) {
throw new Forbidden('Permission denied to create file (filename ' . $name . ')');
}
function createDirectory($name) {
$this->impl()->createDirectory($name);
}
function getChild($name) {
return new UploadFolder($this->impl()->getChild($name));
}
function getChildren() {
return array_map(function($node) {
return new UploadFolder($node);
}, $this->impl()->getChildren());
}
function childExists($name) {
return !is_null($this->getChild($name));
}
function delete() {
$this->impl()->delete();
}
function getName() {
return 'uploads';
}
function setName($name) {
throw new Forbidden('Permission denied to rename this folder');
}
function getLastModified() {
return $this->impl()->getLastModified();
}
/**
* @return Directory
*/
private function impl() {
$rootView = new View();
$user = \OC::$server->getUserSession()->getUser();
Filesystem::initMountPoints($user->getUID());
if (!$rootView->file_exists('/' . $user->getUID() . '/uploads')) {
$rootView->mkdir('/' . $user->getUID() . '/uploads');
}
$view = new View('/' . $user->getUID() . '/uploads');
$rootInfo = $view->getFileInfo('');
$impl = new Directory($view, $rootInfo);
return $impl;
}
}

View File

@ -0,0 +1,47 @@
<?php
class AssemblyStreamTest extends \PHPUnit_Framework_TestCase {
/**
* @dataProvider providesNodes()
*/
public function testGetContents($expected, $nodes) {
$stream = \OCA\DAV\Upload\AssemblyStream::wrap($nodes);
$content = stream_get_contents($stream);
$this->assertEquals($expected, $content);
}
function providesNodes() {
return[
'one node only' => ['1234567890', [
$this->buildNode('0', '1234567890')
]],
'two nodes' => ['1234567890', [
$this->buildNode('1', '67890'),
$this->buildNode('0', '12345')
]]
];
}
private function buildNode($name, $data) {
$node = $this->getMockBuilder('\Sabre\DAV\File')
->setMethods(['getName', 'get', 'getSize'])
->getMockForAbstractClass();
$node->expects($this->any())
->method('getName')
->willReturn($name);
$node->expects($this->any())
->method('get')
->willReturn($data);
$node->expects($this->any())
->method('getSize')
->willReturn(strlen($data));
return $node;
}
}

View File

@ -0,0 +1,89 @@
<?php
class FutureFileTest extends \PHPUnit_Framework_TestCase {
public function testGetContentType() {
$f = $this->mockFutureFile();
$this->assertEquals('application/octet-stream', $f->getContentType());
}
public function testGetETag() {
$f = $this->mockFutureFile();
$this->assertEquals('1234567890', $f->getETag());
}
public function testGetName() {
$f = $this->mockFutureFile();
$this->assertEquals('foo.txt', $f->getName());
}
public function testGetLastModified() {
$f = $this->mockFutureFile();
$this->assertEquals(12121212, $f->getLastModified());
}
public function testGetSize() {
$f = $this->mockFutureFile();
$this->assertEquals(0, $f->getSize());
}
public function testGet() {
$f = $this->mockFutureFile();
$stream = $f->get();
$this->assertTrue(is_resource($stream));
}
public function testDelete() {
$d = $this->getMockBuilder('OCA\DAV\Connector\Sabre\Directory')
->disableOriginalConstructor()
->setMethods(['delete'])
->getMock();
$d->expects($this->once())
->method('delete');
$f = new \OCA\DAV\Upload\FutureFile($d, 'foo.txt');
$f->delete();
}
/**
* @expectedException Sabre\DAV\Exception\Forbidden
*/
public function testPut() {
$f = $this->mockFutureFile();
$f->put('');
}
/**
* @expectedException Sabre\DAV\Exception\Forbidden
*/
public function testSetName() {
$f = $this->mockFutureFile();
$f->setName('');
}
/**
* @return \OCA\DAV\Upload\FutureFile
*/
private function mockFutureFile() {
$d = $this->getMockBuilder('OCA\DAV\Connector\Sabre\Directory')
->disableOriginalConstructor()
->setMethods(['getETag', 'getLastModified', 'getChildren'])
->getMock();
$d->expects($this->any())
->method('getETag')
->willReturn('1234567890');
$d->expects($this->any())
->method('getLastModified')
->willReturn(12121212);
$d->expects($this->any())
->method('getChildren')
->willReturn([]);
return new \OCA\DAV\Upload\FutureFile($d, 'foo.txt');
}
}