From e8010209bb1ec8ef9ecc1cff7ac2b2d4d414bd74 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Sun, 8 Jul 2012 22:11:13 +0200 Subject: [PATCH 1/4] Custom chunking support --- lib/connector/sabre/directory.php | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/connector/sabre/directory.php b/lib/connector/sabre/directory.php index b75bb5c50f..894256d5b2 100644 --- a/lib/connector/sabre/directory.php +++ b/lib/connector/sabre/directory.php @@ -33,10 +33,31 @@ class OC_Connector_Sabre_Directory extends OC_Connector_Sabre_Node implements Sa * @return void */ public function createFile($name, $data = null) { - - $newPath = $this->path . '/' . $name; - OC_Filesystem::file_put_contents($newPath,$data); - + if (isset($_SERVER['HTTP_OC_CHUNKED'])) { + $cache = new OC_Cache_File(); + $cache->set($name, $data); + preg_match('/(?P.*)-chunking-(?P\d+)-(?P\d+)-(?P\d+)/', $name, $matches); + $prefix = $matches['name'].'-chunking-'.$matches['transferid'].'-'.$matches['chunkcount'].'-'; + $parts = 0; + for($i=0; $i < $matches['chunkcount']; $i++) { + if ($cache->hasKey($prefix.$i)) { + $parts ++; + } + } + if ($parts == $matches['chunkcount']) { + $newPath = $this->path . '/' . $matches['name']; + $f = OC_Filesystem::fopen($newPath, 'w'); + for($i=0; $i < $matches['chunkcount']; $i++) { + $chunk = $cache->get($prefix.$i); + $cache->remove($prefix.$i); + fwrite($f,$chunk); + } + fclose($f); + } + } else { + $newPath = $this->path . '/' . $name; + OC_Filesystem::file_put_contents($newPath,$data); + } } /** From 2d85ef0e045380691b1dcf136b4b61b46104c2c6 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Fri, 27 Jul 2012 11:59:52 +0200 Subject: [PATCH 2/4] Chunked upload: Refactor to static class --- lib/connector/sabre/directory.php | 23 +++----------- lib/filechunking.php | 53 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 18 deletions(-) create mode 100644 lib/filechunking.php diff --git a/lib/connector/sabre/directory.php b/lib/connector/sabre/directory.php index bed75015ae..b88d0f3286 100644 --- a/lib/connector/sabre/directory.php +++ b/lib/connector/sabre/directory.php @@ -49,25 +49,12 @@ class OC_Connector_Sabre_Directory extends OC_Connector_Sabre_Node implements Sa */ public function createFile($name, $data = null) { if (isset($_SERVER['HTTP_OC_CHUNKED'])) { - $cache = new OC_Cache_File(); - $cache->set($name, $data); - preg_match('/(?P.*)-chunking-(?P\d+)-(?P\d+)-(?P\d+)/', $name, $matches); - $prefix = $matches['name'].'-chunking-'.$matches['transferid'].'-'.$matches['chunkcount'].'-'; - $parts = 0; - for($i=0; $i < $matches['chunkcount']; $i++) { - if ($cache->hasKey($prefix.$i)) { - $parts ++; - } - } - if ($parts == $matches['chunkcount']) { - $newPath = $this->path . '/' . $matches['name']; + OC_FileChunking::store($name, $data); + $info = OC_FileChunking::decodeName($name); + if (OC_FileChunking::isComplete($info)) { + $newPath = $this->path . '/' . $info['name']; $f = OC_Filesystem::fopen($newPath, 'w'); - for($i=0; $i < $matches['chunkcount']; $i++) { - $chunk = $cache->get($prefix.$i); - $cache->remove($prefix.$i); - fwrite($f,$chunk); - } - fclose($f); + OC_FileChunking::assemble($info, $f); return OC_Connector_Sabre_Node::getETagPropertyForPath($newPath); } } else { diff --git a/lib/filechunking.php b/lib/filechunking.php new file mode 100644 index 0000000000..4e4918d49d --- /dev/null +++ b/lib/filechunking.php @@ -0,0 +1,53 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + + +class OC_FileChunking { + static public function decodeName($name) { + preg_match('/(?P.*)-chunking-(?P\d+)-(?P\d+)-(?P\d+)/', $name, $matches); + return $matches; + } + + static public function getPrefix($name, $transferid, $chunkcount) { + return $name.'-chunking-'.$transferid.'-'.$chunkcount.'-'; + } + + static public function store($name, $data) { + $cache = new OC_Cache_File(); + $cache->set($name, $data); + } + + static public function isComplete($info) { + $prefix = OC_FileChunking::getPrefix($info['name'], + $info['transferid'], + $info['chunkcount'] + ); + $parts = 0; + $cache = new OC_Cache_File(); + for($i=0; $i < $info['chunkcount']; $i++) { + if ($cache->hasKey($prefix.$i)) { + $parts ++; + } + } + return $parts == $info['chunkcount']; + } + + static public function assemble($info, $f) { + $cache = new OC_Cache_File(); + $prefix = OC_FileChunking::getPrefix($info['name'], + $info['transferid'], + $info['chunkcount'] + ); + for($i=0; $i < $info['chunkcount']; $i++) { + $chunk = $cache->get($prefix.$i); + $cache->remove($prefix.$i); + fwrite($f,$chunk); + } + fclose($f); + } +} From 1ea33ff36bf70ee2099f8d225f488dc220e3bcff Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Fri, 27 Jul 2012 12:14:18 +0200 Subject: [PATCH 3/4] Chunked upload: Refactor OC_FileChunking to object --- lib/connector/sabre/directory.php | 7 +++-- lib/filechunking.php | 49 +++++++++++++++++++------------ 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/lib/connector/sabre/directory.php b/lib/connector/sabre/directory.php index b88d0f3286..09c65f19b8 100644 --- a/lib/connector/sabre/directory.php +++ b/lib/connector/sabre/directory.php @@ -49,12 +49,13 @@ class OC_Connector_Sabre_Directory extends OC_Connector_Sabre_Node implements Sa */ public function createFile($name, $data = null) { if (isset($_SERVER['HTTP_OC_CHUNKED'])) { - OC_FileChunking::store($name, $data); $info = OC_FileChunking::decodeName($name); - if (OC_FileChunking::isComplete($info)) { + $chunk_handler = new OC_FileChunking($info); + $chunk_handler->store($info['index'], $data); + if ($chunk_handler->isComplete()) { $newPath = $this->path . '/' . $info['name']; $f = OC_Filesystem::fopen($newPath, 'w'); - OC_FileChunking::assemble($info, $f); + $chunk_handler->assemble($f); return OC_Connector_Sabre_Node::getETagPropertyForPath($newPath); } } else { diff --git a/lib/filechunking.php b/lib/filechunking.php index 4e4918d49d..9aead4c12f 100644 --- a/lib/filechunking.php +++ b/lib/filechunking.php @@ -8,42 +8,55 @@ class OC_FileChunking { + protected $info; + protected $cache; + static public function decodeName($name) { preg_match('/(?P.*)-chunking-(?P\d+)-(?P\d+)-(?P\d+)/', $name, $matches); return $matches; } - static public function getPrefix($name, $transferid, $chunkcount) { + public function __construct($info) { + $this->info = $info; + } + + public function getPrefix() { + $name = $this->info['name']; + $transferid = $this->info['transferid']; + $chunkcount = $this->info['chunkcount']; + return $name.'-chunking-'.$transferid.'-'.$chunkcount.'-'; } - static public function store($name, $data) { - $cache = new OC_Cache_File(); + protected function getCache() { + if (!isset($this->cache)) { + $this->cache = new OC_Cache_File(); + } + return $this->cache; + } + + public function store($index, $data) { + $cache = $this->getCache(); + $name = $this->getPrefix().$index; $cache->set($name, $data); } - static public function isComplete($info) { - $prefix = OC_FileChunking::getPrefix($info['name'], - $info['transferid'], - $info['chunkcount'] - ); + public function isComplete() { + $prefix = $this->getPrefix(); $parts = 0; - $cache = new OC_Cache_File(); - for($i=0; $i < $info['chunkcount']; $i++) { + $cache = $this->getCache(); + for($i=0; $i < $this->info['chunkcount']; $i++) { if ($cache->hasKey($prefix.$i)) { $parts ++; } } - return $parts == $info['chunkcount']; + return $parts == $this->info['chunkcount']; } - static public function assemble($info, $f) { - $cache = new OC_Cache_File(); - $prefix = OC_FileChunking::getPrefix($info['name'], - $info['transferid'], - $info['chunkcount'] - ); - for($i=0; $i < $info['chunkcount']; $i++) { + public function assemble($f) { + $cache = $this->getCache(); + $prefix = $this->getPrefix(); + for($i=0; $i < $this->info['chunkcount']; $i++) { $chunk = $cache->get($prefix.$i); $cache->remove($prefix.$i); fwrite($f,$chunk); From 896d27de36dd26515f30fa3b0748c9c6089600af Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Fri, 27 Jul 2012 19:02:23 +0200 Subject: [PATCH 4/4] Chunked upload: Support reusing local chunks --- apps/files/appinfo/filesync.php | 64 +++++++++++++++++++++++++++++++++ apps/files/appinfo/info.xml | 1 + apps/files/appinfo/version | 2 +- lib/filechunking.php | 32 +++++++++++++++-- 4 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 apps/files/appinfo/filesync.php diff --git a/apps/files/appinfo/filesync.php b/apps/files/appinfo/filesync.php new file mode 100644 index 0000000000..707ee2435c --- /dev/null +++ b/apps/files/appinfo/filesync.php @@ -0,0 +1,64 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +/** + * filesync can be called with a PUT method. + * PUT takes a stream starting with a 2 byte blocksize, + * followed by binary md5 of the blocks. Everything in big-endian. + * The return is a json encoded with: + * - 'transferid' + * - 'needed' chunks + * - 'last' checked chunk + * The URL is made of 3 parts, the service url (remote.php/filesync/), the sync + * type and the path in ownCloud. + * At the moment the only supported sync type is 'oc_chunked'. + * The final URL will look like http://.../remote.php/filesync/oc_chunked/path/to/file + */ + +// only need filesystem apps +$RUNTIME_APPTYPES=array('filesystem','authentication'); +OC_App::loadApps($RUNTIME_APPTYPES); +if(!OC_User::isLoggedIn()){ + if(!isset($_SERVER['PHP_AUTH_USER'])){ + header('WWW-Authenticate: Basic realm="ownCloud Server"'); + header('HTTP/1.0 401 Unauthorized'); + echo 'Valid credentials must be supplied'; + exit(); + } else { + if(!OC_User::login($_SERVER["PHP_AUTH_USER"], $_SERVER["PHP_AUTH_PW"])){ + exit(); + } + } +} + +list($type,$file) = explode('/', substr($path_info,1+strlen($service)+1), 2); + +if ($type != 'oc_chunked') { + OC_Response::setStatus(OC_Response::STATUS_NOT_FOUND); + die; +} + +if (!OC_Filesystem::is_file($file)) { + OC_Response::setStatus(OC_Response::STATUS_NOT_FOUND); + die; +} + +switch($_SERVER['REQUEST_METHOD']) { + case 'PUT': + $input = fopen("php://input", "r"); + $org_file = OC_Filesystem::fopen($file, 'rb'); + $info = array( + 'name' => basename($file), + ); + $sync = new OC_FileChunking($info); + $result = $sync->signature_split($org_file, $input); + echo json_encode($result); + break; + default: + OC_Response::setStatus(OC_Response::STATUS_NOT_FOUND); +} diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index 105df092ce..e58f83c5a0 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -15,5 +15,6 @@ appinfo/remote.php appinfo/remote.php + appinfo/filesync.php diff --git a/apps/files/appinfo/version b/apps/files/appinfo/version index 1b87bcd0b0..e25d8d9f35 100644 --- a/apps/files/appinfo/version +++ b/apps/files/appinfo/version @@ -1 +1 @@ -1.1.4 \ No newline at end of file +1.1.5 diff --git a/lib/filechunking.php b/lib/filechunking.php index 9aead4c12f..d03af226d8 100644 --- a/lib/filechunking.php +++ b/lib/filechunking.php @@ -23,9 +23,8 @@ class OC_FileChunking { public function getPrefix() { $name = $this->info['name']; $transferid = $this->info['transferid']; - $chunkcount = $this->info['chunkcount']; - return $name.'-chunking-'.$transferid.'-'.$chunkcount.'-'; + return $name.'-chunking-'.$transferid.'-'; } protected function getCache() { @@ -63,4 +62,33 @@ class OC_FileChunking { } fclose($f); } + + public function signature_split($orgfile, $input) { + $info = unpack('n', fread($input, 2)); + $blocksize = $info[1]; + $this->info['transferid'] = mt_rand(); + $count = 0; + $needed = array(); + $cache = $this->getCache(); + $prefix = $this->getPrefix(); + while (!feof($orgfile)) { + $new_md5 = fread($input, 16); + if (feof($input)) { + break; + } + $data = fread($orgfile, $blocksize); + $org_md5 = md5($data, true); + if ($org_md5 == $new_md5) { + $cache->set($prefix.$count, $data); + } else { + $needed[] = $count; + } + $count++; + } + return array( + 'transferid' => $this->info['transferid'], + 'needed' => $needed, + 'count' => $count, + ); + } }