From 6195f13bda086b242cbcdbff86260a81bf6163c9 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 16 Oct 2013 15:37:58 +0200 Subject: [PATCH 01/17] Use CURL to get filesize on 32bit systems. --- lib/private/files/storage/local.php | 75 ++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index 943c416308..fc2a590f6c 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -90,10 +90,10 @@ if (\OC_Util::runningOnWindows()) { $fullPath = $this->datadir . $path; $statResult = stat($fullPath); - if ($statResult['size'] < 0) { - $size = self::getFileSizeFromOS($fullPath); - $statResult['size'] = $size; - $statResult[7] = $size; + $filesize = self::getFileSizeWithTricks($fullPath); + if (!is_null($filesize)) { + $statResult['size'] = $filesize; + $statResult[7] = $filesize; } return $statResult; } @@ -111,12 +111,13 @@ if (\OC_Util::runningOnWindows()) { return 0; } else { $fullPath = $this->datadir . $path; - $fileSize = filesize($fullPath); - if ($fileSize < 0) { - return self::getFileSizeFromOS($fullPath); + + $filesize = self::getFileSizeWithTricks($fullPath); + if (!is_null($filesize)) { + return $filesize; } - return $fileSize; + return filesize($fullPath); } } @@ -221,8 +222,60 @@ if (\OC_Util::runningOnWindows()) { } /** - * @param string $fullPath - */ + * @brief Tries to get the filesize via various workarounds if necessary. + * @param string $fullPath + * @return mixed Number of bytes as float on success and workaround necessary, null otherwise. + */ + private static function getFileSizeWithTricks($fullPath) { + if (PHP_INT_SIZE === 4) { + // filesize() and stat() are unreliable on 32bit systems + // for big files. + // In some cases they cause an E_WARNING and return false, + // in some other cases they silently wrap around at 2^32, + // i.e. e.g. report 674347008 bytes instead of 4969314304. + $filesize = self::getFileSizeFromCurl($fullPath); + if (!is_null($filesize)) { + return $filesize; + } + $filesize = self::getFileSizeFromOS($fullPath); + if (!is_null($filesize)) { + return $filesize; + } + } + + return null; + } + + /** + * @brief Tries to get the filesize via a CURL HEAD request. + * @param string $fullPath + * @return mixed Number of bytes as float on success, null otherwise. + */ + private static function getFileSizeFromCurl($fullPath) { + if (function_exists('curl_init')) { + $ch = curl_init("file://$fullPath"); + curl_setopt($ch, CURLOPT_NOBODY, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HEADER, true); + $data = curl_exec($ch); + curl_close($ch); + if ($data !== false) { + $matches = array(); + preg_match('/Content-Length: (\d+)/', $data, $matches); + if (isset($matches[1])) { + return (float) $matches[1]; + } + } + } + + return null; + } + + /** + * @brief Tries to get the filesize via COM and exec(). + * @param string $fullPath + * @return mixed Number of bytes as float on success, null otherwise. + */ private static function getFileSizeFromOS($fullPath) { $name = strtolower(php_uname('s')); // Windows OS: we use COM to access the filesystem @@ -246,7 +299,7 @@ if (\OC_Util::runningOnWindows()) { \OC_Log::ERROR); } - return 0; + return null; } public function hash($type, $path, $raw = false) { From 3f8f8027d25ab39283d0ab13afcf7252dde8f240 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 7 Jan 2014 13:50:57 +0100 Subject: [PATCH 02/17] Cast to numeric instead of float, i.e. use an integer if possible. --- lib/private/files/storage/local.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index fc2a590f6c..7fa748a7fd 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -224,7 +224,7 @@ if (\OC_Util::runningOnWindows()) { /** * @brief Tries to get the filesize via various workarounds if necessary. * @param string $fullPath - * @return mixed Number of bytes as float on success and workaround necessary, null otherwise. + * @return mixed Number of bytes on success and workaround necessary, null otherwise. */ private static function getFileSizeWithTricks($fullPath) { if (PHP_INT_SIZE === 4) { @@ -249,7 +249,7 @@ if (\OC_Util::runningOnWindows()) { /** * @brief Tries to get the filesize via a CURL HEAD request. * @param string $fullPath - * @return mixed Number of bytes as float on success, null otherwise. + * @return mixed Number of bytes on success, null otherwise. */ private static function getFileSizeFromCurl($fullPath) { if (function_exists('curl_init')) { @@ -263,7 +263,7 @@ if (\OC_Util::runningOnWindows()) { $matches = array(); preg_match('/Content-Length: (\d+)/', $data, $matches); if (isset($matches[1])) { - return (float) $matches[1]; + return 0 + $matches[1]; } } } @@ -274,7 +274,7 @@ if (\OC_Util::runningOnWindows()) { /** * @brief Tries to get the filesize via COM and exec(). * @param string $fullPath - * @return mixed Number of bytes as float on success, null otherwise. + * @return mixed Number of bytes on success, null otherwise. */ private static function getFileSizeFromOS($fullPath) { $name = strtolower(php_uname('s')); @@ -287,11 +287,11 @@ if (\OC_Util::runningOnWindows()) { } } else if (strpos($name, 'bsd') !== false) { if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -f %z ' . escapeshellarg($fullPath)); + return 0 + exec('stat -f %z ' . escapeshellarg($fullPath)); } } else if (strpos($name, 'linux') !== false) { if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -c %s ' . escapeshellarg($fullPath)); + return 0 + exec('stat -c %s ' . escapeshellarg($fullPath)); } } else { \OC_Log::write('core', From c8fa1fd68784c48c1df537310f16d6753f79b029 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 9 Feb 2014 01:25:33 +0100 Subject: [PATCH 03/17] Refactor Large File handling code. --- lib/private/files/storage/local.php | 98 ++------------------ lib/private/files/storage/mappedlocal.php | 55 +++--------- lib/private/largefilehelper.php | 103 ++++++++++++++++++++++ 3 files changed, 124 insertions(+), 132 deletions(-) create mode 100644 lib/private/largefilehelper.php diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index 7fa748a7fd..883a69d768 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -89,9 +89,8 @@ if (\OC_Util::runningOnWindows()) { public function stat($path) { $fullPath = $this->datadir . $path; $statResult = stat($fullPath); - - $filesize = self::getFileSizeWithTricks($fullPath); - if (!is_null($filesize)) { + if (PHP_INT_SIZE === 4) { + $filesize = $this->filesize($path); $statResult['size'] = $filesize; $statResult[7] = $filesize; } @@ -109,16 +108,16 @@ if (\OC_Util::runningOnWindows()) { public function filesize($path) { if ($this->is_dir($path)) { return 0; - } else { - $fullPath = $this->datadir . $path; - - $filesize = self::getFileSizeWithTricks($fullPath); + } + $fullPath = $this->datadir . $path; + if (PHP_INT_SIZE === 4) { + $helper = new \OC\LargeFileHelper; + $filesize = $helper->getFilesize($fullPath); if (!is_null($filesize)) { return $filesize; } - - return filesize($fullPath); } + return filesize($fullPath); } public function isReadable($path) { @@ -221,87 +220,6 @@ if (\OC_Util::runningOnWindows()) { return $return; } - /** - * @brief Tries to get the filesize via various workarounds if necessary. - * @param string $fullPath - * @return mixed Number of bytes on success and workaround necessary, null otherwise. - */ - private static function getFileSizeWithTricks($fullPath) { - if (PHP_INT_SIZE === 4) { - // filesize() and stat() are unreliable on 32bit systems - // for big files. - // In some cases they cause an E_WARNING and return false, - // in some other cases they silently wrap around at 2^32, - // i.e. e.g. report 674347008 bytes instead of 4969314304. - $filesize = self::getFileSizeFromCurl($fullPath); - if (!is_null($filesize)) { - return $filesize; - } - $filesize = self::getFileSizeFromOS($fullPath); - if (!is_null($filesize)) { - return $filesize; - } - } - - return null; - } - - /** - * @brief Tries to get the filesize via a CURL HEAD request. - * @param string $fullPath - * @return mixed Number of bytes on success, null otherwise. - */ - private static function getFileSizeFromCurl($fullPath) { - if (function_exists('curl_init')) { - $ch = curl_init("file://$fullPath"); - curl_setopt($ch, CURLOPT_NOBODY, true); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HEADER, true); - $data = curl_exec($ch); - curl_close($ch); - if ($data !== false) { - $matches = array(); - preg_match('/Content-Length: (\d+)/', $data, $matches); - if (isset($matches[1])) { - return 0 + $matches[1]; - } - } - } - - return null; - } - - /** - * @brief Tries to get the filesize via COM and exec(). - * @param string $fullPath - * @return mixed Number of bytes on success, null otherwise. - */ - private static function getFileSizeFromOS($fullPath) { - $name = strtolower(php_uname('s')); - // Windows OS: we use COM to access the filesystem - if (strpos($name, 'win') !== false) { - if (class_exists('COM')) { - $fsobj = new \COM("Scripting.FileSystemObject"); - $f = $fsobj->GetFile($fullPath); - return $f->Size; - } - } else if (strpos($name, 'bsd') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return 0 + exec('stat -f %z ' . escapeshellarg($fullPath)); - } - } else if (strpos($name, 'linux') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return 0 + exec('stat -c %s ' . escapeshellarg($fullPath)); - } - } else { - \OC_Log::write('core', - 'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name, - \OC_Log::ERROR); - } - - return null; - } - public function hash($type, $path, $raw = false) { return hash_file($type, $this->datadir . $path, $raw); } diff --git a/lib/private/files/storage/mappedlocal.php b/lib/private/files/storage/mappedlocal.php index 3ebdcf9538..78d2616ba9 100644 --- a/lib/private/files/storage/mappedlocal.php +++ b/lib/private/files/storage/mappedlocal.php @@ -111,11 +111,10 @@ class MappedLocal extends \OC\Files\Storage\Common { public function stat($path) { $fullPath = $this->buildPath($path); $statResult = stat($fullPath); - - if ($statResult['size'] < 0) { - $size = self::getFileSizeFromOS($fullPath); - $statResult['size'] = $size; - $statResult[7] = $size; + if (PHP_INT_SIZE === 4) { + $filesize = $this->filesize($path); + $statResult['size'] = $filesize; + $statResult[7] = $filesize; } return $statResult; } @@ -131,15 +130,16 @@ class MappedLocal extends \OC\Files\Storage\Common { public function filesize($path) { if ($this->is_dir($path)) { return 0; - } else { - $fullPath = $this->buildPath($path); - $fileSize = filesize($fullPath); - if ($fileSize < 0) { - return self::getFileSizeFromOS($fullPath); - } - - return $fileSize; } + $fullPath = $this->buildPath($path); + if (PHP_INT_SIZE === 4) { + $helper = new \OC\LargeFileHelper; + $filesize = $helper->getFilesize($fullPath); + if (!is_null($filesize)) { + return $filesize; + } + } + return filesize($fullPath); } public function isReadable($path) { @@ -294,35 +294,6 @@ class MappedLocal extends \OC\Files\Storage\Common { return $return; } - /** - * @param string $fullPath - */ - private static function getFileSizeFromOS($fullPath) { - $name = strtolower(php_uname('s')); - // Windows OS: we use COM to access the filesystem - if (strpos($name, 'win') !== false) { - if (class_exists('COM')) { - $fsobj = new \COM("Scripting.FileSystemObject"); - $f = $fsobj->GetFile($fullPath); - return $f->Size; - } - } else if (strpos($name, 'bsd') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -f %z ' . escapeshellarg($fullPath)); - } - } else if (strpos($name, 'linux') !== false) { - if (\OC_Helper::is_function_enabled('exec')) { - return (float)exec('stat -c %s ' . escapeshellarg($fullPath)); - } - } else { - \OC_Log::write('core', - 'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name, - \OC_Log::ERROR); - } - - return 0; - } - public function hash($type, $path, $raw = false) { return hash_file($type, $this->buildPath($path), $raw); } diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php new file mode 100644 index 0000000000..ca8f752217 --- /dev/null +++ b/lib/private/largefilehelper.php @@ -0,0 +1,103 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC; + +/** + * Helper class for large files on 32-bit platforms. + */ +class LargeFileHelper { + /** + * @brief Tries to get the filesize of a file via various workarounds that + * even work for large files on 32-bit platforms. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesize($filename) { + $filesize = $this->getFilesizeViaCurl($filename); + if (!is_null($filesize)) { + return $filesize; + } + $filesize = $this->getFilesizeViaCOM($filename); + if (!is_null($filesize)) { + return $filesize; + } + $filesize = $this->getFilesizeViaExec($filename); + if (!is_null($filesize)) { + return $filesize; + } + return null; + } + + /** + * @brief Tries to get the filesize of a file via a CURL HEAD request. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesizeViaCurl($filename) { + if (function_exists('curl_init')) { + $ch = curl_init("file://$filename"); + curl_setopt($ch, CURLOPT_NOBODY, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_HEADER, true); + $data = curl_exec($ch); + curl_close($ch); + if ($data !== false) { + $matches = array(); + preg_match('/Content-Length: (\d+)/', $data, $matches); + if (isset($matches[1])) { + return 0 + $matches[1]; + } + } + } + return null; + } + + /** + * @brief Tries to get the filesize of a file via the Windows DOM extension. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesizeViaCOM($filename) { + if (class_exists('COM')) { + $fsobj = new \COM("Scripting.FileSystemObject"); + $file = $fsobj->GetFile($filename); + return 0 + $file->Size; + } + return null; + } + + /** + * @brief Tries to get the filesize of a file via an exec() call. + * + * @param string $filename Path to the file. + * + * @return null|int|float Number of bytes as number (float or int) or + * null on failure. + */ + public function getFilesizeViaExec($filename) { + if (\OC_Helper::is_function_enabled('exec')) { + $os = strtolower(php_uname('s')); + if (strpos($os, 'linux') !== false) { + return 0 + exec('stat -c %s ' . escapeshellarg($filename)); + } else if (strpos($os, 'bsd') !== false) { + return 0 + exec('stat -f %z ' . escapeshellarg($filename)); + } + } + return null; + } +} From 68cc0ba2e188c39b87f9eea0011846c0b644e09b Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 9 Feb 2014 15:53:58 +0100 Subject: [PATCH 04/17] Unit Tests for LargeFileHelper. --- tests/lib/largefilehelper.php | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/lib/largefilehelper.php diff --git a/tests/lib/largefilehelper.php b/tests/lib/largefilehelper.php new file mode 100644 index 0000000000..43721a3ee8 --- /dev/null +++ b/tests/lib/largefilehelper.php @@ -0,0 +1,62 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test; + +/** +* Tests whether LargeFileHelper is able to determine filesize at all. +* Large files are not considered yet. +*/ +class LargeFileHelper extends \PHPUnit_Framework_TestCase { + protected $filename; + protected $filesize; + protected $helper; + + public function setUp() { + parent::setUp(); + $this->filename = __DIR__ . '/../data/data.tar.gz'; + $this->filesize = 4195; + $this->helper = new \OC\LargeFileHelper; + } + + public function testGetFilesizeViaCurl() { + if (!extension_loaded('curl')) { + $this->markTestSkipped( + 'The PHP curl extension is required for this test.' + ); + } + $this->assertSame( + $this->filesize, + $this->helper->getFilesizeViaCurl($this->filename) + ); + } + + public function testGetFilesizeViaCOM() { + if (!extension_loaded('COM')) { + $this->markTestSkipped( + 'The PHP Windows COM extension is required for this test.' + ); + } + $this->assertSame( + $this->filesize, + $this->helper->getFilesizeViaDOM($this->filename) + ); + } + + public function testGetFilesizeViaExec() { + if (!\OC_Helper::is_function_enabled('exec')) { + $this->markTestSkipped( + 'The exec() function needs to be enabled for this test.' + ); + } + $this->assertSame( + $this->filesize, + $this->helper->getFilesizeViaExec($this->filename) + ); + } +} From 626e87aa542adaee123c7c4104411238b43b3333 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 9 Feb 2014 15:56:26 +0100 Subject: [PATCH 05/17] Output validation for exec() method. --- lib/private/largefilehelper.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index ca8f752217..751e60de53 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -92,10 +92,15 @@ class LargeFileHelper { public function getFilesizeViaExec($filename) { if (\OC_Helper::is_function_enabled('exec')) { $os = strtolower(php_uname('s')); + $result = ''; if (strpos($os, 'linux') !== false) { - return 0 + exec('stat -c %s ' . escapeshellarg($filename)); + $result = trim(exec('stat -c %s ' . escapeshellarg($filename))); } else if (strpos($os, 'bsd') !== false) { - return 0 + exec('stat -f %z ' . escapeshellarg($filename)); + $result = trim(exec('stat -f %z ' . escapeshellarg($filename))); + } + + if (ctype_digit($result)) { + return 0 + $result; } } return null; From df29eec72b2b094acca97636c6b91ab2eafc245d Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 9 Feb 2014 16:16:01 +0100 Subject: [PATCH 06/17] Windows exec() implementation. --- lib/private/largefilehelper.php | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index 751e60de53..5f5e14aca3 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -92,17 +92,26 @@ class LargeFileHelper { public function getFilesizeViaExec($filename) { if (\OC_Helper::is_function_enabled('exec')) { $os = strtolower(php_uname('s')); + $arg = escapeshellarg($filename); $result = ''; if (strpos($os, 'linux') !== false) { - $result = trim(exec('stat -c %s ' . escapeshellarg($filename))); + $result = $this->exec("stat -c %s $arg"); } else if (strpos($os, 'bsd') !== false) { - $result = trim(exec('stat -f %z ' . escapeshellarg($filename))); - } - - if (ctype_digit($result)) { - return 0 + $result; + $result = $this->exec("stat -f %z $arg"); + } else if (strpos($os, 'win') !== false) { + $result = $this->exec("for %F in ($arg) do @echo %~zF"); + if (is_null($result)) { + // PowerShell + $result = $this->exec("(Get-Item $arg).length"); + } } + return $result; } return null; } + + protected function exec($cmd) { + $result = trim(exec($cmd)); + return ctype_digit($result) ? 0 + $result : null; + } } From fb7ec2bb22055886d55a779b9c19f48daee438c5 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 12 Feb 2014 13:58:07 +0100 Subject: [PATCH 07/17] Only call $this->filesize() for files. --- lib/private/files/storage/local.php | 2 +- lib/private/files/storage/mappedlocal.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index 883a69d768..f4bc508536 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -89,7 +89,7 @@ if (\OC_Util::runningOnWindows()) { public function stat($path) { $fullPath = $this->datadir . $path; $statResult = stat($fullPath); - if (PHP_INT_SIZE === 4) { + if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) { $filesize = $this->filesize($path); $statResult['size'] = $filesize; $statResult[7] = $filesize; diff --git a/lib/private/files/storage/mappedlocal.php b/lib/private/files/storage/mappedlocal.php index 78d2616ba9..f38b48db5d 100644 --- a/lib/private/files/storage/mappedlocal.php +++ b/lib/private/files/storage/mappedlocal.php @@ -111,7 +111,7 @@ class MappedLocal extends \OC\Files\Storage\Common { public function stat($path) { $fullPath = $this->buildPath($path); $statResult = stat($fullPath); - if (PHP_INT_SIZE === 4) { + if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) { $filesize = $this->filesize($path); $statResult['size'] = $filesize; $statResult[7] = $filesize; From 82e17155bf11161a4f86a4d23872ebee753d63e2 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sat, 15 Feb 2014 23:06:46 +0100 Subject: [PATCH 08/17] Rename: LargeFileHelper -> LargeFileHelperGetFilesize --- .../lib/{largefilehelper.php => largefilehelpergetfilesize.php} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/lib/{largefilehelper.php => largefilehelpergetfilesize.php} (95%) diff --git a/tests/lib/largefilehelper.php b/tests/lib/largefilehelpergetfilesize.php similarity index 95% rename from tests/lib/largefilehelper.php rename to tests/lib/largefilehelpergetfilesize.php index 43721a3ee8..001f636a52 100644 --- a/tests/lib/largefilehelper.php +++ b/tests/lib/largefilehelpergetfilesize.php @@ -12,7 +12,7 @@ namespace Test; * Tests whether LargeFileHelper is able to determine filesize at all. * Large files are not considered yet. */ -class LargeFileHelper extends \PHPUnit_Framework_TestCase { +class LargeFileHelperGetFilesize extends \PHPUnit_Framework_TestCase { protected $filename; protected $filesize; protected $helper; From 2c36a4b07a088bca4f20c2f6386765dfc8ad07b7 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sat, 15 Feb 2014 23:21:23 +0100 Subject: [PATCH 09/17] Add helper method for turning int|float into base-10 unsigned integer string. --- lib/private/largefilehelper.php | 24 +++++++++++++++++ tests/lib/largefilehelper.php | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/lib/largefilehelper.php diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index 5f5e14aca3..66626b4a7f 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -12,6 +12,30 @@ namespace OC; * Helper class for large files on 32-bit platforms. */ class LargeFileHelper { + /** + * @brief Formats a signed integer or float as an unsigned integer base-10 + * string. Passed strings will be checked for being base-10. + * + * @param int|float|string $number Number containing unsigned integer data + * + * @return string Unsigned integer base-10 string + */ + public function formatUnsignedInteger($number) { + if (is_float($number)) { + // Undo the effect of the php.ini setting 'precision'. + return number_format($number, 0, '', ''); + } else if (is_string($number) && ctype_digit($number)) { + return $number; + } else if (is_int($number)) { + // Interpret signed integer as unsigned integer. + return sprintf('%u', $number); + } else { + throw new \UnexpectedValueException( + 'Expected int, float or base-10 string' + ); + } + } + /** * @brief Tries to get the filesize of a file via various workarounds that * even work for large files on 32-bit platforms. diff --git a/tests/lib/largefilehelper.php b/tests/lib/largefilehelper.php new file mode 100644 index 0000000000..5db1f9c5a7 --- /dev/null +++ b/tests/lib/largefilehelper.php @@ -0,0 +1,46 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test; + +class LargeFileHelper extends \PHPUnit_Framework_TestCase { + protected $helper; + + public function setUp() { + parent::setUp(); + $this->helper = new \OC\LargeFileHelper; + } + + public function testFormatUnsignedIntegerFloat() { + $this->assertSame( + '9007199254740992', + $this->helper->formatUnsignedInteger((float) 9007199254740992) + ); + } + + public function testFormatUnsignedIntegerInt() { + $this->assertSame( + PHP_INT_SIZE === 4 ? '4294967295' : '18446744073709551615', + $this->helper->formatUnsignedInteger(-1) + ); + } + + public function testFormatUnsignedIntegerString() { + $this->assertSame( + '9007199254740993', + $this->helper->formatUnsignedInteger('9007199254740993') + ); + } + + /** + * @expectedException \UnexpectedValueException + */ + public function testFormatUnsignedIntegerStringException() { + $this->helper->formatUnsignedInteger('900ABCD254740993'); + } +} From a9b28323dd1116aef8c53d8e050380895fa1bb74 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sat, 15 Feb 2014 23:41:58 +0100 Subject: [PATCH 10/17] Add LargeFileHelper::__construct() verifying that our assumptions hold. --- lib/private/largefilehelper.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index 66626b4a7f..08869d7c82 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -12,6 +12,31 @@ namespace OC; * Helper class for large files on 32-bit platforms. */ class LargeFileHelper { + /** + * pow(2, 53) as a base-10 string. + * @var string + */ + const POW_2_53 = '9007199254740992'; + + /** + * pow(2, 53) - 1 as a base-10 string. + * @var string + */ + const POW_2_53_MINUS_1 = '9007199254740991'; + + /** + * @brief Constructor. Checks whether our assumptions hold on the platform + * we are on, throws an exception if they do not hold. + */ + public function __construct() { + $pow_2_53 = floatval(self::POW_2_53_MINUS_1) + 1.0; + if ($this->formatUnsignedInteger($pow_2_53) !== self::POW_2_53) { + throw new \RunTimeException( + 'This class assumes floats to be double precision or "better".' + ); + } + } + /** * @brief Formats a signed integer or float as an unsigned integer base-10 * string. Passed strings will be checked for being base-10. From a34aa1959aeeb9bc92c71c6f217db80a09cfbbd3 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 16 Feb 2014 00:50:03 +0100 Subject: [PATCH 11/17] Cast to number instead of integer in OC\Files\Cache\Cache --- lib/private/files/cache/cache.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/private/files/cache/cache.php b/lib/private/files/cache/cache.php index 3e4f6dfb13..59963f41e3 100644 --- a/lib/private/files/cache/cache.php +++ b/lib/private/files/cache/cache.php @@ -142,11 +142,11 @@ class Cache { } else { //fix types $data['fileid'] = (int)$data['fileid']; - $data['size'] = (int)$data['size']; + $data['size'] = 0 + $data['size']; $data['mtime'] = (int)$data['mtime']; $data['storage_mtime'] = (int)$data['storage_mtime']; $data['encrypted'] = (bool)$data['encrypted']; - $data['unencrypted_size'] = (int)$data['unencrypted_size']; + $data['unencrypted_size'] = 0 + $data['unencrypted_size']; $data['storage'] = $this->storageId; $data['mimetype'] = $this->getMimetype($data['mimetype']); $data['mimepart'] = $this->getMimetype($data['mimepart']); @@ -532,9 +532,9 @@ class Cache { $result = \OC_DB::executeAudited($sql, array($id, $this->getNumericStorageId())); if ($row = $result->fetchRow()) { list($sum, $min, $unencryptedSum) = array_values($row); - $sum = (int)$sum; - $min = (int)$min; - $unencryptedSum = (int)$unencryptedSum; + $sum = 0 + $sum; + $min = 0 + $min; + $unencryptedSum = 0 + $unencryptedSum; if ($min === -1) { $totalSize = $min; } else { From 0bae68017ee93ec087e04acce83f189f5c3eeea7 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 16 Feb 2014 01:03:39 +0100 Subject: [PATCH 12/17] Cast to number instead of integer in OC\Files\Cache\HomeCache --- lib/private/files/cache/homecache.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/private/files/cache/homecache.php b/lib/private/files/cache/homecache.php index 2326c46e8d..f61769f0b9 100644 --- a/lib/private/files/cache/homecache.php +++ b/lib/private/files/cache/homecache.php @@ -36,8 +36,10 @@ class HomeCache extends Cache { $result = \OC_DB::executeAudited($sql, array($id, $this->getNumericStorageId())); if ($row = $result->fetchRow()) { list($sum, $unencryptedSum) = array_values($row); - $totalSize = (int)$sum; - $unencryptedSize = (int)$unencryptedSum; + $totalSize = 0 + $sum; + $unencryptedSize = 0 + $unencryptedSum; + $entry['size'] += 0; + $entry['unencrypted_size'] += 0; if ($entry['size'] !== $totalSize) { $this->update($id, array('size' => $totalSize)); } From fb4556033a9d39698dbc15b59f3ba76ef6510e33 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 16 Feb 2014 14:42:59 +0100 Subject: [PATCH 13/17] Cast '{DAV:}getcontentlength' to number instead of int. --- lib/private/connector/sabre/file.php | 2 +- lib/private/connector/sabre/server.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/private/connector/sabre/file.php b/lib/private/connector/sabre/file.php index ab9d3e47d0..8a16ba55e7 100644 --- a/lib/private/connector/sabre/file.php +++ b/lib/private/connector/sabre/file.php @@ -156,7 +156,7 @@ class OC_Connector_Sabre_File extends OC_Connector_Sabre_Node implements Sabre_D /** * Returns the size of the node, in bytes * - * @return int + * @return int|float */ public function getSize() { return $this->info->getSize(); diff --git a/lib/private/connector/sabre/server.php b/lib/private/connector/sabre/server.php index 2660b043f4..cf28b11163 100644 --- a/lib/private/connector/sabre/server.php +++ b/lib/private/connector/sabre/server.php @@ -170,7 +170,7 @@ class OC_Connector_Sabre_Server extends Sabre_DAV_Server { if ($node instanceof Sabre_DAV_IFile) { $size = $node->getSize(); if (!is_null($size)) { - $newProperties[200][$prop] = (int)$node->getSize(); + $newProperties[200][$prop] = 0 + $size; } } break; From 0417e52134e87c379b3b4c22a53a0c06a711baef Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 18 Feb 2014 12:57:44 +0100 Subject: [PATCH 14/17] Increase file size limit from 2 GiB to 4 GiB when workarounds are unavailable. --- lib/private/files/storage/local.php | 5 +---- lib/private/files/storage/mappedlocal.php | 5 +---- lib/private/largefilehelper.php | 23 ++++++++++++++++++++++- tests/lib/largefilehelpergetfilesize.php | 7 +++++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/lib/private/files/storage/local.php b/lib/private/files/storage/local.php index f4bc508536..e33747bbd5 100644 --- a/lib/private/files/storage/local.php +++ b/lib/private/files/storage/local.php @@ -112,10 +112,7 @@ if (\OC_Util::runningOnWindows()) { $fullPath = $this->datadir . $path; if (PHP_INT_SIZE === 4) { $helper = new \OC\LargeFileHelper; - $filesize = $helper->getFilesize($fullPath); - if (!is_null($filesize)) { - return $filesize; - } + return $helper->getFilesize($fullPath); } return filesize($fullPath); } diff --git a/lib/private/files/storage/mappedlocal.php b/lib/private/files/storage/mappedlocal.php index f38b48db5d..ea4deaa66e 100644 --- a/lib/private/files/storage/mappedlocal.php +++ b/lib/private/files/storage/mappedlocal.php @@ -134,10 +134,7 @@ class MappedLocal extends \OC\Files\Storage\Common { $fullPath = $this->buildPath($path); if (PHP_INT_SIZE === 4) { $helper = new \OC\LargeFileHelper; - $filesize = $helper->getFilesize($fullPath); - if (!is_null($filesize)) { - return $filesize; - } + return $helper->getFilesize($fullPath); } return filesize($fullPath); } diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index 08869d7c82..3d15e78604 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -83,7 +83,7 @@ class LargeFileHelper { if (!is_null($filesize)) { return $filesize; } - return null; + return $this->getFilesizeNative($filename); } /** @@ -159,6 +159,27 @@ class LargeFileHelper { return null; } + /** + * @brief Gets the filesize via a filesize() call and converts negative + * signed int to positive float. As the result of filesize() will + * wrap around after a filesize of 2^32 bytes = 4 GiB, this should + * only be used as a last resort. + * + * @param string $filename Path to the file. + * + * @return int|float Number of bytes as number (float or int). + */ + public function getFilesizeNative($filename) { + $result = filesize($filename); + if ($result < 0) { + // For filesizes between 2 GiB and 4 GiB, filesize() will return a + // negative int, as the PHP data type int is signed. Interpret the + // returned int as an unsigned integer and put it into a float. + return (float) sprintf('%u', $result); + } + return $result; + } + protected function exec($cmd) { $result = trim(exec($cmd)); return ctype_digit($result) ? 0 + $result : null; diff --git a/tests/lib/largefilehelpergetfilesize.php b/tests/lib/largefilehelpergetfilesize.php index 001f636a52..699dd6891a 100644 --- a/tests/lib/largefilehelpergetfilesize.php +++ b/tests/lib/largefilehelpergetfilesize.php @@ -59,4 +59,11 @@ class LargeFileHelperGetFilesize extends \PHPUnit_Framework_TestCase { $this->helper->getFilesizeViaExec($this->filename) ); } + + public function testGetFilesizeNative() { + $this->assertSame( + $this->filesize, + $this->helper->getFilesizeNative($this->filename) + ); + } } From ea246d058ed86acc6634d34888d61f2a953e6ce1 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 16 Mar 2014 19:54:05 +0100 Subject: [PATCH 15/17] Use "file size" instead of "filesize", then also apply camel case. --- lib/private/largefilehelper.php | 48 ++++++++++++------------ tests/lib/largefilehelpergetfilesize.php | 32 ++++++++-------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index 3d15e78604..4bab197b2e 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -62,7 +62,7 @@ class LargeFileHelper { } /** - * @brief Tries to get the filesize of a file via various workarounds that + * @brief Tries to get the size of a file via various workarounds that * even work for large files on 32-bit platforms. * * @param string $filename Path to the file. @@ -70,31 +70,31 @@ class LargeFileHelper { * @return null|int|float Number of bytes as number (float or int) or * null on failure. */ - public function getFilesize($filename) { - $filesize = $this->getFilesizeViaCurl($filename); - if (!is_null($filesize)) { - return $filesize; + public function getFileSize($filename) { + $fileSize = $this->getFileSizeViaCurl($filename); + if (!is_null($fileSize)) { + return $fileSize; } - $filesize = $this->getFilesizeViaCOM($filename); - if (!is_null($filesize)) { - return $filesize; + $fileSize = $this->getFileSizeViaCOM($filename); + if (!is_null($fileSize)) { + return $fileSize; } - $filesize = $this->getFilesizeViaExec($filename); - if (!is_null($filesize)) { - return $filesize; + $fileSize = $this->getFileSizeViaExec($filename); + if (!is_null($fileSize)) { + return $fileSize; } - return $this->getFilesizeNative($filename); + return $this->getFileSizeNative($filename); } /** - * @brief Tries to get the filesize of a file via a CURL HEAD request. + * @brief Tries to get the size of a file via a CURL HEAD request. * * @param string $filename Path to the file. * * @return null|int|float Number of bytes as number (float or int) or * null on failure. */ - public function getFilesizeViaCurl($filename) { + public function getFileSizeViaCurl($filename) { if (function_exists('curl_init')) { $ch = curl_init("file://$filename"); curl_setopt($ch, CURLOPT_NOBODY, true); @@ -114,14 +114,14 @@ class LargeFileHelper { } /** - * @brief Tries to get the filesize of a file via the Windows DOM extension. + * @brief Tries to get the size of a file via the Windows DOM extension. * * @param string $filename Path to the file. * * @return null|int|float Number of bytes as number (float or int) or * null on failure. */ - public function getFilesizeViaCOM($filename) { + public function getFileSizeViaCOM($filename) { if (class_exists('COM')) { $fsobj = new \COM("Scripting.FileSystemObject"); $file = $fsobj->GetFile($filename); @@ -131,14 +131,14 @@ class LargeFileHelper { } /** - * @brief Tries to get the filesize of a file via an exec() call. + * @brief Tries to get the size of a file via an exec() call. * * @param string $filename Path to the file. * * @return null|int|float Number of bytes as number (float or int) or * null on failure. */ - public function getFilesizeViaExec($filename) { + public function getFileSizeViaExec($filename) { if (\OC_Helper::is_function_enabled('exec')) { $os = strtolower(php_uname('s')); $arg = escapeshellarg($filename); @@ -160,19 +160,19 @@ class LargeFileHelper { } /** - * @brief Gets the filesize via a filesize() call and converts negative - * signed int to positive float. As the result of filesize() will - * wrap around after a filesize of 2^32 bytes = 4 GiB, this should - * only be used as a last resort. + * @brief Gets the size of a file via a filesize() call and converts + * negative signed int to positive float. As the result of filesize() + * will wrap around after a file size of 2^32 bytes = 4 GiB, this + * should only be used as a last resort. * * @param string $filename Path to the file. * * @return int|float Number of bytes as number (float or int). */ - public function getFilesizeNative($filename) { + public function getFileSizeNative($filename) { $result = filesize($filename); if ($result < 0) { - // For filesizes between 2 GiB and 4 GiB, filesize() will return a + // For file sizes between 2 GiB and 4 GiB, filesize() will return a // negative int, as the PHP data type int is signed. Interpret the // returned int as an unsigned integer and put it into a float. return (float) sprintf('%u', $result); diff --git a/tests/lib/largefilehelpergetfilesize.php b/tests/lib/largefilehelpergetfilesize.php index 699dd6891a..cc25d54a36 100644 --- a/tests/lib/largefilehelpergetfilesize.php +++ b/tests/lib/largefilehelpergetfilesize.php @@ -9,61 +9,61 @@ namespace Test; /** -* Tests whether LargeFileHelper is able to determine filesize at all. +* Tests whether LargeFileHelper is able to determine file size at all. * Large files are not considered yet. */ -class LargeFileHelperGetFilesize extends \PHPUnit_Framework_TestCase { +class LargeFileHelperGetFileSize extends \PHPUnit_Framework_TestCase { protected $filename; - protected $filesize; + protected $fileSize; protected $helper; public function setUp() { parent::setUp(); $this->filename = __DIR__ . '/../data/data.tar.gz'; - $this->filesize = 4195; + $this->fileSize = 4195; $this->helper = new \OC\LargeFileHelper; } - public function testGetFilesizeViaCurl() { + public function testGetFileSizeViaCurl() { if (!extension_loaded('curl')) { $this->markTestSkipped( 'The PHP curl extension is required for this test.' ); } $this->assertSame( - $this->filesize, - $this->helper->getFilesizeViaCurl($this->filename) + $this->fileSize, + $this->helper->getFileSizeViaCurl($this->filename) ); } - public function testGetFilesizeViaCOM() { + public function testGetFileSizeViaCOM() { if (!extension_loaded('COM')) { $this->markTestSkipped( 'The PHP Windows COM extension is required for this test.' ); } $this->assertSame( - $this->filesize, - $this->helper->getFilesizeViaDOM($this->filename) + $this->fileSize, + $this->helper->getFileSizeViaDOM($this->filename) ); } - public function testGetFilesizeViaExec() { + public function testGetFileSizeViaExec() { if (!\OC_Helper::is_function_enabled('exec')) { $this->markTestSkipped( 'The exec() function needs to be enabled for this test.' ); } $this->assertSame( - $this->filesize, - $this->helper->getFilesizeViaExec($this->filename) + $this->fileSize, + $this->helper->getFileSizeViaExec($this->filename) ); } - public function testGetFilesizeNative() { + public function testGetFileSizeNative() { $this->assertSame( - $this->filesize, - $this->helper->getFilesizeNative($this->filename) + $this->fileSize, + $this->helper->getFileSizeNative($this->filename) ); } } From 2929d19c7f0b87d8207cbeff35607b8c27674ebf Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 16 Mar 2014 20:05:06 +0100 Subject: [PATCH 16/17] Document exceptions thrown by \OC\LargeFileHelper. --- lib/private/largefilehelper.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/private/largefilehelper.php b/lib/private/largefilehelper.php index 4bab197b2e..293e09fe2c 100644 --- a/lib/private/largefilehelper.php +++ b/lib/private/largefilehelper.php @@ -25,8 +25,10 @@ class LargeFileHelper { const POW_2_53_MINUS_1 = '9007199254740991'; /** - * @brief Constructor. Checks whether our assumptions hold on the platform - * we are on, throws an exception if they do not hold. + * @brief Checks whether our assumptions hold on the PHP platform we are on. + * + * @throws \RunTimeException if our assumptions do not hold on the current + * PHP platform. */ public function __construct() { $pow_2_53 = floatval(self::POW_2_53_MINUS_1) + 1.0; @@ -43,6 +45,9 @@ class LargeFileHelper { * * @param int|float|string $number Number containing unsigned integer data * + * @throws \UnexpectedValueException if $number is not a float, not an int + * and not a base-10 string. + * * @return string Unsigned integer base-10 string */ public function formatUnsignedInteger($number) { From 129d8099b948c0b5c7456af42cf9a618f5f6920d Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 16 Mar 2014 21:21:17 +0100 Subject: [PATCH 17/17] Typo: getFileSizeViaDOM -> getFileSizeViaCOM --- tests/lib/largefilehelpergetfilesize.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/largefilehelpergetfilesize.php b/tests/lib/largefilehelpergetfilesize.php index cc25d54a36..86ce6d295c 100644 --- a/tests/lib/largefilehelpergetfilesize.php +++ b/tests/lib/largefilehelpergetfilesize.php @@ -44,7 +44,7 @@ class LargeFileHelperGetFileSize extends \PHPUnit_Framework_TestCase { } $this->assertSame( $this->fileSize, - $this->helper->getFileSizeViaDOM($this->filename) + $this->helper->getFileSizeViaCOM($this->filename) ); }