Merge pull request #5365 from owncloud/filesize-improvements-32bit

Add LargeFileHelper / Add CURL filesize workaround / Fix some 32-bit filesize issues

lib/private/connector/sabre/file.php

@@ -156,7 +156,7 @@ class OC_Connector_Sabre_File extends OC_Connector_Sabre_Node implements Sabre_D
 	/**
 	 * Returns the size of the node, in bytes
 	 *
-	 * @return int
+	 * @return int|float
 	 */
 	public function getSize() {
 		return $this->info->getSize();

lib/private/connector/sabre/server.php

@@ -170,7 +170,7 @@ class OC_Connector_Sabre_Server extends Sabre_DAV_Server {
 						if ($node instanceof Sabre_DAV_IFile) {
 							$size = $node->getSize();
 							if (!is_null($size)) {
-								$newProperties[200][$prop] = (int)$node->getSize();
+								$newProperties[200][$prop] = 0 + $size;
 							}
 						}
 						break;

lib/private/files/cache/cache.php

@@ -142,11 +142,11 @@ class Cache {
 		} else {
 			//fix types
 			$data['fileid'] = (int)$data['fileid'];
-			$data['size'] = (int)$data['size'];
+			$data['size'] = 0 + $data['size'];
 			$data['mtime'] = (int)$data['mtime'];
 			$data['storage_mtime'] = (int)$data['storage_mtime'];
 			$data['encrypted'] = (bool)$data['encrypted'];
-            $data['unencrypted_size'] = (int)$data['unencrypted_size'];
+            $data['unencrypted_size'] = 0 + $data['unencrypted_size'];
 			$data['storage'] = $this->storageId;
 			$data['mimetype'] = $this->getMimetype($data['mimetype']);
 			$data['mimepart'] = $this->getMimetype($data['mimepart']);
@@ -532,9 +532,9 @@ class Cache {
 			$result = \OC_DB::executeAudited($sql, array($id, $this->getNumericStorageId()));
 			if ($row = $result->fetchRow()) {
 				list($sum, $min, $unencryptedSum) = array_values($row);
-				$sum = (int)$sum;
+				$sum = 0 + $sum;
-				$min = (int)$min;
+				$min = 0 + $min;
-				$unencryptedSum = (int)$unencryptedSum;
+				$unencryptedSum = 0 + $unencryptedSum;
 				if ($min === -1) {
 					$totalSize = $min;
 				} else {

lib/private/files/cache/homecache.php

@@ -36,8 +36,10 @@ class HomeCache extends Cache {
 			$result = \OC_DB::executeAudited($sql, array($id, $this->getNumericStorageId()));
 			if ($row = $result->fetchRow()) {
 				list($sum, $unencryptedSum) = array_values($row);
-				$totalSize = (int)$sum;
+				$totalSize = 0 + $sum;
-				$unencryptedSize = (int)$unencryptedSum;
+				$unencryptedSize = 0 + $unencryptedSum;
+				$entry['size'] += 0;
+				$entry['unencrypted_size'] += 0;
 				if ($entry['size'] !== $totalSize) {
 					$this->update($id, array('size' => $totalSize));
 				}

lib/private/files/storage/local.php

@@ -89,11 +89,10 @@ if (\OC_Util::runningOnWindows()) {
 		public function stat($path) {
 			$fullPath = $this->datadir . $path;
 			$statResult = stat($fullPath);
-
+			if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) {
-			if ($statResult['size'] < 0) {
+				$filesize = $this->filesize($path);
-				$size = self::getFileSizeFromOS($fullPath);
+				$statResult['size'] = $filesize;
-				$statResult['size'] = $size;
+				$statResult[7] = $filesize;
-				$statResult[7] = $size;
 			}
 			return $statResult;
 		}
@@ -109,15 +108,13 @@ if (\OC_Util::runningOnWindows()) {
 		public function filesize($path) {
 			if ($this->is_dir($path)) {
 				return 0;
-			} else {
-				$fullPath = $this->datadir . $path;
-				$fileSize = filesize($fullPath);
-				if ($fileSize < 0) {
-					return self::getFileSizeFromOS($fullPath);
-				}
-
-				return $fileSize;
 			}
+			$fullPath = $this->datadir . $path;
+			if (PHP_INT_SIZE === 4) {
+				$helper = new \OC\LargeFileHelper;
+				return $helper->getFilesize($fullPath);
+			}
+			return filesize($fullPath);
 		}
 
 		public function isReadable($path) {
@@ -220,35 +217,6 @@ if (\OC_Util::runningOnWindows()) {
 			return $return;
 		}
 
-		/**
-		 * @param string $fullPath
-		 */
-		private static function getFileSizeFromOS($fullPath) {
-			$name = strtolower(php_uname('s'));
-			// Windows OS: we use COM to access the filesystem
-			if (strpos($name, 'win') !== false) {
-				if (class_exists('COM')) {
-					$fsobj = new \COM("Scripting.FileSystemObject");
-					$f = $fsobj->GetFile($fullPath);
-					return $f->Size;
-				}
-			} else if (strpos($name, 'bsd') !== false) {
-				if (\OC_Helper::is_function_enabled('exec')) {
-					return (float)exec('stat -f %z ' . escapeshellarg($fullPath));
-				}
-			} else if (strpos($name, 'linux') !== false) {
-				if (\OC_Helper::is_function_enabled('exec')) {
-					return (float)exec('stat -c %s ' . escapeshellarg($fullPath));
-				}
-			} else {
-				\OC_Log::write('core',
-					'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name,
-					\OC_Log::ERROR);
-			}
-
-			return 0;
-		}
-
 		public function hash($type, $path, $raw = false) {
 			return hash_file($type, $this->datadir . $path, $raw);
 		}

lib/private/files/storage/mappedlocal.php

@@ -111,11 +111,10 @@ class MappedLocal extends \OC\Files\Storage\Common {
 	public function stat($path) {
 		$fullPath = $this->buildPath($path);
 		$statResult = stat($fullPath);
-
+		if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) {
-		if ($statResult['size'] < 0) {
+			$filesize = $this->filesize($path);
-			$size = self::getFileSizeFromOS($fullPath);
+			$statResult['size'] = $filesize;
-			$statResult['size'] = $size;
+			$statResult[7] = $filesize;
-			$statResult[7] = $size;
 		}
 		return $statResult;
 	}
@@ -131,15 +130,13 @@ class MappedLocal extends \OC\Files\Storage\Common {
 	public function filesize($path) {
 		if ($this->is_dir($path)) {
 			return 0;
-		} else {
-			$fullPath = $this->buildPath($path);
-			$fileSize = filesize($fullPath);
-			if ($fileSize < 0) {
-				return self::getFileSizeFromOS($fullPath);
-			}
-
-			return $fileSize;
 		}
+		$fullPath = $this->buildPath($path);
+		if (PHP_INT_SIZE === 4) {
+			$helper = new \OC\LargeFileHelper;
+			return $helper->getFilesize($fullPath);
+		}
+		return filesize($fullPath);
 	}
 
 	public function isReadable($path) {
@@ -294,35 +291,6 @@ class MappedLocal extends \OC\Files\Storage\Common {
 		return $return;
 	}
 
-	/**
-	 * @param string $fullPath
-	 */
-	private static function getFileSizeFromOS($fullPath) {
-		$name = strtolower(php_uname('s'));
-		// Windows OS: we use COM to access the filesystem
-		if (strpos($name, 'win') !== false) {
-			if (class_exists('COM')) {
-				$fsobj = new \COM("Scripting.FileSystemObject");
-				$f = $fsobj->GetFile($fullPath);
-				return $f->Size;
-			}
-		} else if (strpos($name, 'bsd') !== false) {
-			if (\OC_Helper::is_function_enabled('exec')) {
-				return (float)exec('stat -f %z ' . escapeshellarg($fullPath));
-			}
-		} else if (strpos($name, 'linux') !== false) {
-			if (\OC_Helper::is_function_enabled('exec')) {
-				return (float)exec('stat -c %s ' . escapeshellarg($fullPath));
-			}
-		} else {
-			\OC_Log::write('core',
-				'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name,
-				\OC_Log::ERROR);
-		}
-
-		return 0;
-	}
-
 	public function hash($type, $path, $raw = false) {
 		return hash_file($type, $this->buildPath($path), $raw);
 	}

lib/private/largefilehelper.php

@@ -0,0 +1,192 @@
+<?php
+/**
+ * Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC;
+
+/**
+ * Helper class for large files on 32-bit platforms.
+ */
+class LargeFileHelper {
+	/**
+	* pow(2, 53) as a base-10 string.
+	* @var string
+	*/
+	const POW_2_53 = '9007199254740992';
+
+	/**
+	* pow(2, 53) - 1 as a base-10 string.
+	* @var string
+	*/
+	const POW_2_53_MINUS_1 = '9007199254740991';
+
+	/**
+	* @brief Checks whether our assumptions hold on the PHP platform we are on.
+	*
+	* @throws \RunTimeException if our assumptions do not hold on the current
+	*                           PHP platform.
+	*/
+	public function __construct() {
+		$pow_2_53 = floatval(self::POW_2_53_MINUS_1) + 1.0;
+		if ($this->formatUnsignedInteger($pow_2_53) !== self::POW_2_53) {
+			throw new \RunTimeException(
+				'This class assumes floats to be double precision or "better".'
+			);
+		}
+	}
+
+	/**
+	* @brief Formats a signed integer or float as an unsigned integer base-10
+	*        string. Passed strings will be checked for being base-10.
+	*
+	* @param int|float|string $number Number containing unsigned integer data
+	*
+	* @throws \UnexpectedValueException if $number is not a float, not an int
+	*                                   and not a base-10 string.
+	*
+	* @return string Unsigned integer base-10 string
+	*/
+	public function formatUnsignedInteger($number) {
+		if (is_float($number)) {
+			// Undo the effect of the php.ini setting 'precision'.
+			return number_format($number, 0, '', '');
+		} else if (is_string($number) && ctype_digit($number)) {
+			return $number;
+		} else if (is_int($number)) {
+			// Interpret signed integer as unsigned integer.
+			return sprintf('%u', $number);
+		} else {
+			throw new \UnexpectedValueException(
+				'Expected int, float or base-10 string'
+			);
+		}
+	}
+
+	/**
+	* @brief Tries to get the size of a file via various workarounds that
+	*        even work for large files on 32-bit platforms.
+	*
+	* @param string $filename Path to the file.
+	*
+	* @return null|int|float Number of bytes as number (float or int) or
+	*                        null on failure.
+	*/
+	public function getFileSize($filename) {
+		$fileSize = $this->getFileSizeViaCurl($filename);
+		if (!is_null($fileSize)) {
+			return $fileSize;
+		}
+		$fileSize = $this->getFileSizeViaCOM($filename);
+		if (!is_null($fileSize)) {
+			return $fileSize;
+		}
+		$fileSize = $this->getFileSizeViaExec($filename);
+		if (!is_null($fileSize)) {
+			return $fileSize;
+		}
+		return $this->getFileSizeNative($filename);
+	}
+
+	/**
+	* @brief Tries to get the size of a file via a CURL HEAD request.
+	*
+	* @param string $filename Path to the file.
+	*
+	* @return null|int|float Number of bytes as number (float or int) or
+	*                        null on failure.
+	*/
+	public function getFileSizeViaCurl($filename) {
+		if (function_exists('curl_init')) {
+			$ch = curl_init("file://$filename");
+			curl_setopt($ch, CURLOPT_NOBODY, true);
+			curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+			curl_setopt($ch, CURLOPT_HEADER, true);
+			$data = curl_exec($ch);
+			curl_close($ch);
+			if ($data !== false) {
+				$matches = array();
+				preg_match('/Content-Length: (\d+)/', $data, $matches);
+				if (isset($matches[1])) {
+					return 0 + $matches[1];
+				}
+			}
+		}
+		return null;
+	}
+
+	/**
+	* @brief Tries to get the size of a file via the Windows DOM extension.
+	*
+	* @param string $filename Path to the file.
+	*
+	* @return null|int|float Number of bytes as number (float or int) or
+	*                        null on failure.
+	*/
+	public function getFileSizeViaCOM($filename) {
+		if (class_exists('COM')) {
+			$fsobj = new \COM("Scripting.FileSystemObject");
+			$file = $fsobj->GetFile($filename);
+			return 0 + $file->Size;
+		}
+		return null;
+	}
+
+	/**
+	* @brief Tries to get the size of a file via an exec() call.
+	*
+	* @param string $filename Path to the file.
+	*
+	* @return null|int|float Number of bytes as number (float or int) or
+	*                        null on failure.
+	*/
+	public function getFileSizeViaExec($filename) {
+		if (\OC_Helper::is_function_enabled('exec')) {
+			$os = strtolower(php_uname('s'));
+			$arg = escapeshellarg($filename);
+			$result = '';
+			if (strpos($os, 'linux') !== false) {
+				$result = $this->exec("stat -c %s $arg");
+			} else if (strpos($os, 'bsd') !== false) {
+				$result = $this->exec("stat -f %z $arg");
+			} else if (strpos($os, 'win') !== false) {
+				$result = $this->exec("for %F in ($arg) do @echo %~zF");
+				if (is_null($result)) {
+					// PowerShell
+					$result = $this->exec("(Get-Item $arg).length");
+				}
+			}
+			return $result;
+		}
+		return null;
+	}
+
+	/**
+	* @brief Gets the size of a file via a filesize() call and converts
+	*        negative signed int to positive float. As the result of filesize()
+	*        will wrap around after a file size of 2^32 bytes = 4 GiB, this
+	*        should only be used as a last resort.
+	*
+	* @param string $filename Path to the file.
+	*
+	* @return int|float Number of bytes as number (float or int).
+	*/
+	public function getFileSizeNative($filename) {
+		$result = filesize($filename);
+		if ($result < 0) {
+			// For file sizes between 2 GiB and 4 GiB, filesize() will return a
+			// negative int, as the PHP data type int is signed. Interpret the
+			// returned int as an unsigned integer and put it into a float.
+			return (float) sprintf('%u', $result);
+		}
+		return $result;
+	}
+
+	protected function exec($cmd) {
+		$result = trim(exec($cmd));
+		return ctype_digit($result) ? 0 + $result : null;
+	}
+}

tests/lib/largefilehelper.php

@@ -0,0 +1,46 @@
+<?php
+/**
+ * Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace Test;
+
+class LargeFileHelper extends \PHPUnit_Framework_TestCase {
+	protected $helper;
+
+	public function setUp() {
+		parent::setUp();
+		$this->helper = new \OC\LargeFileHelper;
+	}
+
+	public function testFormatUnsignedIntegerFloat() {
+		$this->assertSame(
+			'9007199254740992',
+			$this->helper->formatUnsignedInteger((float) 9007199254740992)
+		);
+	}
+
+	public function testFormatUnsignedIntegerInt() {
+		$this->assertSame(
+			PHP_INT_SIZE === 4 ? '4294967295' : '18446744073709551615',
+			$this->helper->formatUnsignedInteger(-1)
+		);
+	}
+
+	public function testFormatUnsignedIntegerString() {
+		$this->assertSame(
+			'9007199254740993',
+			$this->helper->formatUnsignedInteger('9007199254740993')
+		);
+	}
+
+	/**
+	* @expectedException \UnexpectedValueException
+	*/
+	public function testFormatUnsignedIntegerStringException() {
+		$this->helper->formatUnsignedInteger('900ABCD254740993');
+	}
+}

tests/lib/largefilehelpergetfilesize.php

@@ -0,0 +1,69 @@
+<?php
+/**
+ * Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace Test;
+
+/**
+* Tests whether LargeFileHelper is able to determine file size at all.
+* Large files are not considered yet.
+*/
+class LargeFileHelperGetFileSize extends \PHPUnit_Framework_TestCase {
+	protected $filename;
+	protected $fileSize;
+	protected $helper;
+
+	public function setUp() {
+		parent::setUp();
+		$this->filename = __DIR__ . '/../data/data.tar.gz';
+		$this->fileSize = 4195;
+		$this->helper = new \OC\LargeFileHelper;
+	}
+
+	public function testGetFileSizeViaCurl() {
+		if (!extension_loaded('curl')) {
+			$this->markTestSkipped(
+				'The PHP curl extension is required for this test.'
+			);
+		}
+		$this->assertSame(
+			$this->fileSize,
+			$this->helper->getFileSizeViaCurl($this->filename)
+		);
+	}
+
+	public function testGetFileSizeViaCOM() {
+		if (!extension_loaded('COM')) {
+			$this->markTestSkipped(
+				'The PHP Windows COM extension is required for this test.'
+			);
+		}
+		$this->assertSame(
+			$this->fileSize,
+			$this->helper->getFileSizeViaCOM($this->filename)
+		);
+	}
+
+	public function testGetFileSizeViaExec() {
+		if (!\OC_Helper::is_function_enabled('exec')) {
+			$this->markTestSkipped(
+				'The exec() function needs to be enabled for this test.'
+			);
+		}
+		$this->assertSame(
+			$this->fileSize,
+			$this->helper->getFileSizeViaExec($this->filename)
+		);
+	}
+
+	public function testGetFileSizeNative() {
+		$this->assertSame(
+			$this->fileSize,
+			$this->helper->getFileSizeNative($this->filename)
+		);
+	}
+}