mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #5365 from owncloud/filesize-improvements-32bit 

Add LargeFileHelper / Add CURL filesize workaround / Fix some 32-bit filesize issues
This commit is contained in:
Thomas Müller 2014-05-30 16:13:48 +02:00
parent 2ba5701b1a 129d8099b9
commit 31ad1cbdd8
9 changed files with 338 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/private/connector/sabre/file.php
View File

@ -156,7 +156,7 @@ class OC_Connector_Sabre_File extends OC_Connector_Sabre_Node implements Sabre_D
/**
* Returns the size of the node, in bytes
*
* @return int
* @return int|float
*/
public function getSize() {
return $this->info->getSize();

2
lib/private/connector/sabre/server.php
View File

@ -170,7 +170,7 @@ class OC_Connector_Sabre_Server extends Sabre_DAV_Server {
if ($node instanceof Sabre_DAV_IFile) {
$size = $node->getSize();
if (!is_null($size)) {
$newProperties[200][$prop] = (int)$node->getSize();
$newProperties[200][$prop] = 0 + $size;
}
}
break;

10
lib/private/files/cache/cache.php vendored
View File

@ -142,11 +142,11 @@ class Cache {
} else {
//fix types
$data['fileid'] = (int)$data['fileid'];
$data['size'] = (int)$data['size'];
$data['size'] = 0 + $data['size'];
$data['mtime'] = (int)$data['mtime'];
$data['storage_mtime'] = (int)$data['storage_mtime'];
$data['encrypted'] = (bool)$data['encrypted'];
$data['unencrypted_size'] = (int)$data['unencrypted_size'];
$data['unencrypted_size'] = 0 + $data['unencrypted_size'];
$data['storage'] = $this->storageId;
$data['mimetype'] = $this->getMimetype($data['mimetype']);
$data['mimepart'] = $this->getMimetype($data['mimepart']);
@ -532,9 +532,9 @@ class Cache {
$result = \OC_DB::executeAudited($sql, array($id, $this->getNumericStorageId()));
if ($row = $result->fetchRow()) {
list($sum, $min, $unencryptedSum) = array_values($row);
$sum = (int)$sum;
$min = (int)$min;
$unencryptedSum = (int)$unencryptedSum;
$sum = 0 + $sum;
$min = 0 + $min;
$unencryptedSum = 0 + $unencryptedSum;
if ($min === -1) {
$totalSize = $min;
} else {

6
lib/private/files/cache/homecache.php vendored
View File

@ -36,8 +36,10 @@ class HomeCache extends Cache {
$result = \OC_DB::executeAudited($sql, array($id, $this->getNumericStorageId()));
if ($row = $result->fetchRow()) {
list($sum, $unencryptedSum) = array_values($row);
$totalSize = (int)$sum;
$unencryptedSize = (int)$unencryptedSum;
$totalSize = 0 + $sum;
$unencryptedSize = 0 + $unencryptedSum;
$entry['size'] += 0;
$entry['unencrypted_size'] += 0;
if ($entry['size'] !== $totalSize) {
$this->update($id, array('size' => $totalSize));
}

50
lib/private/files/storage/local.php
View File

@ -89,11 +89,10 @@ if (\OC_Util::runningOnWindows()) {
public function stat($path) {
$fullPath = $this->datadir . $path;
$statResult = stat($fullPath);
if ($statResult['size'] < 0) {
$size = self::getFileSizeFromOS($fullPath);
$statResult['size'] = $size;
$statResult[7] = $size;
if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) {
$filesize = $this->filesize($path);
$statResult['size'] = $filesize;
$statResult[7] = $filesize;
}
return $statResult;
}
@ -109,15 +108,13 @@ if (\OC_Util::runningOnWindows()) {
public function filesize($path) {
if ($this->is_dir($path)) {
return 0;
} else {
}
$fullPath = $this->datadir . $path;
$fileSize = filesize($fullPath);
if ($fileSize < 0) {
return self::getFileSizeFromOS($fullPath);
}
return $fileSize;
if (PHP_INT_SIZE === 4) {
$helper = new \OC\LargeFileHelper;
return $helper->getFilesize($fullPath);
}
return filesize($fullPath);
}
public function isReadable($path) {
@ -220,35 +217,6 @@ if (\OC_Util::runningOnWindows()) {
return $return;
}
/**
* @param string $fullPath
*/
private static function getFileSizeFromOS($fullPath) {
$name = strtolower(php_uname('s'));
// Windows OS: we use COM to access the filesystem
if (strpos($name, 'win') !== false) {
if (class_exists('COM')) {
$fsobj = new \COM("Scripting.FileSystemObject");
$f = $fsobj->GetFile($fullPath);
return $f->Size;
}
} else if (strpos($name, 'bsd') !== false) {
if (\OC_Helper::is_function_enabled('exec')) {
return (float)exec('stat -f %z ' . escapeshellarg($fullPath));
}
} else if (strpos($name, 'linux') !== false) {
if (\OC_Helper::is_function_enabled('exec')) {
return (float)exec('stat -c %s ' . escapeshellarg($fullPath));
}
} else {
\OC_Log::write('core',
'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name,
\OC_Log::ERROR);
}
return 0;
}
public function hash($type, $path, $raw = false) {
return hash_file($type, $this->datadir . $path, $raw);
}

50
lib/private/files/storage/mappedlocal.php
View File

@ -111,11 +111,10 @@ class MappedLocal extends \OC\Files\Storage\Common {
public function stat($path) {
$fullPath = $this->buildPath($path);
$statResult = stat($fullPath);
if ($statResult['size'] < 0) {
$size = self::getFileSizeFromOS($fullPath);
$statResult['size'] = $size;
$statResult[7] = $size;
if (PHP_INT_SIZE === 4 && !$this->is_dir($path)) {
$filesize = $this->filesize($path);
$statResult['size'] = $filesize;
$statResult[7] = $filesize;
}
return $statResult;
}
@ -131,15 +130,13 @@ class MappedLocal extends \OC\Files\Storage\Common {
public function filesize($path) {
if ($this->is_dir($path)) {
return 0;
} else {
}
$fullPath = $this->buildPath($path);
$fileSize = filesize($fullPath);
if ($fileSize < 0) {
return self::getFileSizeFromOS($fullPath);
}
return $fileSize;
if (PHP_INT_SIZE === 4) {
$helper = new \OC\LargeFileHelper;
return $helper->getFilesize($fullPath);
}
return filesize($fullPath);
}
public function isReadable($path) {
@ -294,35 +291,6 @@ class MappedLocal extends \OC\Files\Storage\Common {
return $return;
}
/**
* @param string $fullPath
*/
private static function getFileSizeFromOS($fullPath) {
$name = strtolower(php_uname('s'));
// Windows OS: we use COM to access the filesystem
if (strpos($name, 'win') !== false) {
if (class_exists('COM')) {
$fsobj = new \COM("Scripting.FileSystemObject");
$f = $fsobj->GetFile($fullPath);
return $f->Size;
}
} else if (strpos($name, 'bsd') !== false) {
if (\OC_Helper::is_function_enabled('exec')) {
return (float)exec('stat -f %z ' . escapeshellarg($fullPath));
}
} else if (strpos($name, 'linux') !== false) {
if (\OC_Helper::is_function_enabled('exec')) {
return (float)exec('stat -c %s ' . escapeshellarg($fullPath));
}
} else {
\OC_Log::write('core',
'Unable to determine file size of "' . $fullPath . '". Unknown OS: ' . $name,
\OC_Log::ERROR);
}
return 0;
}
public function hash($type, $path, $raw = false) {
return hash_file($type, $this->buildPath($path), $raw);
}

192
lib/private/largefilehelper.php Normal file
View File

@ -0,0 +1,192 @@
<?php
/**
* Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC;
/**
* Helper class for large files on 32-bit platforms.
*/
class LargeFileHelper {
/**
* pow(2, 53) as a base-10 string.
* @var string
*/
const POW_2_53 = '9007199254740992';
/**
* pow(2, 53) - 1 as a base-10 string.
* @var string
*/
const POW_2_53_MINUS_1 = '9007199254740991';
/**
* @brief Checks whether our assumptions hold on the PHP platform we are on.
*
* @throws \RunTimeException if our assumptions do not hold on the current
* PHP platform.
*/
public function __construct() {
$pow_2_53 = floatval(self::POW_2_53_MINUS_1) + 1.0;
if ($this->formatUnsignedInteger($pow_2_53) !== self::POW_2_53) {
throw new \RunTimeException(
'This class assumes floats to be double precision or "better".'
);
}
}
/**
* @brief Formats a signed integer or float as an unsigned integer base-10
* string. Passed strings will be checked for being base-10.
*
* @param int|float|string $number Number containing unsigned integer data
*
* @throws \UnexpectedValueException if $number is not a float, not an int
* and not a base-10 string.
*
* @return string Unsigned integer base-10 string
*/
public function formatUnsignedInteger($number) {
if (is_float($number)) {
// Undo the effect of the php.ini setting 'precision'.
return number_format($number, 0, '', '');
} else if (is_string($number) && ctype_digit($number)) {
return $number;
} else if (is_int($number)) {
// Interpret signed integer as unsigned integer.
return sprintf('%u', $number);
} else {
throw new \UnexpectedValueException(
'Expected int, float or base-10 string'
);
}
}
/**
* @brief Tries to get the size of a file via various workarounds that
* even work for large files on 32-bit platforms.
*
* @param string $filename Path to the file.
*
* @return null|int|float Number of bytes as number (float or int) or
* null on failure.
*/
public function getFileSize($filename) {
$fileSize = $this->getFileSizeViaCurl($filename);
if (!is_null($fileSize)) {
return $fileSize;
}
$fileSize = $this->getFileSizeViaCOM($filename);
if (!is_null($fileSize)) {
return $fileSize;
}
$fileSize = $this->getFileSizeViaExec($filename);
if (!is_null($fileSize)) {
return $fileSize;
}
return $this->getFileSizeNative($filename);
}
/**
* @brief Tries to get the size of a file via a CURL HEAD request.
*
* @param string $filename Path to the file.
*
* @return null|int|float Number of bytes as number (float or int) or
* null on failure.
*/
public function getFileSizeViaCurl($filename) {
if (function_exists('curl_init')) {
$ch = curl_init("file://$filename");
curl_setopt($ch, CURLOPT_NOBODY, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HEADER, true);
$data = curl_exec($ch);
curl_close($ch);
if ($data !== false) {
$matches = array();
preg_match('/Content-Length: (\d+)/', $data, $matches);
if (isset($matches[1])) {
return 0 + $matches[1];
}
}
}
return null;
}
/**
* @brief Tries to get the size of a file via the Windows DOM extension.
*
* @param string $filename Path to the file.
*
* @return null|int|float Number of bytes as number (float or int) or
* null on failure.
*/
public function getFileSizeViaCOM($filename) {
if (class_exists('COM')) {
$fsobj = new \COM("Scripting.FileSystemObject");
$file = $fsobj->GetFile($filename);
return 0 + $file->Size;
}
return null;
}
/**
* @brief Tries to get the size of a file via an exec() call.
*
* @param string $filename Path to the file.
*
* @return null|int|float Number of bytes as number (float or int) or
* null on failure.
*/
public function getFileSizeViaExec($filename) {
if (\OC_Helper::is_function_enabled('exec')) {
$os = strtolower(php_uname('s'));
$arg = escapeshellarg($filename);
$result = '';
if (strpos($os, 'linux') !== false) {
$result = $this->exec("stat -c %s $arg");
} else if (strpos($os, 'bsd') !== false) {
$result = $this->exec("stat -f %z $arg");
} else if (strpos($os, 'win') !== false) {
$result = $this->exec("for %F in ($arg) do @echo %~zF");
if (is_null($result)) {
// PowerShell
$result = $this->exec("(Get-Item $arg).length");
}
}
return $result;
}
return null;
}
/**
* @brief Gets the size of a file via a filesize() call and converts
* negative signed int to positive float. As the result of filesize()
* will wrap around after a file size of 2^32 bytes = 4 GiB, this
* should only be used as a last resort.
*
* @param string $filename Path to the file.
*
* @return int|float Number of bytes as number (float or int).
*/
public function getFileSizeNative($filename) {
$result = filesize($filename);
if ($result < 0) {
// For file sizes between 2 GiB and 4 GiB, filesize() will return a
// negative int, as the PHP data type int is signed. Interpret the
// returned int as an unsigned integer and put it into a float.
return (float) sprintf('%u', $result);
}
return $result;
}
protected function exec($cmd) {
$result = trim(exec($cmd));
return ctype_digit($result) ? 0 + $result : null;
}
}

46
tests/lib/largefilehelper.php Normal file
View File

@ -0,0 +1,46 @@
<?php
/**
* Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test;
class LargeFileHelper extends \PHPUnit_Framework_TestCase {
protected $helper;
public function setUp() {
parent::setUp();
$this->helper = new \OC\LargeFileHelper;
}
public function testFormatUnsignedIntegerFloat() {
$this->assertSame(
'9007199254740992',
$this->helper->formatUnsignedInteger((float) 9007199254740992)
);
}
public function testFormatUnsignedIntegerInt() {
$this->assertSame(
PHP_INT_SIZE === 4 ? '4294967295' : '18446744073709551615',
$this->helper->formatUnsignedInteger(-1)
);
}
public function testFormatUnsignedIntegerString() {
$this->assertSame(
'9007199254740993',
$this->helper->formatUnsignedInteger('9007199254740993')
);
}
/**
* @expectedException \UnexpectedValueException
*/
public function testFormatUnsignedIntegerStringException() {
$this->helper->formatUnsignedInteger('900ABCD254740993');
}
}

69
tests/lib/largefilehelpergetfilesize.php Normal file
View File

@ -0,0 +1,69 @@
<?php
/**
* Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test;
/**
* Tests whether LargeFileHelper is able to determine file size at all.
* Large files are not considered yet.
*/
class LargeFileHelperGetFileSize extends \PHPUnit_Framework_TestCase {
protected $filename;
protected $fileSize;
protected $helper;
public function setUp() {
parent::setUp();
$this->filename = __DIR__ . '/../data/data.tar.gz';
$this->fileSize = 4195;
$this->helper = new \OC\LargeFileHelper;
}
public function testGetFileSizeViaCurl() {
if (!extension_loaded('curl')) {
$this->markTestSkipped(
'The PHP curl extension is required for this test.'
);
}
$this->assertSame(
$this->fileSize,
$this->helper->getFileSizeViaCurl($this->filename)
);
}
public function testGetFileSizeViaCOM() {
if (!extension_loaded('COM')) {
$this->markTestSkipped(
'The PHP Windows COM extension is required for this test.'
);
}
$this->assertSame(
$this->fileSize,
$this->helper->getFileSizeViaCOM($this->filename)
);
}
public function testGetFileSizeViaExec() {
if (!\OC_Helper::is_function_enabled('exec')) {
$this->markTestSkipped(
'The exec() function needs to be enabled for this test.'
);
}
$this->assertSame(
$this->fileSize,
$this->helper->getFileSizeViaExec($this->filename)
);
}
public function testGetFileSizeNative() {
$this->assertSame(
$this->fileSize,
$this->helper->getFileSizeNative($this->filename)
);
}
}