From fd8cb9974be30aaca0d65d1807d6a4f784da5f0b Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 6 Feb 2013 23:36:38 +0100 Subject: [PATCH] initial version of a local storage implementation which will use unique slugified filename on the local filesystem. This implementation will only be enabled on windows based system to solve the issues around UTF-8 file names with php on windows. --- db_structure.xml | 44 ++++ lib/files/mapper.php | 216 ++++++++++++++++++ lib/files/storage/local.php | 5 + lib/files/storage/mappedlocal.php | 335 ++++++++++++++++++++++++++++ lib/files/storage/temporary.php | 1 + tests/lib/files/storage/storage.php | 17 +- 6 files changed, 614 insertions(+), 4 deletions(-) create mode 100644 lib/files/mapper.php create mode 100644 lib/files/storage/mappedlocal.php diff --git a/db_structure.xml b/db_structure.xml index f4111bfabd..fc7f1082ff 100644 --- a/db_structure.xml +++ b/db_structure.xml @@ -94,6 +94,50 @@ + + + *dbprefix*file_map + + + + + logic_path + text + + true + 512 + + + + physic_path + text + + true + 512 + + + + file_map_lp_index + true + + logic_path + ascending + + + + + file_map_pp_index + true + + physic_path + ascending + + + + + +
+ *dbprefix*mimetypes diff --git a/lib/files/mapper.php b/lib/files/mapper.php new file mode 100644 index 0000000000..90e4e1ca66 --- /dev/null +++ b/lib/files/mapper.php @@ -0,0 +1,216 @@ +resolveLogicPath($logicPath); + if ($physicalPath !== null) { + return $physicalPath; + } + + return $this->create($logicPath, $create); + } + + /** + * @param string $physicalPath + * @return string|null + */ + public function physicalToLogic($physicalPath) { + $logicPath = $this->resolvePhysicalPath($physicalPath); + if ($logicPath !== null) { + return $logicPath; + } + + $this->insert($physicalPath, $physicalPath); + return $physicalPath; + } + + /** + * @param string $path + * @param bool $isLogicPath indicates if $path is logical or physical + * @param $recursive + */ + public function removePath($path, $isLogicPath, $recursive) { + if ($recursive) { + $path=$path.'%'; + } + + if ($isLogicPath) { + $query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?'); + $query->execute(array($path)); + } else { + $query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `physic_path` LIKE ?'); + $query->execute(array($path)); + } + } + + /** + * @param $path1 + * @param $path2 + * @throws \Exception + */ + public function copy($path1, $path2) + { + $path1 = $this->stripLast($path1); + $path2 = $this->stripLast($path2); + $physicPath1 = $this->logicToPhysical($path1, true); + $physicPath2 = $this->logicToPhysical($path2, true); + + $query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?'); + $result = $query->execute(array($path1.'%')); + $updateQuery = \OC_DB::prepare('UPDATE `*PREFIX*file_map`' + .' SET `logic_path` = ?' + .' AND `physic_path` = ?' + .' WHERE `logic_path` = ?'); + while( $row = $result->fetchRow()) { + $currentLogic = $row['logic_path']; + $currentPhysic = $row['physic_path']; + $newLogic = $path2.$this->stripRootFolder($currentLogic, $path1); + $newPhysic = $physicPath2.$this->stripRootFolder($currentPhysic, $physicPath1); + if ($path1 !== $currentLogic) { + try { + $updateQuery->execute(array($newLogic, $newPhysic, $currentLogic)); + } catch (\Exception $e) { + error_log('Mapper::Copy failed '.$currentLogic.' -> '.$newLogic.'\n'.$e); + throw $e; + } + } + } + } + + /** + * @param $path + * @param $root + * @return bool|string + */ + public function stripRootFolder($path, $root) { + if (strpos($path, $root) !== 0) { + // throw exception ??? + return false; + } + if (strlen($path) > strlen($root)) { + return substr($path, strlen($root)); + } + + return ''; + } + + private function stripLast($path) { + if (substr($path, -1) == '/') { + $path = substr_replace($path ,'',-1); + } + return $path; + } + + private function resolveLogicPath($logicPath) { + $logicPath = $this->stripLast($logicPath); + $query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` = ?'); + $result = $query->execute(array($logicPath)); + $result = $result->fetchRow(); + + return $result['physic_path']; + } + + private function resolvePhysicalPath($physicalPath) { + $physicalPath = $this->stripLast($physicalPath); + $query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `physic_path` = ?'); + $result = $query->execute(array($physicalPath)); + $result = $result->fetchRow(); + + return $result['logic_path']; + } + + private function create($logicPath, $store) { + $logicPath = $this->stripLast($logicPath); + $index = 0; + + // create the slugified path + $physicalPath = $this->slugifyPath($logicPath); + + // detect duplicates + while ($this->resolvePhysicalPath($physicalPath) !== null) { + $physicalPath = $this->slugifyPath($physicalPath, $index++); + } + + // insert the new path mapping if requested + if ($store) { + $this->insert($logicPath, $physicalPath); + } + + return $physicalPath; + } + + private function insert($logicPath, $physicalPath) { + $query = \OC_DB::prepare('INSERT INTO `*PREFIX*file_map`(`logic_path`,`physic_path`) VALUES(?,?)'); + $query->execute(array($logicPath, $physicalPath)); + } + + private function slugifyPath($path, $index=null) { + $pathElements = explode('/', $path); + $sluggedElements = array(); + + // skip slugging the drive letter on windows - TODO: test if local path + if (strpos(strtolower(php_uname('s')), 'win') !== false) { + $sluggedElements[]= $pathElements[0]; + array_shift($pathElements); + } + foreach ($pathElements as $pathElement) { + // TODO: remove file ext before slugify on last element + $sluggedElements[] = self::slugify($pathElement); + } + + // + // TODO: add the index before the file extension + // + if ($index !== null) { + $last= end($sluggedElements); + array_pop($sluggedElements); + array_push($sluggedElements, $last.'-'.$index); + } + return implode(DIRECTORY_SEPARATOR, $sluggedElements); + } + + /** + * Modifies a string to remove all non ASCII characters and spaces. + * + * @param string $text + * @return string + */ + private function slugify($text) + { + // replace non letter or digits by - + $text = preg_replace('~[^\\pL\d]+~u', '-', $text); + + // trim + $text = trim($text, '-'); + + // transliterate + if (function_exists('iconv')) { + $text = iconv('utf-8', 'us-ascii//TRANSLIT', $text); + } + + // lowercase + $text = strtolower($text); + + // remove unwanted characters + $text = preg_replace('~[^-\w]+~', '', $text); + + if (empty($text)) + { + // TODO: we better generate a guid in this case + return 'n-a'; + } + + return $text; + } +} diff --git a/lib/files/storage/local.php b/lib/files/storage/local.php index a5db4ba919..d387a89832 100644 --- a/lib/files/storage/local.php +++ b/lib/files/storage/local.php @@ -8,6 +8,10 @@ namespace OC\Files\Storage; +if (\OC_Util::runningOnWindows()) { + require_once 'mappedlocal.php'; +} else { + /** * for local filestore, we only have to map the paths */ @@ -245,3 +249,4 @@ class Local extends \OC\Files\Storage\Common{ return $this->filemtime($path)>$time; } } +} diff --git a/lib/files/storage/mappedlocal.php b/lib/files/storage/mappedlocal.php new file mode 100644 index 0000000000..80dd79bc41 --- /dev/null +++ b/lib/files/storage/mappedlocal.php @@ -0,0 +1,335 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ +namespace OC\Files\Storage; + +/** + * for local filestore, we only have to map the paths + */ +class Local extends \OC\Files\Storage\Common{ + protected $datadir; + private $mapper; + + public function __construct($arguments) { + $this->datadir=$arguments['datadir']; + if(substr($this->datadir, -1)!=='/') { + $this->datadir.='/'; + } + + $this->mapper= new \OC\Files\Mapper(); + } + public function __destruct() { + if (defined('PHPUNIT_RUN')) { + $this->mapper->removePath($this->datadir, true, true); + } + } + public function getId(){ + return 'local::'.$this->datadir; + } + public function mkdir($path) { + return @mkdir($this->buildPath($path)); + } + public function rmdir($path) { + if ($result = @rmdir($this->buildPath($path))) { + $this->cleanMapper($path); + } + return $result; + } + public function opendir($path) { + $files = array('.', '..'); + $physicalPath= $this->buildPath($path); + + $logicalPath = $this->mapper->physicalToLogic($physicalPath); + $dh = opendir($physicalPath); + while ($file = readdir($dh)) { + if ($file === '.' or $file === '..') { + continue; + } + + $logicalFilePath = $this->mapper->physicalToLogic($physicalPath.DIRECTORY_SEPARATOR.$file); + + $file= $this->mapper->stripRootFolder($logicalFilePath, $logicalPath); + $file = $this->stripLeading($file); + $files[]= $file; + } + + \OC\Files\Stream\Dir::register('local-win32'.$path, $files); + return opendir('fakedir://local-win32'.$path); + } + public function is_dir($path) { + if(substr($path,-1)=='/') { + $path=substr($path, 0, -1); + } + return is_dir($this->buildPath($path)); + } + public function is_file($path) { + return is_file($this->buildPath($path)); + } + public function stat($path) { + $fullPath = $this->buildPath($path); + $statResult = stat($fullPath); + + if ($statResult['size'] < 0) { + $size = self::getFileSizeFromOS($fullPath); + $statResult['size'] = $size; + $statResult[7] = $size; + } + return $statResult; + } + public function filetype($path) { + $filetype=filetype($this->buildPath($path)); + if($filetype=='link') { + $filetype=filetype(realpath($this->buildPath($path))); + } + return $filetype; + } + public function filesize($path) { + if($this->is_dir($path)) { + return 0; + }else{ + $fullPath = $this->buildPath($path); + $fileSize = filesize($fullPath); + if ($fileSize < 0) { + return self::getFileSizeFromOS($fullPath); + } + + return $fileSize; + } + } + public function isReadable($path) { + return is_readable($this->buildPath($path)); + } + public function isUpdatable($path) { + return is_writable($this->buildPath($path)); + } + public function file_exists($path) { + return file_exists($this->buildPath($path)); + } + public function filemtime($path) { + return filemtime($this->buildPath($path)); + } + public function touch($path, $mtime=null) { + // sets the modification time of the file to the given value. + // If mtime is nil the current time is set. + // note that the access time of the file always changes to the current time. + if(!is_null($mtime)) { + $result=touch( $this->buildPath($path), $mtime ); + }else{ + $result=touch( $this->buildPath($path)); + } + if( $result ) { + clearstatcache( true, $this->buildPath($path) ); + } + + return $result; + } + public function file_get_contents($path) { + return file_get_contents($this->buildPath($path)); + } + public function file_put_contents($path, $data) {//trigger_error("$path = ".var_export($path, 1)); + return file_put_contents($this->buildPath($path), $data); + } + public function unlink($path) { + return $this->delTree($path); + } + public function rename($path1, $path2) { + if (!$this->isUpdatable($path1)) { + \OC_Log::write('core','unable to rename, file is not writable : '.$path1,\OC_Log::ERROR); + return false; + } + if(! $this->file_exists($path1)) { + \OC_Log::write('core','unable to rename, file does not exists : '.$path1,\OC_Log::ERROR); + return false; + } + + $physicPath1 = $this->buildPath($path1); + $physicPath2 = $this->buildPath($path2); + if($return=rename($physicPath1, $physicPath2)) { + // mapper needs to create copies or all children + $this->copyMapping($path1, $path2); + $this->cleanMapper($physicPath1, false, true); + } + return $return; + } + public function copy($path1, $path2) { + if($this->is_dir($path2)) { + if(!$this->file_exists($path2)) { + $this->mkdir($path2); + } + $source=substr($path1, strrpos($path1, '/')+1); + $path2.=$source; + } + if($return=copy($this->buildPath($path1), $this->buildPath($path2))) { + // mapper needs to create copies or all children + $this->copyMapping($path1, $path2); + } + return $return; + } + public function fopen($path, $mode) { + if($return=fopen($this->buildPath($path), $mode)) { + switch($mode) { + case 'r': + break; + case 'r+': + case 'w+': + case 'x+': + case 'a+': + break; + case 'w': + case 'x': + case 'a': + break; + } + } + return $return; + } + + public function getMimeType($path) { + if($this->isReadable($path)) { + return \OC_Helper::getMimeType($this->buildPath($path)); + }else{ + return false; + } + } + + private function delTree($dir, $isLogicPath=true) { + $dirRelative=$dir; + if ($isLogicPath) { + $dir=$this->buildPath($dir); + } + if (!file_exists($dir)) { + return true; + } + if (!is_dir($dir) || is_link($dir)) { + if($return=unlink($dir)) { + $this->cleanMapper($dir, false); + return $return; + } + } + foreach (scandir($dir) as $item) { + if ($item == '.' || $item == '..') { + continue; + } + if(is_file($dir.'/'.$item)) { + if(unlink($dir.'/'.$item)) { + $this->cleanMapper($dir.'/'.$item, false); + } + }elseif(is_dir($dir.'/'.$item)) { + if (!$this->delTree($dir. "/" . $item, false)) { + return false; + }; + } + } + if($return=rmdir($dir)) { + $this->cleanMapper($dir, false); + } + return $return; + } + + private static function getFileSizeFromOS($fullPath) { + $name = strtolower(php_uname('s')); + // Windows OS: we use COM to access the filesystem + if (strpos($name, 'win') !== false) { + if (class_exists('COM')) { + $fsobj = new \COM("Scripting.FileSystemObject"); + $f = $fsobj->GetFile($fullPath); + return $f->Size; + } + } else if (strpos($name, 'bsd') !== false) { + if (\OC_Helper::is_function_enabled('exec')) { + return (float)exec('stat -f %z ' . escapeshellarg($fullPath)); + } + } else if (strpos($name, 'linux') !== false) { + if (\OC_Helper::is_function_enabled('exec')) { + return (float)exec('stat -c %s ' . escapeshellarg($fullPath)); + } + } else { + \OC_Log::write('core', 'Unable to determine file size of "'.$fullPath.'". Unknown OS: '.$name, \OC_Log::ERROR); + } + + return 0; + } + + public function hash($path, $type, $raw=false) { + return hash_file($type, $this->buildPath($path), $raw); + } + + public function free_space($path) { + return @disk_free_space($this->buildPath($path)); + } + + public function search($query) { + return $this->searchInDir($query); + } + public function getLocalFile($path) { + return $this->buildPath($path); + } + public function getLocalFolder($path) { + return $this->buildPath($path); + } + + protected function searchInDir($query, $dir='', $isLogicPath=true) { + $files=array(); + $physicalDir = $this->buildPath($dir); + foreach (scandir($physicalDir) as $item) { + if ($item == '.' || $item == '..') + continue; + $physicalItem = $this->mapper->physicalToLogic($physicalDir.DIRECTORY_SEPARATOR.$item); + $item = substr($physicalItem, strlen($physicalDir)+1); + + if(strstr(strtolower($item), strtolower($query)) !== false) { + $files[]=$dir.'/'.$item; + } + if(is_dir($physicalItem)) { + $files=array_merge($files, $this->searchInDir($query, $physicalItem, false)); + } + } + return $files; + } + + /** + * check if a file or folder has been updated since $time + * @param string $path + * @param int $time + * @return bool + */ + public function hasUpdated($path, $time) { + return $this->filemtime($path)>$time; + } + + private function buildPath($path, $create=true) { + $path = $this->stripLeading($path); + $fullPath = $this->datadir.$path; + return $this->mapper->logicToPhysical($fullPath, $create); + } + + private function cleanMapper($path, $isLogicPath=true, $recursive=true) { + $fullPath = $path; + if ($isLogicPath) { + $fullPath = $this->datadir.$path; + } + $this->mapper->removePath($fullPath, $isLogicPath, $recursive); + } + + private function copyMapping($path1, $path2) { + $path1 = $this->stripLeading($path1); + $path2 = $this->stripLeading($path2); + + $fullPath1 = $this->datadir.$path1; + $fullPath2 = $this->datadir.$path2; + + $this->mapper->copy($fullPath1, $fullPath2); + } + + private function stripLeading($path) { + if(strpos($path, '/') === 0) { + $path = substr($path, 1); + } + + return $path; + } +} diff --git a/lib/files/storage/temporary.php b/lib/files/storage/temporary.php index 542d2cd9f4..d84dbda2e3 100644 --- a/lib/files/storage/temporary.php +++ b/lib/files/storage/temporary.php @@ -21,6 +21,7 @@ class Temporary extends Local{ } public function __destruct() { + parent::__destruct(); $this->cleanUp(); } } diff --git a/tests/lib/files/storage/storage.php b/tests/lib/files/storage/storage.php index 781c0f92c9..c74a16f509 100644 --- a/tests/lib/files/storage/storage.php +++ b/tests/lib/files/storage/storage.php @@ -146,10 +146,19 @@ abstract class Storage extends \PHPUnit_Framework_TestCase { $localFolder = $this->instance->getLocalFolder('/folder'); $this->assertTrue(is_dir($localFolder)); - $this->assertTrue(file_exists($localFolder . '/lorem.txt')); - $this->assertEquals(file_get_contents($localFolder . '/lorem.txt'), file_get_contents($textFile)); - $this->assertEquals(file_get_contents($localFolder . '/bar.txt'), 'asd'); - $this->assertEquals(file_get_contents($localFolder . '/recursive/file.txt'), 'foo'); + + // test below require to use instance->getLocalFile because the physical storage might be different + $localFile = $this->instance->getLocalFile('/folder/lorem.txt'); + $this->assertTrue(file_exists($localFile)); + $this->assertEquals(file_get_contents($localFile), file_get_contents($textFile)); + + $localFile = $this->instance->getLocalFile('/folder/bar.txt'); + $this->assertTrue(file_exists($localFile)); + $this->assertEquals(file_get_contents($localFile), 'asd'); + + $localFile = $this->instance->getLocalFile('/folder/recursive/file.txt'); + $this->assertTrue(file_exists($localFile)); + $this->assertEquals(file_get_contents($localFile), 'foo'); } public function testStat() {