Merge pull request #1512 from owncloud/mapped-filename-storage

initial version of a local storage implementation which will use unique ...

db_structure.xml

@@ -94,6 +94,50 @@
 
 	</table>
 
+	<table>
+
+		<name>*dbprefix*file_map</name>
+
+		<declaration>
+
+			<field>
+				<name>logic_path</name>
+				<type>text</type>
+				<default></default>
+				<notnull>true</notnull>
+				<length>512</length>
+			</field>
+
+			<field>
+				<name>physic_path</name>
+				<type>text</type>
+				<default></default>
+				<notnull>true</notnull>
+				<length>512</length>
+			</field>
+
+			<index>
+				<name>file_map_lp_index</name>
+				<unique>true</unique>
+				<field>
+					<name>logic_path</name>
+					<sorting>ascending</sorting>
+				</field>
+			</index>
+
+			<index>
+				<name>file_map_pp_index</name>
+				<unique>true</unique>
+				<field>
+					<name>physic_path</name>
+					<sorting>ascending</sorting>
+				</field>
+			</index>
+
+		</declaration>
+
+	</table>
+
 	<table>
 
 		<name>*dbprefix*mimetypes</name>

lib/files/mapper.php

@@ -0,0 +1,216 @@
+<?php
+
+namespace OC\Files;
+
+/**
+ * class Mapper is responsible to translate logical paths to physical paths and reverse
+ */
+class Mapper
+{
+	/**
+	 * @param string $logicPath
+	 * @param bool $create indicates if the generated physical name shall be stored in the database or not
+	 * @return string the physical path
+	 */
+	public function logicToPhysical($logicPath, $create) {
+		$physicalPath = $this->resolveLogicPath($logicPath);
+		if ($physicalPath !== null) {
+			return $physicalPath;
+		}
+
+		return $this->create($logicPath, $create);
+	}
+
+	/**
+	 * @param string $physicalPath
+	 * @return string|null
+	 */
+	public function physicalToLogic($physicalPath) {
+		$logicPath = $this->resolvePhysicalPath($physicalPath);
+		if ($logicPath !== null) {
+			return $logicPath;
+		}
+
+		$this->insert($physicalPath, $physicalPath);
+		return $physicalPath;
+	}
+
+	/**
+	 * @param string $path
+	 * @param bool $isLogicPath indicates if $path is logical or physical
+	 * @param $recursive
+	 */
+	public function removePath($path, $isLogicPath, $recursive) {
+		if ($recursive) {
+			$path=$path.'%';
+		}
+
+		if ($isLogicPath) {
+			$query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?');
+			$query->execute(array($path));
+		} else {
+			$query = \OC_DB::prepare('DELETE FROM `*PREFIX*file_map` WHERE `physic_path` LIKE ?');
+			$query->execute(array($path));
+		}
+	}
+
+	/**
+	 * @param $path1
+	 * @param $path2
+	 * @throws \Exception
+	 */
+	public function copy($path1, $path2)
+	{
+		$path1 = $this->stripLast($path1);
+		$path2 = $this->stripLast($path2);
+		$physicPath1 = $this->logicToPhysical($path1, true);
+		$physicPath2 = $this->logicToPhysical($path2, true);
+
+		$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` LIKE ?');
+		$result = $query->execute(array($path1.'%'));
+		$updateQuery = \OC_DB::prepare('UPDATE `*PREFIX*file_map`'
+			.' SET `logic_path` = ?'
+			.' AND `physic_path` = ?'
+			.' WHERE `logic_path` = ?');
+		while( $row = $result->fetchRow()) {
+			$currentLogic = $row['logic_path'];
+			$currentPhysic = $row['physic_path'];
+			$newLogic = $path2.$this->stripRootFolder($currentLogic, $path1);
+			$newPhysic = $physicPath2.$this->stripRootFolder($currentPhysic, $physicPath1);
+			if ($path1 !== $currentLogic) {
+				try {
+					$updateQuery->execute(array($newLogic, $newPhysic, $currentLogic));
+				} catch (\Exception $e) {
+					error_log('Mapper::Copy failed '.$currentLogic.' -> '.$newLogic.'\n'.$e);
+					throw $e;
+				}
+			}
+		}
+	}
+
+	/**
+	 * @param $path
+	 * @param $root
+	 * @return bool|string
+	 */
+	public function stripRootFolder($path, $root) {
+		if (strpos($path, $root) !== 0) {
+			// throw exception ???
+			return false;
+		}
+		if (strlen($path) > strlen($root)) {
+			return substr($path, strlen($root));
+		}
+
+		return '';
+	}
+
+	private function stripLast($path) {
+		if (substr($path, -1) == '/') {
+			$path = substr_replace($path ,'',-1);
+		}
+		return $path;
+	}
+
+	private function resolveLogicPath($logicPath) {
+		$logicPath = $this->stripLast($logicPath);
+		$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `logic_path` = ?');
+		$result = $query->execute(array($logicPath));
+		$result = $result->fetchRow();
+
+		return $result['physic_path'];
+	}
+
+	private function resolvePhysicalPath($physicalPath) {
+		$physicalPath = $this->stripLast($physicalPath);
+		$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*file_map` WHERE `physic_path` = ?');
+		$result = $query->execute(array($physicalPath));
+		$result = $result->fetchRow();
+
+		return $result['logic_path'];
+	}
+
+	private function create($logicPath, $store) {
+		$logicPath = $this->stripLast($logicPath);
+		$index = 0;
+
+		// create the slugified path
+		$physicalPath = $this->slugifyPath($logicPath);
+
+		// detect duplicates
+		while ($this->resolvePhysicalPath($physicalPath) !== null) {
+			$physicalPath = $this->slugifyPath($physicalPath, $index++);
+		}
+
+		// insert the new path mapping if requested
+		if ($store) {
+			$this->insert($logicPath, $physicalPath);
+		}
+
+		return $physicalPath;
+	}
+
+	private function insert($logicPath, $physicalPath) {
+		$query = \OC_DB::prepare('INSERT INTO `*PREFIX*file_map`(`logic_path`,`physic_path`) VALUES(?,?)');
+		$query->execute(array($logicPath, $physicalPath));
+	}
+
+	private function slugifyPath($path, $index=null) {
+		$pathElements = explode('/', $path);
+		$sluggedElements = array();
+
+		// skip slugging the drive letter on windows - TODO: test if local path
+		if (strpos(strtolower(php_uname('s')), 'win') !== false) {
+			$sluggedElements[]= $pathElements[0];
+			array_shift($pathElements);
+		}
+		foreach ($pathElements as $pathElement) {
+			// TODO: remove file ext before slugify on last element
+			$sluggedElements[] = self::slugify($pathElement);
+		}
+
+		//
+		// TODO: add the index before the file extension
+		//
+		if ($index !== null) {
+			$last= end($sluggedElements);
+			array_pop($sluggedElements);
+			array_push($sluggedElements, $last.'-'.$index);
+		}
+		return implode(DIRECTORY_SEPARATOR, $sluggedElements);
+	}
+
+	/**
+	 * Modifies a string to remove all non ASCII characters and spaces.
+	 *
+	 * @param string $text
+	 * @return string
+	 */
+	private function slugify($text)
+	{
+		// replace non letter or digits by -
+		$text = preg_replace('~[^\\pL\d]+~u', '-', $text);
+
+		// trim
+		$text = trim($text, '-');
+
+		// transliterate
+		if (function_exists('iconv')) {
+			$text = iconv('utf-8', 'us-ascii//TRANSLIT', $text);
+		}
+
+		// lowercase
+		$text = strtolower($text);
+
+		// remove unwanted characters
+		$text = preg_replace('~[^-\w]+~', '', $text);
+
+		if (empty($text))
+		{
+			// TODO: we better generate a guid in this case
+			return 'n-a';
+		}
+
+		return $text;
+	}
+}

lib/files/storage/local.php

@@ -8,6 +8,10 @@
 
 namespace OC\Files\Storage;
 
+if (\OC_Util::runningOnWindows()) {
+	require_once 'mappedlocal.php';
+} else {
+
 /**
  * for local filestore, we only have to map the paths
  */
@@ -245,3 +249,4 @@ class Local extends \OC\Files\Storage\Common{
 		return $this->filemtime($path)>$time;
 	}
 }
+}

lib/files/storage/mappedlocal.php

@@ -0,0 +1,335 @@
+<?php
+/**
+ * Copyright (c) 2012 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+namespace OC\Files\Storage;
+
+/**
+ * for local filestore, we only have to map the paths
+ */
+class Local extends \OC\Files\Storage\Common{
+	protected $datadir;
+	private $mapper;
+
+	public function __construct($arguments) {
+		$this->datadir=$arguments['datadir'];
+		if(substr($this->datadir, -1)!=='/') {
+			$this->datadir.='/';
+		}
+
+		$this->mapper= new \OC\Files\Mapper();
+	}
+	public function __destruct() {
+		if (defined('PHPUNIT_RUN')) {
+			$this->mapper->removePath($this->datadir, true, true);
+		}
+	}
+	public function getId(){
+		return 'local::'.$this->datadir;
+	}
+	public function mkdir($path) {
+		return @mkdir($this->buildPath($path));
+	}
+	public function rmdir($path) {
+		if ($result = @rmdir($this->buildPath($path))) {
+			$this->cleanMapper($path);
+		}
+		return $result;
+	}
+	public function opendir($path) {
+		$files = array('.', '..');
+		$physicalPath= $this->buildPath($path);
+
+		$logicalPath = $this->mapper->physicalToLogic($physicalPath);
+		$dh = opendir($physicalPath);
+		while ($file = readdir($dh)) {
+			if ($file === '.' or $file === '..') {
+				continue;
+			}
+
+			$logicalFilePath = $this->mapper->physicalToLogic($physicalPath.DIRECTORY_SEPARATOR.$file);
+
+			$file= $this->mapper->stripRootFolder($logicalFilePath, $logicalPath);
+			$file = $this->stripLeading($file);
+			$files[]= $file;
+		}
+
+		\OC\Files\Stream\Dir::register('local-win32'.$path, $files);
+		return opendir('fakedir://local-win32'.$path);
+	}
+	public function is_dir($path) {
+		if(substr($path,-1)=='/') {
+			$path=substr($path, 0, -1);
+		}
+		return is_dir($this->buildPath($path));
+	}
+	public function is_file($path) {
+		return is_file($this->buildPath($path));
+	}
+	public function stat($path) {
+		$fullPath = $this->buildPath($path);
+		$statResult = stat($fullPath);
+
+		if ($statResult['size'] < 0) {
+			$size = self::getFileSizeFromOS($fullPath);
+			$statResult['size'] = $size;
+			$statResult[7] = $size;
+		}
+		return $statResult;
+	}
+	public function filetype($path) {
+		$filetype=filetype($this->buildPath($path));
+		if($filetype=='link') {
+			$filetype=filetype(realpath($this->buildPath($path)));
+		}
+		return $filetype;
+	}
+	public function filesize($path) {
+		if($this->is_dir($path)) {
+			return 0;
+		}else{
+			$fullPath = $this->buildPath($path);
+			$fileSize = filesize($fullPath);
+			if ($fileSize < 0) {
+				return self::getFileSizeFromOS($fullPath);
+			}
+
+			return $fileSize;
+		}
+	}
+	public function isReadable($path) {
+		return is_readable($this->buildPath($path));
+	}
+	public function isUpdatable($path) {
+		return is_writable($this->buildPath($path));
+	}
+	public function file_exists($path) {
+		return file_exists($this->buildPath($path));
+	}
+	public function filemtime($path) {
+		return filemtime($this->buildPath($path));
+	}
+	public function touch($path, $mtime=null) {
+		// sets the modification time of the file to the given value.
+		// If mtime is nil the current time is set.
+		// note that the access time of the file always changes to the current time.
+		if(!is_null($mtime)) {
+			$result=touch( $this->buildPath($path), $mtime );
+		}else{
+			$result=touch( $this->buildPath($path));
+		}
+		if( $result ) {
+			clearstatcache( true, $this->buildPath($path) );
+		}
+
+		return $result;
+	}
+	public function file_get_contents($path) {
+		return file_get_contents($this->buildPath($path));
+	}
+	public function file_put_contents($path, $data) {//trigger_error("$path = ".var_export($path, 1));
+		return file_put_contents($this->buildPath($path), $data);
+	}
+	public function unlink($path) {
+		return $this->delTree($path);
+	}
+	public function rename($path1, $path2) {
+		if (!$this->isUpdatable($path1)) {
+			\OC_Log::write('core','unable to rename, file is not writable : '.$path1,\OC_Log::ERROR);
+			return false;
+		}
+		if(! $this->file_exists($path1)) {
+			\OC_Log::write('core','unable to rename, file does not exists : '.$path1,\OC_Log::ERROR);
+			return false;
+		}
+
+		$physicPath1 = $this->buildPath($path1);
+		$physicPath2 = $this->buildPath($path2);
+		if($return=rename($physicPath1, $physicPath2)) {
+			// mapper needs to create copies or all children
+			$this->copyMapping($path1, $path2);
+			$this->cleanMapper($physicPath1, false, true);
+		}
+		return $return;
+	}
+	public function copy($path1, $path2) {
+		if($this->is_dir($path2)) {
+			if(!$this->file_exists($path2)) {
+				$this->mkdir($path2);
+			}
+			$source=substr($path1, strrpos($path1, '/')+1);
+			$path2.=$source;
+		}
+		if($return=copy($this->buildPath($path1), $this->buildPath($path2))) {
+			// mapper needs to create copies or all children
+			$this->copyMapping($path1, $path2);
+		}
+		return $return;
+	}
+	public function fopen($path, $mode) {
+		if($return=fopen($this->buildPath($path), $mode)) {
+			switch($mode) {
+				case 'r':
+					break;
+				case 'r+':
+				case 'w+':
+				case 'x+':
+				case 'a+':
+					break;
+				case 'w':
+				case 'x':
+				case 'a':
+					break;
+			}
+		}
+		return $return;
+	}
+
+	public function getMimeType($path) {
+		if($this->isReadable($path)) {
+			return \OC_Helper::getMimeType($this->buildPath($path));
+		}else{
+			return false;
+		}
+	}
+
+	private function delTree($dir, $isLogicPath=true) {
+		$dirRelative=$dir;
+		if ($isLogicPath) {
+			$dir=$this->buildPath($dir);
+		}
+		if (!file_exists($dir)) {
+			return true;
+		}
+		if (!is_dir($dir) || is_link($dir)) {
+			if($return=unlink($dir)) {
+				$this->cleanMapper($dir, false);
+				return $return;
+			}
+		}
+		foreach (scandir($dir) as $item) {
+			if ($item == '.' || $item == '..') {
+				continue;
+			}
+			if(is_file($dir.'/'.$item)) {
+				if(unlink($dir.'/'.$item)) {
+					$this->cleanMapper($dir.'/'.$item, false);
+				}
+			}elseif(is_dir($dir.'/'.$item)) {
+				if (!$this->delTree($dir. "/" . $item, false)) {
+					return false;
+				};
+			}
+		}
+		if($return=rmdir($dir)) {
+			$this->cleanMapper($dir, false);
+		}
+		return $return;
+	}
+
+	private static function getFileSizeFromOS($fullPath) {
+		$name = strtolower(php_uname('s'));
+		// Windows OS: we use COM to access the filesystem
+		if (strpos($name, 'win') !== false) {
+			if (class_exists('COM')) {
+				$fsobj = new \COM("Scripting.FileSystemObject");
+				$f = $fsobj->GetFile($fullPath);
+				return $f->Size;
+			}
+		} else if (strpos($name, 'bsd') !== false) {
+			if (\OC_Helper::is_function_enabled('exec')) {
+				return (float)exec('stat -f %z ' . escapeshellarg($fullPath));
+			}
+		} else if (strpos($name, 'linux') !== false) {
+			if (\OC_Helper::is_function_enabled('exec')) {
+				return (float)exec('stat -c %s ' . escapeshellarg($fullPath));
+			}
+		} else {
+			\OC_Log::write('core', 'Unable to determine file size of "'.$fullPath.'". Unknown OS: '.$name, \OC_Log::ERROR);
+		}
+
+		return 0;
+	}
+
+	public function hash($path, $type, $raw=false) {
+		return hash_file($type, $this->buildPath($path), $raw);
+	}
+
+	public function free_space($path) {
+		return @disk_free_space($this->buildPath($path));
+	}
+
+	public function search($query) {
+		return $this->searchInDir($query);
+	}
+	public function getLocalFile($path) {
+		return $this->buildPath($path);
+	}
+	public function getLocalFolder($path) {
+		return $this->buildPath($path);
+	}
+
+	protected function searchInDir($query, $dir='', $isLogicPath=true) {
+		$files=array();
+		$physicalDir = $this->buildPath($dir);
+		foreach (scandir($physicalDir) as $item) {
+			if ($item == '.' || $item == '..')
+				continue;
+			$physicalItem = $this->mapper->physicalToLogic($physicalDir.DIRECTORY_SEPARATOR.$item);
+			$item = substr($physicalItem, strlen($physicalDir)+1);
+
+			if(strstr(strtolower($item), strtolower($query)) !== false) {
+				$files[]=$dir.'/'.$item;
+			}
+			if(is_dir($physicalItem)) {
+				$files=array_merge($files, $this->searchInDir($query, $physicalItem, false));
+			}
+		}
+		return $files;
+	}
+
+	/**
+	 * check if a file or folder has been updated since $time
+	 * @param string $path
+	 * @param int $time
+	 * @return bool
+	 */
+	public function hasUpdated($path, $time) {
+		return $this->filemtime($path)>$time;
+	}
+
+	private function buildPath($path, $create=true) {
+		$path = $this->stripLeading($path);
+		$fullPath = $this->datadir.$path;
+		return $this->mapper->logicToPhysical($fullPath, $create);
+	}
+
+	private function cleanMapper($path, $isLogicPath=true, $recursive=true) {
+		$fullPath = $path;
+		if ($isLogicPath) {
+			$fullPath = $this->datadir.$path;
+		}
+		$this->mapper->removePath($fullPath, $isLogicPath, $recursive);
+	}
+
+	private function copyMapping($path1, $path2) {
+		$path1 = $this->stripLeading($path1);
+		$path2 = $this->stripLeading($path2);
+
+		$fullPath1 = $this->datadir.$path1;
+		$fullPath2 = $this->datadir.$path2;
+
+		$this->mapper->copy($fullPath1, $fullPath2);
+	}
+
+	private function stripLeading($path) {
+		if(strpos($path, '/') === 0) {
+			$path = substr($path, 1);
+		}
+
+		return $path;
+	}
+}

lib/files/storage/temporary.php

@@ -21,6 +21,7 @@ class Temporary extends Local{
 	}
 
 	public function __destruct() {
+		parent::__destruct();
 		$this->cleanUp();
 	}
 }

tests/lib/files/storage/storage.php

@@ -146,10 +146,19 @@ abstract class Storage extends \PHPUnit_Framework_TestCase {
 		$localFolder = $this->instance->getLocalFolder('/folder');
 
 		$this->assertTrue(is_dir($localFolder));
-		$this->assertTrue(file_exists($localFolder . '/lorem.txt'));
-		$this->assertEquals(file_get_contents($localFolder . '/lorem.txt'), file_get_contents($textFile));
-		$this->assertEquals(file_get_contents($localFolder . '/bar.txt'), 'asd');
-		$this->assertEquals(file_get_contents($localFolder . '/recursive/file.txt'), 'foo');
+
+		// test below require to use instance->getLocalFile because the physical storage might be different
+		$localFile = $this->instance->getLocalFile('/folder/lorem.txt');
+		$this->assertTrue(file_exists($localFile));
+		$this->assertEquals(file_get_contents($localFile), file_get_contents($textFile));
+
+		$localFile = $this->instance->getLocalFile('/folder/bar.txt');
+		$this->assertTrue(file_exists($localFile));
+		$this->assertEquals(file_get_contents($localFile), 'asd');
+
+		$localFile = $this->instance->getLocalFile('/folder/recursive/file.txt');
+		$this->assertTrue(file_exists($localFile));
+		$this->assertEquals(file_get_contents($localFile), 'foo');
 	}
 
 	public function testStat() {