mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

provide caching for file metadata

This commit is contained in:
Robin Appelman 2011-11-09 18:41:57 +01:00
parent 5cc6635eb8
commit 5a6aba1e11
9 changed files with 446 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/files_sharing/lib_share.php
View File

@ -89,8 +89,8 @@ class OC_Share {
}
$query->execute(array($uid_owner, $uid, $source, $target, $permissions));
// Clear the folder size cache for the 'Shared' folder
$clearFolderSize = OC_DB::prepare("DELETE FROM *PREFIX*foldersize WHERE path = ?");
$clearFolderSize->execute(array($sharedFolder));
// $clearFolderSize = OC_DB::prepare("DELETE FROM *PREFIX*foldersize WHERE path = ?");
// $clearFolderSize->execute(array($sharedFolder));
}
}
}

12
apps/files_sharing/sharedstorage.php
View File

@ -225,8 +225,8 @@ class OC_Filestorage_Shared extends OC_Filestorage {
$path = ltrim($path, "/");
$path = preg_replace('{(/)\1+}', "/", $path);
$dbpath = rtrim($this->datadir.$path, "/");
$query = OC_DB::prepare("SELECT size FROM *PREFIX*foldersize WHERE path = ?");
$size = $query->execute(array($dbpath))->fetchAll();
// $query = OC_DB::prepare("SELECT size FROM *PREFIX*foldersize WHERE path = ?");
// $size = $query->execute(array($dbpath))->fetchAll();
if (count($size) > 0) {
return $size[0]['size'];
} else {
@ -252,8 +252,8 @@ class OC_Filestorage_Shared extends OC_Filestorage {
}
if ($size > 0) {
$dbpath = rtrim($this->datadir.$path, "/");
$query = OC_DB::prepare("INSERT INTO *PREFIX*foldersize VALUES(?,?)");
$result = $query->execute(array($dbpath, $size));
// $query = OC_DB::prepare("INSERT INTO *PREFIX*foldersize VALUES(?,?)");
// $result = $query->execute(array($dbpath, $size));
}
}
return $size;
@ -266,8 +266,8 @@ class OC_Filestorage_Shared extends OC_Filestorage {
$path = dirname($path);
}
$dbpath = rtrim($this->datadir.$path, "/");
$query = OC_DB::prepare("DELETE FROM *PREFIX*foldersize WHERE path = ?");
$result = $query->execute(array($dbpath));
// $query = OC_DB::prepare("DELETE FROM *PREFIX*/*foldersize*/ WHERE path = ?");
// $result = $query->execute(array($dbpath));
if ($path != "/" && $path != "") {
$parts = explode("/", $path);
$part = array_pop($parts);

86
db_structure.xml
View File

@ -43,10 +43,19 @@
<table>
<name>*dbprefix*foldersize</name>
<name>*dbprefix*fscache</name>
<declaration>
<field>
<name>id</name>
<autoincrement>1</autoincrement>
<type>integer</type>
<default>0</default>
<notnull>true</notnull>
<length>4</length>
</field>
<field>
<name>path</name>
<type>text</type>
@ -55,6 +64,24 @@
<length>512</length>
</field>
<field>
<name>parent</name>
<type>integer</type>
<default>
</default>
<notnull>true</notnull>
<length>4</length>
</field>
<field>
<name>name</name>
<type>text</type>
<default>
</default>
<notnull>true</notnull>
<length>512</length>
</field>
<field>
<name>size</name>
<type>integer</type>
@ -63,14 +90,71 @@
<length>4</length>
</field>
<field>
<name>ctime</name>
<type>integer</type>
<default>
</default>
<notnull>true</notnull>
<length>4</length>
</field>
<field>
<name>mtime</name>
<type>integer</type>
<default>
</default>
<notnull>true</notnull>
<length>4</length>
</field>
<field>
<name>mimetype</name>
<type>text</type>
<default>
</default>
<notnull>true</notnull>
<length>32</length>
</field>
<field>
<name>mimepart</name>
<type>text</type>
<default>
</default>
<notnull>true</notnull>
<length>32</length>
</field>
<index>
<name>path_index</name>
<unique>true</unique>
<field>
<name>path</name>
<sorting>ascending</sorting>
</field>
</index>
<index>
<name>parent_index</name>
<field>
<name>parent</name>
<sorting>ascending</sorting>
</field>
</index>
<index>
<name>parent_name_index</name>
<field>
<name>parent</name>
<sorting>ascending</sorting>
</field>
<field>
<name>name</name>
<sorting>ascending</sorting>
</field>
</index>
</declaration>
</table>

4
files/templates/part.list.php
View File

@ -5,8 +5,8 @@
$relative_modified_date = relative_modified_date($file['mtime']);
$relative_date_color = round((time()-$file['mtime'])/60/60/24*14); // the older the file, the brighter the shade of grey; days*14
if($relative_date_color>200) $relative_date_color = 200; ?>
<tr data-file="<?php echo str_replace('+','%20',urlencode($file['name']));?>" data-type="<?php echo ($file['type'] == 'dir')?'dir':'file'?>" data-mime="<?php echo $file['mime']?>" data-size='<?php echo $file['size'];?>'>
<td class="filename svg" style="background-image:url(<?php if($file['type'] == 'dir') echo mimetype_icon('dir'); else echo mimetype_icon($file['mime']); ?>)">
<tr data-file="<?php echo str_replace('+','%20',urlencode($file['name']));?>" data-type="<?php echo ($file['type'] == 'dir')?'dir':'file'?>" data-mime="<?php echo $file['mimetype']?>" data-size='<?php echo $file['size'];?>'>
<td class="filename svg" style="background-image:url(<?php if($file['type'] == 'dir') echo mimetype_icon('dir'); else echo mimetype_icon($file['mimetype']); ?>)">
<?php if(!isset($_['readonly']) || !$_['readonly']) { ?><input type="checkbox" /><?php } ?>
<a class="name" href="<?php if($file['type'] == 'dir') echo $_['baseURL'].$file['directory'].'/'.$file['name']; else echo $_['downloadURL'].urlencode($file['directory']).'/'.urlencode($file['name']); ?>" title="">
<span class="nametext">

331
lib/filecache.php Normal file
View File

@ -0,0 +1,331 @@
<?php
/**
* @author Robin Appelman
* @copyright 2011 Robin Appelman icewind1991@gmail.com
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
* License as published by the Free Software Foundation; either
* version 3 of the License, or any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU Affero General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
*
*/
/**
* provide caching for filesystem info in the database
*
* not used by OC_Filesystem for reading filesystem info,
* instread apps should use OC_FileCache::get where possible
*
* It will try to keep the data up to date but changes from outside ownCloud can invalidate the cache
*/
class OC_FileCache{
/**
* get the filesystem info from the cache
* @param string path
* @return array
*
* returns an assiciative array with the following keys:
* - size
* - mtime
* - ctime
* - mimetype
*/
public static function get($path){
$path=OC_Filesystem::getRoot().$path;
$query=OC_DB::prepare('SELECT ctime,mtime,mimetype,size FROM *PREFIX*fscache WHERE path=?');
$result=$query->execute(array($path))->fetchRow();
if(is_array($result)){
return $result;
}else{
OC_Log::write('file not found in cache ('.$path.')','core',OC_Log::DEBUG);
return false;
}
}
/**
* put filesystem info in the cache
* @param string $path
* @param array data
*
* $data is an assiciative array in the same format as returned by get
*/
public static function put($path,$data){
$path=OC_Filesystem::getRoot().$path;
if($id=self::getFileId($path)!=-1){
self::update($id,$data);
}
if($path=='/'){
$parent=-1;
}else{
$parent=self::getFileId(dirname($path));
}
$mimePart=dirname($data['mimetype']);
$query=OC_DB::prepare('INSERT INTO *PREFIX*fscache(parent, name, path, size, mtime, ctime, mimetype, mimepart) VALUES(?,?,?,?,?,?,?,?)');
$query->execute(array($parent,basename($path),$path,$data['size'],$data['mtime'],$data['ctime'],$data['mimetype'],$mimePart));
}
/**
* update filesystem info of a file
* @param int $id
* @param array $data
*/
private static function update($id,$data){
$mimePart=dirname($data['mimetype']);
$query=OC_DB::prepare('UPDATE *PREFIX*fscache SET size=? ,mtime=? ,ctime=? ,mimetype=? , mimepart=? WHERE id=?');
$query->execute(array($data['size'],$data['mtime'],$data['ctime'],$data['mimetype'],$mimePart,$id));
}
/**
* register a file move in the cache
* @param string oldPath
* @param string newPath
*/
public static function move($oldPath,$newPath){
$oldPath=OC_Filesystem::getRoot().$oldPath;
$newPath=OC_Filesystem::getRoot().$newPath;
$newParent=self::getParentId($newPath);
$query=OC_DB::prepare('UPDATE *PREFIX*fscache SET parent=? ,name=?, path=? WHERE path=?');
$query->execute(array($newParent,basename($newPath),$newPath,$oldPath));
}
/**
* delete info from the cache
* @param string $path
*/
public static function delete($path){
$path=OC_Filesystem::getRoot().$path;
$query=OC_DB::prepare('DELETE FROM *PREFIX*fscache WHERE path=?');
$query->execute(array($path));
}
/**
* return array of filenames matching the querty
* @param string $query
* @return array of filepaths
*/
public static function search($query){
$query=OC_DB::prepare('SELECT path FROM *PREFIX*fscache WHERE name LIKE ?');
$result=$query->execute(array("%$query%"));
$names=array();
while($row=$result->fetchRow()){
$names[]=$row['path'];
}
return $names;
}
/**
* get all files and folders in a folder
* @param string path
* @return array
*
* returns an array of assiciative arrays with the following keys:
* - name
* - size
* - mtime
* - ctime
* - mimetype
*/
public static function getFolderContent($path){
$path=OC_Filesystem::getRoot().$path;
$parent=self::getFileId($path);
$query=OC_DB::prepare('SELECT name,ctime,mtime,mimetype,size FROM *PREFIX*fscache WHERE parent=?');
$result=$query->execute(array($parent))->fetchAll();
if(is_array($result)){
return $result;
}else{
OC_Log::write('file not found in cache ('.$path.')','core',OC_Log::DEBUG);
return false;
}
}
/**
* check if a file or folder is in the cache
* @param string $path
* @return bool
*/
public static function inCache($path){
$path=OC_Filesystem::getRoot().$path;
$inCache=self::getFileId($path)!=-1;
return $inCache;
}
/**
* get the file id as used in the cache
* @param string $path
* @return int
*/
private static function getFileId($path){
$query=OC_DB::prepare('SELECT id FROM *PREFIX*fscache WHERE path=?');
$result=$query->execute(array($path))->fetchRow();
if(is_array($result)){
return $result['id'];
}else{
OC_Log::write('file not found in cache ('.$path.')','core',OC_Log::DEBUG);
return -1;
}
}
/**
* get the file id of the parent folder, taking into account '/' has no parent
* @param string $path
* @return int
*/
private static function getParentId($path){
if($path=='/'){
return -1;
}else{
return self::getFileId(dirname($path));
}
}
/**
* called when changes are made to files
*/
public static function fileSystemWatcherWrite($params){
$path=$params['path'];
$fullPath=OC_Filesystem::getRoot().$path;
$mimetype=OC_Filesystem::getMimeType($path);
if($mimetype=='httpd/unix-directory'){
$size=0;
}else{
if(($id=self::getFileId($fullPath))!=-1){
$oldInfo=self::get($fullPath);
$oldSize=$oldInfo['size'];
}else{
$oldSize=0;
}
$size=OC_Filesystem::filesize($path);
self::increaseSize(dirname($fullPath),$size-$oldSize);
}
$mtime=OC_Filesystem::filemtime($path);
$ctime=OC_Filesystem::filectime($path);
self::put($path,array('size'=>$size,'mtime'=>$mtime,'ctime'=>$ctime,'mimetype'=>$mimetype));
}
/**
* called when files are deleted
*/
public static function fileSystemWatcherDelete($params){
$path=$params['path'];
$fullPath=OC_Filesystem::getRoot().$path;
error_log("delete $path");
if(self::getFileId($fullPath)==-1){
return;
}
$size=OC_Filesystem::filesize($path);
self::increaseSize(dirname($fullPath),-$size);
self::delete($path);
}
/**
* called when files are deleted
*/
public static function fileSystemWatcherRename($params){
$oldPath=$params['oldpath'];
$newPath=$params['newpath'];
$fullOldPath=OC_Filesystem::getRoot().$oldPath;
$fullNewPath=OC_Filesystem::getRoot().$newPath;
if(($id=self::getFileId($fullOldPath))!=-1){
$oldInfo=self::get($fullOldPath);
$oldSize=$oldInfo['size'];
}else{
return;
}
$size=OC_Filesystem::filesize($oldPath);
self::increaseSize(dirname($fullOldPath),-$oldSize);
self::increaseSize(dirname($fullNewPath),$oldSize);
self::move($oldPath,$newPath);
}
/**
* adjust the size of the parent folders
* @param string $path
* @param int $sizeDiff
*/
private static function increaseSize($path,$sizeDiff){
while(($id=self::getFileId($path))!=-1){
$query=OC_DB::prepare('UPDATE *PREFIX*fscache SET size=size+? WHERE id=?');
error_log('diff '.$path.' '.$sizeDiff);
$query->execute(array($sizeDiff,$id));
$path=dirname($path);
}
}
/**
* recursively scan the filesystem and fill the cache
* @param string $path
* @param bool $onlyChilds
*/
public static function scan($path,$onlyChilds=false){//PROBLEM due to the order things are added, all parents are -1
$dh=OC_Filesystem::opendir($path);
$stat=OC_Filesystem::stat($path);
$mimetype=OC_Filesystem::getMimeType($path);
$stat['mimetype']=$mimetype;
if($path=='/'){
$path='';
}
self::put($path,$stat);
$fullPath=OC_Filesystem::getRoot().$path;
$totalSize=0;
if($dh){
while (($filename = readdir($dh)) !== false) {
if($filename != '.' and $filename != '..'){
$file=$path.'/'.$filename;
if(OC_Filesystem::is_dir($file)){
self::scan($file,true);
}else{
$stat=OC_Filesystem::stat($file);
$mimetype=OC_Filesystem::getMimeType($file);
$stat['mimetype']=$mimetype;
self::put($file,$stat);
$totalSize+=$stat['size'];
}
}
}
}
self::increaseSize($fullPath,$totalSize);
}
/**
* fine files by mimetype
* @param string $part1
* @param string $part2 (optional)
* @return array of file paths
*
* $part1 and $part2 together form the complete mimetype.
* e.g. searchByMime('text','plain')
*
* seccond mimetype part can be ommited
* e.g. searchByMime('audio')
*/
public static function searchByMime($part1,$part2=''){
if($part2){
$query=OC_DB::prepare('SELECT path FROM *PREFIX*fscache WHERE mimepart=?');
$result=$query->execute(array($part1));
}else{
$query=OC_DB::prepare('SELECT path FROM *PREFIX*fscache WHERE mimetype=?');
$result=$query->execute(array($part1.'/'.$part2));
}
$names=array();
while($row=$result->fetchRow()){
$names[]=$row['path'];
}
return $names;
}
}
//watch for changes and try to keep the cache up to date
OC_Hook::connect('OC_Filesystem','post_write','OC_FileCache','fileSystemWatcherWrite');
OC_Hook::connect('OC_Filesystem','delete','OC_FileCache','fileSystemWatcherDelete');
OC_Hook::connect('OC_Filesystem','rename','OC_FileCache','fileSystemWatcherRename');

41
lib/files.php
View File

@ -36,44 +36,13 @@ class OC_Files {
if(strpos($directory,OC::$CONFIG_DATADIRECTORY)===0){
$directory=substr($directory,strlen(OC::$CONFIG_DATADIRECTORY));
}
$filesfound=true;
$content=array();
$dirs=array();
$file=array();
$files=array();
if(OC_Filesystem::is_dir($directory)) {
if ($dh = OC_Filesystem::opendir($directory)) {
while (($filename = readdir($dh)) !== false) {
if($filename<>'.' and $filename<>'..' and substr($filename,0,1)!='.'){
$file=array();
$filesfound=true;
$file['name']=$filename;
$file['directory']=$directory;
$stat=OC_Filesystem::stat($directory.'/'.$filename);
$file=array_merge($file,$stat);
$file['size']=OC_Filesystem::filesize($directory.'/'.$filename);
$file['mime']=OC_Files::getMimeType($directory .'/'. $filename);
$file['readable']=OC_Filesystem::is_readable($directory .'/'. $filename);
$file['writeable']=OC_Filesystem::is_writeable($directory .'/'. $filename);
$file['type']=OC_Filesystem::filetype($directory .'/'. $filename);
if($file['type']=='dir'){
$dirs[$file['name']]=$file;
}else{
$files[$file['name']]=$file;
}
}
}
closedir($dh);
}
$files=OC_FileCache::getFolderContent($directory);
foreach($files as &$file){
$file['directory']=$directory;
$file['type']=($file['mimetype']=='httpd/unix-directory')?'dir':'file';
}
uksort($dirs, "strnatcasecmp");
uksort($files, "strnatcasecmp");
$content=array_merge($dirs,$files);
if($filesfound){
return $content;
}else{
return false;
}
return $files;
}

83
lib/filestorage/local.php
View File

@ -13,13 +13,11 @@ class OC_Filestorage_Local extends OC_Filestorage{
}
public function mkdir($path){
if($return=mkdir($this->datadir.$path)){
$this->clearFolderSizeCache($path);
}
return $return;
}
public function rmdir($path){
if($return=rmdir($this->datadir.$path)){
$this->clearFolderSizeCache($path);
}
return $return;
}
@ -72,12 +70,10 @@ class OC_Filestorage_Local extends OC_Filestorage{
}
public function file_put_contents($path,$data){
if($return=file_put_contents($this->datadir.$path,$data)){
$this->clearFolderSizeCache($path);
}
}
public function unlink($path){
$return=$this->delTree($path);
$this->clearFolderSizeCache($path);
return $return;
}
public function rename($path1,$path2){
@ -87,8 +83,6 @@ class OC_Filestorage_Local extends OC_Filestorage{
}
if($return=rename($this->datadir.$path1,$this->datadir.$path2)){
$this->clearFolderSizeCache($path1);
$this->clearFolderSizeCache($path2);
}
return $return;
}
@ -101,7 +95,6 @@ class OC_Filestorage_Local extends OC_Filestorage{
$path2.=$source;
}
if($return=copy($this->datadir.$path1,$this->datadir.$path2)){
$this->clearFolderSizeCache($path2);
}
return $return;
}
@ -114,12 +107,10 @@ class OC_Filestorage_Local extends OC_Filestorage{
case 'w+':
case 'x+':
case 'a+':
$this->clearFolderSizeCache($path);
break;
case 'w':
case 'x':
case 'a':
$this->clearFolderSizeCache($path);
break;
}
}
@ -182,7 +173,6 @@ class OC_Filestorage_Local extends OC_Filestorage{
$fileStats = stat($tmpFile);
if(rename($tmpFile,$this->datadir.$path)){
touch($this->datadir.$path, $fileStats['mtime'], $fileStats['atime']);
$this->clearFolderSizeCache($path);
return true;
}else{
return false;
@ -198,7 +188,6 @@ class OC_Filestorage_Local extends OC_Filestorage{
if ($item == '.' || $item == '..') continue;
if(is_file($dir.'/'.$item)){
if(unlink($dir.'/'.$item)){
$this->clearFolderSizeCache($dir);
}
}elseif(is_dir($dir.'/'.$item)){
if (!$this->delTree($dirRelative. "/" . $item)){
@ -207,7 +196,6 @@ class OC_Filestorage_Local extends OC_Filestorage{
}
}
if($return=rmdir($dir)){
$this->clearFolderSizeCache($dir);
}
return $return;
}
@ -247,75 +235,6 @@ class OC_Filestorage_Local extends OC_Filestorage{
* @return int size of folder and it's content
*/
public function getFolderSize($path){
$path=str_replace('//','/',$path);
if($this->is_dir($path) and substr($path,-1)!='/'){
$path.='/';
}
$query=OC_DB::prepare("SELECT size FROM *PREFIX*foldersize WHERE path=?");
$size=$query->execute(array($path))->fetchAll();
if(count($size)>0){// we already the size, just return it
return $size[0]['size'];
}else{//the size of the folder isn't know, calulate it
return $this->calculateFolderSize($path);
}
}
/**
* @brief calulate the size of folder and it's content and cache it
* @param string $path file path
* @return int size of folder and it's content
*/
public function calculateFolderSize($path){
if($this->is_file($path)){
$path=dirname($path);
}
$path=str_replace('//','/',$path);
if($this->is_dir($path) and substr($path,-1)!='/'){
$path.='/';
}
$size=0;
if ($dh = $this->opendir($path)) {
while (($filename = readdir($dh)) !== false) {
if($filename!='.' and $filename!='..'){
$subFile=$path.'/'.$filename;
if($this->is_file($subFile)){
$size+=$this->filesize($subFile);
}else{
$size+=$this->getFolderSize($subFile);
}
}
}
if($size>0){
$query=OC_DB::prepare("INSERT INTO *PREFIX*foldersize VALUES(?,?)");
$result=$query->execute(array($path,$size));
}
}
return $size;
}
/**
* @brief clear the folder size cache of folders containing a file
* @param string $path
*/
public function clearFolderSizeCache($path){
if($this->is_file($path)){
$path=dirname($path);
}
$path=str_replace('//','/',$path);
if($this->is_dir($path) and substr($path,-1)!='/'){
$path.='/';
}
$query=OC_DB::prepare("DELETE FROM *PREFIX*foldersize WHERE path = ?");
$result=$query->execute(array($path));
if($path!='/' and $path!=''){
$parts=explode('/',$path);
//pop empty part
$part=array_pop($parts);
if(empty($part)){
array_pop($parts);
}
$parent=implode('/',$parts);
$this->clearFolderSizeCache($parent);
}
return 0;//depricated, use OC_FileCach instead
}
}

9
lib/filesystem.php
View File

@ -42,6 +42,7 @@
*
* the &run parameter can be set to false to prevent the operation from occuring
*/
class OC_Filesystem{
static private $storages=array();
static private $mounts=array();
@ -84,6 +85,14 @@ class OC_Filesystem{
}
self::$fakeRoot=$fakeRoot;
}
/**
* get the fake root
* @return string
*/
static public function getRoot(){
return self::$fakeRoot;
}
/**
* get the part of the path relative to the mountpoint of the storage it's stored in

5
lib/util.php
View File

@ -49,6 +49,11 @@ class OC_Util {
$quotaProxy=new OC_FileProxy_Quota();
OC_FileProxy::register($quotaProxy);
self::$fsSetup=true;
//create the file cache if necesary
if(!OC_FileCache::inCache('')){
OC_FileCache::scan('');
}
}
}