* @author Bart Visscher * @author Björn Schießle * @author Felix Moeller * @author Florin Peter * @author Georg Ehrke * @author Joas Schilling * @author Jörn Friedrich Dreyer * @author Lukas Reschke * @author Morris Jobke * @author Robin Appelman * @author Robin McCorkell * @author Thomas Müller * @author Victor Dubiniuk * @author Vincent Petry * * @copyright Copyright (c) 2016, ownCloud, Inc. * @license AGPL-3.0 * * This code is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, * as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License, version 3, * along with this program. If not, see * */ /** * Versions * * A class to handle the versioning of files. */ namespace OCA\Files_Versions; use OC\Files\Filesystem; use OC\Files\View; use OCA\Files_Versions\AppInfo\Application; use OCA\Files_Versions\Command\Expire; use OCP\Lock\ILockingProvider; use OCP\User; class Storage { const DEFAULTENABLED=true; const DEFAULTMAXSIZE=50; // unit: percentage; 50% of available disk space/quota const VERSIONS_ROOT = 'files_versions/'; const DELETE_TRIGGER_MASTER_REMOVED = 0; const DELETE_TRIGGER_RETENTION_CONSTRAINT = 1; const DELETE_TRIGGER_QUOTA_EXCEEDED = 2; // files for which we can remove the versions after the delete operation was successful private static $deletedFiles = array(); private static $sourcePathAndUser = array(); private static $max_versions_per_interval = array( //first 10sec, one version every 2sec 1 => array('intervalEndsAfter' => 10, 'step' => 2), //next minute, one version every 10sec 2 => array('intervalEndsAfter' => 60, 'step' => 10), //next hour, one version every minute 3 => array('intervalEndsAfter' => 3600, 'step' => 60), //next 24h, one version every hour 4 => array('intervalEndsAfter' => 86400, 'step' => 3600), //next 30days, one version per day 5 => array('intervalEndsAfter' => 2592000, 'step' => 86400), //until the end one version per week 6 => array('intervalEndsAfter' => -1, 'step' => 604800), ); /** @var \OCA\Files_Versions\AppInfo\Application */ private static $application; /** * get the UID of the owner of the file and the path to the file relative to * owners files folder * * @param string $filename * @return array * @throws \OC\User\NoUserException */ public static function getUidAndFilename($filename) { $uid = Filesystem::getOwner($filename); $userManager = \OC::$server->getUserManager(); // if the user with the UID doesn't exists, e.g. because the UID points // to a remote user with a federated cloud ID we use the current logged-in // user. We need a valid local user to create the versions if (!$userManager->userExists($uid)) { $uid = User::getUser(); } Filesystem::initMountPoints($uid); if ( $uid != User::getUser() ) { $info = Filesystem::getFileInfo($filename); $ownerView = new View('/'.$uid.'/files'); try { $filename = $ownerView->getPath($info['fileid']); // make sure that the file name doesn't end with a trailing slash // can for example happen single files shared across servers $filename = rtrim($filename, '/'); } catch (NotFoundException $e) { $filename = null; } } return [$uid, $filename]; } /** * Remember the owner and the owner path of the source file * * @param string $source source path */ public static function setSourcePathAndUser($source) { list($uid, $path) = self::getUidAndFilename($source); self::$sourcePathAndUser[$source] = array('uid' => $uid, 'path' => $path); } /** * Gets the owner and the owner path from the source path * * @param string $source source path * @return array with user id and path */ public static function getSourcePathAndUser($source) { if (isset(self::$sourcePathAndUser[$source])) { $uid = self::$sourcePathAndUser[$source]['uid']; $path = self::$sourcePathAndUser[$source]['path']; unset(self::$sourcePathAndUser[$source]); } else { $uid = $path = false; } return array($uid, $path); } /** * get current size of all versions from a given user * * @param string $user user who owns the versions * @return int versions size */ private static function getVersionsSize($user) { $view = new View('/' . $user); $fileInfo = $view->getFileInfo('/files_versions'); return isset($fileInfo['size']) ? $fileInfo['size'] : 0; } /** * store a new version of a file. */ public static function store($filename) { if(\OCP\Config::getSystemValue('files_versions', Storage::DEFAULTENABLED)=='true') { // if the file gets streamed we need to remove the .part extension // to get the right target $ext = pathinfo($filename, PATHINFO_EXTENSION); if ($ext === 'part') { $filename = substr($filename, 0, strlen($filename) - 5); } // we only handle existing files if (! Filesystem::file_exists($filename) || Filesystem::is_dir($filename)) { return false; } list($uid, $filename) = self::getUidAndFilename($filename); $files_view = new View('/'.$uid .'/files'); $users_view = new View('/'.$uid); // no use making versions for empty files if ($files_view->filesize($filename) === 0) { return false; } // create all parent folders self::createMissingDirectories($filename, $users_view); self::scheduleExpire($uid, $filename); // store a new version of a file $mtime = $users_view->filemtime('files/' . $filename); $users_view->copy('files/' . $filename, 'files_versions/' . $filename . '.v' . $mtime); // call getFileInfo to enforce a file cache entry for the new version $users_view->getFileInfo('files_versions/' . $filename . '.v' . $mtime); } } /** * mark file as deleted so that we can remove the versions if the file is gone * @param string $path */ public static function markDeletedFile($path) { list($uid, $filename) = self::getUidAndFilename($path); self::$deletedFiles[$path] = array( 'uid' => $uid, 'filename' => $filename); } /** * delete the version from the storage and cache * * @param View $view * @param string $path */ protected static function deleteVersion($view, $path) { $view->unlink($path); /** * @var \OC\Files\Storage\Storage $storage * @var string $internalPath */ list($storage, $internalPath) = $view->resolvePath($path); $cache = $storage->getCache($internalPath); $cache->remove($internalPath); } /** * Delete versions of a file */ public static function delete($path) { $deletedFile = self::$deletedFiles[$path]; $uid = $deletedFile['uid']; $filename = $deletedFile['filename']; if (!Filesystem::file_exists($path)) { $view = new View('/' . $uid . '/files_versions'); $versions = self::getVersions($uid, $filename); if (!empty($versions)) { foreach ($versions as $v) { \OC_Hook::emit('\OCP\Versions', 'preDelete', array('path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED)); self::deleteVersion($view, $filename . '.v' . $v['version']); \OC_Hook::emit('\OCP\Versions', 'delete', array('path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED)); } } } unset(self::$deletedFiles[$path]); } /** * Rename or copy versions of a file of the given paths * * @param string $sourcePath source path of the file to move, relative to * the currently logged in user's "files" folder * @param string $targetPath target path of the file to move, relative to * the currently logged in user's "files" folder * @param string $operation can be 'copy' or 'rename' */ public static function renameOrCopy($sourcePath, $targetPath, $operation) { list($sourceOwner, $sourcePath) = self::getSourcePathAndUser($sourcePath); // it was a upload of a existing file if no old path exists // in this case the pre-hook already called the store method and we can // stop here if ($sourcePath === false) { return true; } list($targetOwner, $targetPath) = self::getUidAndFilename($targetPath); $sourcePath = ltrim($sourcePath, '/'); $targetPath = ltrim($targetPath, '/'); $rootView = new View(''); // did we move a directory ? if ($rootView->is_dir('/' . $targetOwner . '/files/' . $targetPath)) { // does the directory exists for versions too ? if ($rootView->is_dir('/' . $sourceOwner . '/files_versions/' . $sourcePath)) { // create missing dirs if necessary self::createMissingDirectories($targetPath, new View('/'. $targetOwner)); // move the directory containing the versions $rootView->$operation( '/' . $sourceOwner . '/files_versions/' . $sourcePath, '/' . $targetOwner . '/files_versions/' . $targetPath ); } } else if ($versions = Storage::getVersions($sourceOwner, '/' . $sourcePath)) { // create missing dirs if necessary self::createMissingDirectories($targetPath, new View('/'. $targetOwner)); foreach ($versions as $v) { // move each version one by one to the target directory $rootView->$operation( '/' . $sourceOwner . '/files_versions/' . $sourcePath.'.v' . $v['version'], '/' . $targetOwner . '/files_versions/' . $targetPath.'.v'.$v['version'] ); } } // if we moved versions directly for a file, schedule expiration check for that file if (!$rootView->is_dir('/' . $targetOwner . '/files/' . $targetPath)) { self::scheduleExpire($targetOwner, $targetPath); } } /** * Rollback to an old version of a file. * * @param string $file file name * @param int $revision revision timestamp */ public static function rollback($file, $revision) { if(\OCP\Config::getSystemValue('files_versions', Storage::DEFAULTENABLED)=='true') { // add expected leading slash $file = '/' . ltrim($file, '/'); list($uid, $filename) = self::getUidAndFilename($file); $users_view = new View('/'.$uid); $files_view = new View('/'. User::getUser().'/files'); $versionCreated = false; //first create a new version $version = 'files_versions'.$filename.'.v'.$users_view->filemtime('files'.$filename); if (!$users_view->file_exists($version)) { $users_view->copy('files'.$filename, 'files_versions'.$filename.'.v'.$users_view->filemtime('files'.$filename)); $versionCreated = true; } $fileToRestore = 'files_versions' . $filename . '.v' . $revision; // Restore encrypted version of the old file for the newly restored file // This has to happen manually here since the file is manually copied below $oldVersion = $users_view->getFileInfo($fileToRestore)->getEncryptedVersion(); $newFileInfo = $files_view->getFileInfo($filename); $cache = $newFileInfo->getStorage()->getCache(); $cache->update($newFileInfo->getId(), ['encrypted' => $oldVersion, 'encryptedVersion' => $oldVersion]); // rollback if (self::copyFileContents($users_view, $fileToRestore, 'files' . $filename)) { $files_view->touch($file, $revision); Storage::scheduleExpire($uid, $file); \OC_Hook::emit('\OCP\Versions', 'rollback', array( 'path' => $filename, 'revision' => $revision, )); return true; } else if ($versionCreated) { self::deleteVersion($users_view, $version); } } return false; } /** * Stream copy file contents from $path1 to $path2 * * @param View $view view to use for copying * @param string $path1 source file to copy * @param string $path2 target file * * @return bool true for success, false otherwise */ private static function copyFileContents($view, $path1, $path2) { /** @var \OC\Files\Storage\Storage $storage1 */ list($storage1, $internalPath1) = $view->resolvePath($path1); /** @var \OC\Files\Storage\Storage $storage2 */ list($storage2, $internalPath2) = $view->resolvePath($path2); $view->lockFile($path1, ILockingProvider::LOCK_EXCLUSIVE); $view->lockFile($path2, ILockingProvider::LOCK_EXCLUSIVE); // TODO add a proper way of overwriting a file while maintaining file ids if ($storage1->instanceOfStorage('\OC\Files\ObjectStore\ObjectStoreStorage') || $storage2->instanceOfStorage('\OC\Files\ObjectStore\ObjectStoreStorage')) { $source = $storage1->fopen($internalPath1, 'r'); $target = $storage2->fopen($internalPath2, 'w'); list(, $result) = \OC_Helper::streamCopy($source, $target); fclose($source); fclose($target); if ($result !== false) { $storage1->unlink($internalPath1); } } else { $result = $storage2->moveFromStorage($storage1, $internalPath1, $internalPath2); } $view->unlockFile($path1, ILockingProvider::LOCK_EXCLUSIVE); $view->unlockFile($path2, ILockingProvider::LOCK_EXCLUSIVE); return ($result !== false); } /** * get a list of all available versions of a file in descending chronological order * @param string $uid user id from the owner of the file * @param string $filename file to find versions of, relative to the user files dir * @param string $userFullPath * @return array versions newest version first */ public static function getVersions($uid, $filename, $userFullPath = '') { $versions = array(); if (empty($filename)) { return $versions; } // fetch for old versions $view = new View('/' . $uid . '/'); $pathinfo = pathinfo($filename); $versionedFile = $pathinfo['basename']; $dir = Filesystem::normalizePath(self::VERSIONS_ROOT . '/' . $pathinfo['dirname']); $dirContent = false; if ($view->is_dir($dir)) { $dirContent = $view->opendir($dir); } if ($dirContent === false) { return $versions; } if (is_resource($dirContent)) { while (($entryName = readdir($dirContent)) !== false) { if (!Filesystem::isIgnoredDir($entryName)) { $pathparts = pathinfo($entryName); $filename = $pathparts['filename']; if ($filename === $versionedFile) { $pathparts = pathinfo($entryName); $timestamp = substr($pathparts['extension'], 1); $filename = $pathparts['filename']; $key = $timestamp . '#' . $filename; $versions[$key]['version'] = $timestamp; $versions[$key]['humanReadableTimestamp'] = self::getHumanReadableTimestamp($timestamp); if (empty($userFullPath)) { $versions[$key]['preview'] = ''; } else { $versions[$key]['preview'] = \OCP\Util::linkToRoute('core_ajax_versions_preview', array('file' => $userFullPath, 'version' => $timestamp)); } $versions[$key]['path'] = Filesystem::normalizePath($pathinfo['dirname'] . '/' . $filename); $versions[$key]['name'] = $versionedFile; $versions[$key]['size'] = $view->filesize($dir . '/' . $entryName); } } } closedir($dirContent); } // sort with newest version first krsort($versions); return $versions; } /** * Expire versions that older than max version retention time * @param string $uid */ public static function expireOlderThanMaxForUser($uid){ $expiration = self::getExpiration(); $threshold = $expiration->getMaxAgeAsTimestamp(); $versions = self::getAllVersions($uid); if (!$threshold || !array_key_exists('all', $versions)) { return; } $toDelete = []; foreach (array_reverse($versions['all']) as $key => $version) { if (intval($version['version'])<$threshold) { $toDelete[$key] = $version; } else { //Versions are sorted by time - nothing mo to iterate. break; } } $view = new View('/' . $uid . '/files_versions'); if (!empty($toDelete)) { foreach ($toDelete as $version) { \OC_Hook::emit('\OCP\Versions', 'preDelete', array('path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT)); self::deleteVersion($view, $version['path'] . '.v' . $version['version']); \OC_Hook::emit('\OCP\Versions', 'delete', array('path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT)); } } } /** * translate a timestamp into a string like "5 days ago" * @param int $timestamp * @return string for example "5 days ago" */ private static function getHumanReadableTimestamp($timestamp) { $diff = time() - $timestamp; if ($diff < 60) { // first minute return $diff . " seconds ago"; } elseif ($diff < 3600) { //first hour return round($diff / 60) . " minutes ago"; } elseif ($diff < 86400) { // first day return round($diff / 3600) . " hours ago"; } elseif ($diff < 604800) { //first week return round($diff / 86400) . " days ago"; } elseif ($diff < 2419200) { //first month return round($diff / 604800) . " weeks ago"; } elseif ($diff < 29030400) { // first year return round($diff / 2419200) . " months ago"; } else { return round($diff / 29030400) . " years ago"; } } /** * returns all stored file versions from a given user * @param string $uid id of the user * @return array with contains two arrays 'all' which contains all versions sorted by age and 'by_file' which contains all versions sorted by filename */ private static function getAllVersions($uid) { $view = new View('/' . $uid . '/'); $dirs = array(self::VERSIONS_ROOT); $versions = array(); while (!empty($dirs)) { $dir = array_pop($dirs); $files = $view->getDirectoryContent($dir); foreach ($files as $file) { if ($file['type'] === 'dir') { array_push($dirs, $file['path']); } else { $versionsBegin = strrpos($file['path'], '.v'); $relPathStart = strlen(self::VERSIONS_ROOT); $version = substr($file['path'], $versionsBegin + 2); $relpath = substr($file['path'], $relPathStart, $versionsBegin - $relPathStart); $key = $version . '#' . $relpath; $versions[$key] = array('path' => $relpath, 'timestamp' => $version); } } } // newest version first krsort($versions); $result = array(); foreach ($versions as $key => $value) { $size = $view->filesize(self::VERSIONS_ROOT.'/'.$value['path'].'.v'.$value['timestamp']); $filename = $value['path']; $result['all'][$key]['version'] = $value['timestamp']; $result['all'][$key]['path'] = $filename; $result['all'][$key]['size'] = $size; $result['by_file'][$filename][$key]['version'] = $value['timestamp']; $result['by_file'][$filename][$key]['path'] = $filename; $result['by_file'][$filename][$key]['size'] = $size; } return $result; } /** * get list of files we want to expire * @param array $versions list of versions * @param integer $time * @param bool $quotaExceeded is versions storage limit reached * @return array containing the list of to deleted versions and the size of them */ protected static function getExpireList($time, $versions, $quotaExceeded = false) { $expiration = self::getExpiration(); if ($expiration->shouldAutoExpire()) { list($toDelete, $size) = self::getAutoExpireList($time, $versions); } else { $size = 0; $toDelete = []; // versions we want to delete } foreach ($versions as $key => $version) { if ($expiration->isExpired($version['version'], $quotaExceeded) && !isset($toDelete[$key])) { $size += $version['size']; $toDelete[$key] = $version['path'] . '.v' . $version['version']; } } return [$toDelete, $size]; } /** * get list of files we want to expire * @param array $versions list of versions * @param integer $time * @return array containing the list of to deleted versions and the size of them */ protected static function getAutoExpireList($time, $versions) { $size = 0; $toDelete = array(); // versions we want to delete $interval = 1; $step = Storage::$max_versions_per_interval[$interval]['step']; if (Storage::$max_versions_per_interval[$interval]['intervalEndsAfter'] == -1) { $nextInterval = -1; } else { $nextInterval = $time - Storage::$max_versions_per_interval[$interval]['intervalEndsAfter']; } $firstVersion = reset($versions); $firstKey = key($versions); $prevTimestamp = $firstVersion['version']; $nextVersion = $firstVersion['version'] - $step; unset($versions[$firstKey]); foreach ($versions as $key => $version) { $newInterval = true; while ($newInterval) { if ($nextInterval == -1 || $prevTimestamp > $nextInterval) { if ($version['version'] > $nextVersion) { //distance between two version too small, mark to delete $toDelete[$key] = $version['path'] . '.v' . $version['version']; $size += $version['size']; \OCP\Util::writeLog('files_versions', 'Mark to expire '. $version['path'] .' next version should be ' . $nextVersion . " or smaller. (prevTimestamp: " . $prevTimestamp . "; step: " . $step, \OCP\Util::DEBUG); } else { $nextVersion = $version['version'] - $step; $prevTimestamp = $version['version']; } $newInterval = false; // version checked so we can move to the next one } else { // time to move on to the next interval $interval++; $step = Storage::$max_versions_per_interval[$interval]['step']; $nextVersion = $prevTimestamp - $step; if (Storage::$max_versions_per_interval[$interval]['intervalEndsAfter'] == -1) { $nextInterval = -1; } else { $nextInterval = $time - Storage::$max_versions_per_interval[$interval]['intervalEndsAfter']; } $newInterval = true; // we changed the interval -> check same version with new interval } } } return array($toDelete, $size); } /** * Schedule versions expiration for the given file * * @param string $uid owner of the file * @param string $fileName file/folder for which to schedule expiration */ private static function scheduleExpire($uid, $fileName) { // let the admin disable auto expire $expiration = self::getExpiration(); if ($expiration->isEnabled()) { $command = new Expire($uid, $fileName); \OC::$server->getCommandBus()->push($command); } } /** * Expire versions which exceed the quota * * @param string $filename * @return bool|int|null */ public static function expire($filename) { $config = \OC::$server->getConfig(); $expiration = self::getExpiration(); if($config->getSystemValue('files_versions', Storage::DEFAULTENABLED)=='true' && $expiration->isEnabled()) { if (!Filesystem::file_exists($filename)) { return false; } list($uid, $filename) = self::getUidAndFilename($filename); if (empty($filename)) { // file maybe renamed or deleted return false; } $versionsFileview = new View('/'.$uid.'/files_versions'); // get available disk space for user $user = \OC::$server->getUserManager()->get($uid); $softQuota = true; $quota = $user->getQuota(); if ( $quota === null || $quota === 'none' ) { $quota = Filesystem::free_space('/'); $softQuota = false; } else { $quota = \OCP\Util::computerFileSize($quota); } // make sure that we have the current size of the version history $versionsSize = self::getVersionsSize($uid); // calculate available space for version history // subtract size of files and current versions size from quota if ($quota >= 0) { if ($softQuota) { $files_view = new View('/' . $uid . '/files'); $rootInfo = $files_view->getFileInfo('/', false); $free = $quota - $rootInfo['size']; // remaining free space for user if ($free > 0) { $availableSpace = ($free * self::DEFAULTMAXSIZE / 100) - $versionsSize; // how much space can be used for versions } else { $availableSpace = $free - $versionsSize; } } else { $availableSpace = $quota; } } else { $availableSpace = PHP_INT_MAX; } $allVersions = Storage::getVersions($uid, $filename); $time = time(); list($toDelete, $sizeOfDeletedVersions) = self::getExpireList($time, $allVersions, $availableSpace <= 0); $availableSpace = $availableSpace + $sizeOfDeletedVersions; $versionsSize = $versionsSize - $sizeOfDeletedVersions; // if still not enough free space we rearrange the versions from all files if ($availableSpace <= 0) { $result = Storage::getAllVersions($uid); $allVersions = $result['all']; foreach ($result['by_file'] as $versions) { list($toDeleteNew, $size) = self::getExpireList($time, $versions, $availableSpace <= 0); $toDelete = array_merge($toDelete, $toDeleteNew); $sizeOfDeletedVersions += $size; } $availableSpace = $availableSpace + $sizeOfDeletedVersions; $versionsSize = $versionsSize - $sizeOfDeletedVersions; } foreach($toDelete as $key => $path) { \OC_Hook::emit('\OCP\Versions', 'preDelete', array('path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED)); self::deleteVersion($versionsFileview, $path); \OC_Hook::emit('\OCP\Versions', 'delete', array('path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED)); unset($allVersions[$key]); // update array with the versions we keep \OCP\Util::writeLog('files_versions', "Expire: " . $path, \OCP\Util::DEBUG); } // Check if enough space is available after versions are rearranged. // If not we delete the oldest versions until we meet the size limit for versions, // but always keep the two latest versions $numOfVersions = count($allVersions) -2 ; $i = 0; // sort oldest first and make sure that we start at the first element ksort($allVersions); reset($allVersions); while ($availableSpace < 0 && $i < $numOfVersions) { $version = current($allVersions); \OC_Hook::emit('\OCP\Versions', 'preDelete', array('path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED)); self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']); \OC_Hook::emit('\OCP\Versions', 'delete', array('path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED)); \OCP\Util::writeLog('files_versions', 'running out of space! Delete oldest version: ' . $version['path'].'.v'.$version['version'] , \OCP\Util::DEBUG); $versionsSize -= $version['size']; $availableSpace += $version['size']; next($allVersions); $i++; } return $versionsSize; // finally return the new size of the version history } return false; } /** * Create recursively missing directories inside of files_versions * that match the given path to a file. * * @param string $filename $path to a file, relative to the user's * "files" folder * @param View $view view on data/user/ */ private static function createMissingDirectories($filename, $view) { $dirname = Filesystem::normalizePath(dirname($filename)); $dirParts = explode('/', $dirname); $dir = "/files_versions"; foreach ($dirParts as $part) { $dir = $dir . '/' . $part; if (!$view->file_exists($dir)) { $view->mkdir($dir); } } } /** * Static workaround * @return Expiration */ protected static function getExpiration(){ if (is_null(self::$application)) { self::$application = new Application(); } return self::$application->getContainer()->query('Expiration'); } }