From 7ef937d8ad33ed2cd3e452bcde386e9d125e6a54 Mon Sep 17 00:00:00 2001 From: Victor Dubiniuk Date: Mon, 17 Aug 2015 21:40:03 +0300 Subject: [PATCH] Add versions obligation --- apps/files_versions/appinfo/application.php | 11 ++ apps/files_versions/lib/expiration.php | 160 ++++++++++++++++ apps/files_versions/lib/storage.php | 42 +++- apps/files_versions/tests/expiration.php | 200 ++++++++++++++++++++ config/config.sample.php | 28 +++ 5 files changed, 437 insertions(+), 4 deletions(-) create mode 100644 apps/files_versions/lib/expiration.php create mode 100644 apps/files_versions/tests/expiration.php diff --git a/apps/files_versions/appinfo/application.php b/apps/files_versions/appinfo/application.php index bab36b4851..b61b03dab1 100644 --- a/apps/files_versions/appinfo/application.php +++ b/apps/files_versions/appinfo/application.php @@ -22,6 +22,7 @@ namespace OCA\Files_Versions\AppInfo; use OCP\AppFramework\App; +use OCA\Files_Versions\Expiration; class Application extends App { public function __construct(array $urlParams = array()) { @@ -33,5 +34,15 @@ class Application extends App { * Register capabilities */ $container->registerCapability('OCA\Files_Versions\Capabilities'); + + /* + * Register expiration + */ + $container->registerService('Expiration', function($c) { + return new Expiration( + $c->query('ServerContainer')->getConfig(), + $c->query('OCP\AppFramework\Utility\ITimeFactory') + ); + }); } } diff --git a/apps/files_versions/lib/expiration.php b/apps/files_versions/lib/expiration.php new file mode 100644 index 0000000000..e1c7f50a20 --- /dev/null +++ b/apps/files_versions/lib/expiration.php @@ -0,0 +1,160 @@ + + * + * @copyright Copyright (c) 2015, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\Files_Versions; + +use \OCP\IConfig; +use \OCP\AppFramework\Utility\ITimeFactory; + +class Expiration { + + // how long do we keep files a version if no other value is defined in the config file (unit: days) + const DEFAULT_RETENTION_OBLIGATION = 30; + const NO_OBLIGATION = -1; + + /** @var ITimeFactory */ + private $timeFactory; + + /** @var string */ + private $retentionObligation; + + /** @var int */ + private $minAge; + + /** @var int */ + private $maxAge; + + /** @var bool */ + private $canPurgeToSaveSpace; + + public function __construct(IConfig $config,ITimeFactory $timeFactory){ + $this->timeFactory = $timeFactory; + $this->retentionObligation = $config->getSystemValue('versions_retention_obligation', 'auto'); + + if ($this->retentionObligation !== 'disabled') { + $this->parseRetentionObligation(); + } + } + + /** + * Is versions expiration enabled + * @return bool + */ + public function isEnabled(){ + return $this->retentionObligation !== 'disabled'; + } + + /** + * Is default expiration active + */ + public function shouldAutoExpire(){ + return $this->minAge === self::NO_OBLIGATION + || $this->maxAge === self::NO_OBLIGATION; + } + + /** + * Check if given timestamp in expiration range + * @param int $timestamp + * @param bool $quotaExceeded + * @return bool + */ + public function isExpired($timestamp, $quotaExceeded = false){ + // No expiration if disabled + if (!$this->isEnabled()) { + return false; + } + + // Purge to save space (if allowed) + if ($quotaExceeded && $this->canPurgeToSaveSpace) { + return true; + } + + $time = $this->timeFactory->getTime(); + // Never expire dates in future e.g. misconfiguration or negative time + // adjustment + if ($time<$timestamp) { + return false; + } + + // Purge as too old + if ($this->maxAge !== self::NO_OBLIGATION) { + $maxTimestamp = $time - ($this->maxAge * 86400); + $isOlderThanMax = $timestamp < $maxTimestamp; + } else { + $isOlderThanMax = false; + } + + if ($this->minAge !== self::NO_OBLIGATION) { + // older than Min obligation and we are running out of quota? + $minTimestamp = $time - ($this->minAge * 86400); + $isMinReached = ($timestamp < $minTimestamp) && $quotaExceeded; + } else { + $isMinReached = false; + } + + return $isOlderThanMax || $isMinReached; + } + + private function parseRetentionObligation(){ + $splitValues = explode(',', $this->retentionObligation); + if (!isset($splitValues[0])) { + $minValue = self::DEFAULT_RETENTION_OBLIGATION; + } else { + $minValue = trim($splitValues[0]); + } + + if (!isset($splitValues[1]) && $minValue === 'auto') { + $maxValue = 'auto'; + } elseif (!isset($splitValues[1])) { + $maxValue = self::DEFAULT_RETENTION_OBLIGATION; + } else { + $maxValue = trim($splitValues[1]); + } + + if ($minValue === 'auto' && $maxValue === 'auto') { + // Default: Keep for 30 days but delete anytime if space needed + $this->minAge = self::DEFAULT_RETENTION_OBLIGATION; + $this->maxAge = self::NO_OBLIGATION; + $this->canPurgeToSaveSpace = true; + } elseif ($minValue !== 'auto' && $maxValue === 'auto') { + // Keep for X days but delete anytime if space needed + $this->minAge = intval($minValue); + $this->maxAge = self::NO_OBLIGATION; + $this->canPurgeToSaveSpace = true; + } elseif ($minValue === 'auto' && $maxValue !== 'auto') { + // Delete anytime if space needed, Delete all older than max automatically + $this->minAge = self::NO_OBLIGATION; + $this->maxAge = intval($maxValue); + $this->canPurgeToSaveSpace = true; + } elseif ($minValue !== 'auto' && $maxValue !== 'auto') { + // Delete all older than max OR older than min if space needed + + // Max < Min as per https://github.com/owncloud/core/issues/16301 + if ($maxValue < $minValue) { + $maxValue = $minValue; + } + + $this->minAge = intval($minValue); + $this->maxAge = intval($maxValue); + $this->canPurgeToSaveSpace = false; + } + } +} diff --git a/apps/files_versions/lib/storage.php b/apps/files_versions/lib/storage.php index e0034f6165..b190ea307e 100644 --- a/apps/files_versions/lib/storage.php +++ b/apps/files_versions/lib/storage.php @@ -40,6 +40,7 @@ namespace OCA\Files_Versions; +use OCA\Files_Versions\AppInfo\Application; use OCA\Files_Versions\Command\Expire; class Storage { @@ -482,7 +483,33 @@ class Storage { * @return array containing the list of to deleted versions and the size of them */ protected static function getExpireList($time, $versions) { + $application = new Application(); + $expiration = $application->getContainer()->query('Expiration'); + if ($expiration->shouldAutoExpire()) { + return self::getAutoExpireList($time, $versions); + } + + $size = 0; + $toDelete = []; // versions we want to delete + + foreach ($versions as $key => $version) { + if ($expiration->isExpired($version['version'])) { + $toDelete[$key] = $version['path'] . '.v' . $version['version']; + $size += $version['size']; + } + } + + return array($toDelete, $size); + } + + /** + * get list of files we want to expire + * @param array $versions list of versions + * @param integer $time + * @return array containing the list of to deleted versions and the size of them + */ + protected static function getAutoExpireList($time, $versions) { $size = 0; $toDelete = array(); // versions we want to delete @@ -529,7 +556,6 @@ class Storage { } return array($toDelete, $size); - } /** @@ -541,8 +567,13 @@ class Storage { * @param int $neededSpace requested versions size */ private static function scheduleExpire($uid, $fileName, $versionsSize = null, $neededSpace = 0) { - $command = new Expire($uid, $fileName, $versionsSize, $neededSpace); - \OC::$server->getCommandBus()->push($command); + // let the admin disable auto expire + $application = new Application(); + $expiration = $application->getContainer()->query('Expiration'); + if ($expiration->isEnabled()) { + $command = new Expire($uid, $fileName, $versionsSize, $neededSpace); + \OC::$server->getCommandBus()->push($command); + } } /** @@ -555,7 +586,10 @@ class Storage { */ public static function expire($filename, $versionsSize = null, $offset = 0) { $config = \OC::$server->getConfig(); - if($config->getSystemValue('files_versions', Storage::DEFAULTENABLED)=='true') { + $application = new Application(); + $expiration = $application->getContainer()->query('Expiration'); + + if($config->getSystemValue('files_versions', Storage::DEFAULTENABLED)=='true' && $expiration->isEnabled()) { list($uid, $filename) = self::getUidAndFilename($filename); if (empty($filename)) { // file maybe renamed or deleted diff --git a/apps/files_versions/tests/expiration.php b/apps/files_versions/tests/expiration.php new file mode 100644 index 0000000000..44d911c74b --- /dev/null +++ b/apps/files_versions/tests/expiration.php @@ -0,0 +1,200 @@ + + * + * @copyright Copyright (c) 2015, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\Files_Versions; + +class Expiration_Test extends \PHPUnit_Framework_TestCase { + const SECONDS_PER_DAY = 86400; //60*60*24 + + public function expirationData(){ + $today = 100*self::SECONDS_PER_DAY; + $back10Days = (100-10)*self::SECONDS_PER_DAY; + $back20Days = (100-20)*self::SECONDS_PER_DAY; + $back30Days = (100-30)*self::SECONDS_PER_DAY; + $back35Days = (100-35)*self::SECONDS_PER_DAY; + + // it should never happen, but who knows :/ + $ahead100Days = (100+100)*self::SECONDS_PER_DAY; + + return [ + // Expiration is disabled - always should return false + [ 'disabled', $today, $back10Days, false, false], + [ 'disabled', $today, $back10Days, true, false], + [ 'disabled', $today, $ahead100Days, true, false], + + // Default: expire in 30 days or earlier when quota requirements are met + [ 'auto', $today, $back10Days, false, false], + [ 'auto', $today, $back35Days, false, false], + [ 'auto', $today, $back10Days, true, true], + [ 'auto', $today, $back35Days, true, true], + [ 'auto', $today, $ahead100Days, true, true], + + // The same with 'auto' + [ 'auto, auto', $today, $back10Days, false, false], + [ 'auto, auto', $today, $back35Days, false, false], + [ 'auto, auto', $today, $back10Days, true, true], + [ 'auto, auto', $today, $back35Days, true, true], + + // Keep for 15 days but expire anytime if space needed + [ '15, auto', $today, $back10Days, false, false], + [ '15, auto', $today, $back20Days, false, false], + [ '15, auto', $today, $back10Days, true, true], + [ '15, auto', $today, $back20Days, true, true], + [ '15, auto', $today, $ahead100Days, true, true], + + // Expire anytime if space needed, Expire all older than max + [ 'auto, 15', $today, $back10Days, false, false], + [ 'auto, 15', $today, $back20Days, false, true], + [ 'auto, 15', $today, $back10Days, true, true], + [ 'auto, 15', $today, $back20Days, true, true], + [ 'auto, 15', $today, $ahead100Days, true, true], + + // Expire all older than max OR older than min if space needed + [ '15, 25', $today, $back10Days, false, false], + [ '15, 25', $today, $back20Days, false, false], + [ '15, 25', $today, $back30Days, false, true], + [ '15, 25', $today, $back10Days, false, false], + [ '15, 25', $today, $back20Days, true, true], + [ '15, 25', $today, $back30Days, true, true], + [ '15, 25', $today, $ahead100Days, true, false], + + // Expire all older than max OR older than min if space needed + // MaxgetMockedConfig($retentionObligation); + $mockedTimeFactory = $this->getMockedTimeFactory($timeNow); + + $expiration = new Expiration($mockedConfig, $mockedTimeFactory); + $actualResult = $expiration->isExpired($timestamp, $quotaExceeded); + + $this->assertEquals($expectedResult, $actualResult); + } + + + public function configData(){ + return [ + [ 'disabled', null, null, null], + [ 'auto', Expiration::DEFAULT_RETENTION_OBLIGATION, Expiration::NO_OBLIGATION, true ], + [ 'auto,auto', Expiration::DEFAULT_RETENTION_OBLIGATION, Expiration::NO_OBLIGATION, true ], + [ 'auto, auto', Expiration::DEFAULT_RETENTION_OBLIGATION, Expiration::NO_OBLIGATION, true ], + [ 'auto, 3', Expiration::NO_OBLIGATION, 3, true ], + [ '5, auto', 5, Expiration::NO_OBLIGATION, true ], + [ '3, 5', 3, 5, false ], + [ '10, 3', 10, 10, false ], + ]; + } + + + /** + * @dataProvider configData + * + * @param string $configValue + * @param int $expectedMinAge + * @param int $expectedMaxAge + * @param bool $expectedCanPurgeToSaveSpace + */ + public function testParseRetentionObligation($configValue, $expectedMinAge, $expectedMaxAge, $expectedCanPurgeToSaveSpace){ + $mockedConfig = $this->getMockedConfig($configValue); + $mockedTimeFactory = $this->getMockedTimeFactory( + time() + ); + + $expiration = new Expiration($mockedConfig, $mockedTimeFactory); + $this->assertAttributeEquals($expectedMinAge, 'minAge', $expiration); + $this->assertAttributeEquals($expectedMaxAge, 'maxAge', $expiration); + $this->assertAttributeEquals($expectedCanPurgeToSaveSpace, 'canPurgeToSaveSpace', $expiration); + } + + /** + * + * @param int $time + * @return \OCP\AppFramework\Utility\ITimeFactory + */ + private function getMockedTimeFactory($time){ + $mockedTimeFactory = $this->getMockBuilder('\OCP\AppFramework\Utility\ITimeFactory') + ->disableOriginalConstructor() + ->setMethods(['getTime']) + ->getMock() + ; + $mockedTimeFactory->expects($this->any())->method('getTime')->will( + $this->returnValue($time) + ); + + return $mockedTimeFactory; + } + + /** + * + * @param string $returnValue + * @return \OCP\IConfig + */ + private function getMockedConfig($returnValue){ + $mockedConfig = $this->getMockBuilder('\OCP\IConfig') + ->disableOriginalConstructor() + ->setMethods( + [ + 'setSystemValues', + 'setSystemValue', + 'getSystemValue', + 'deleteSystemValue', + 'getAppKeys', + 'setAppValue', + 'getAppValue', + 'deleteAppValue', + 'deleteAppValues', + 'setUserValue', + 'getUserValue', + 'getUserValueForUsers', + 'getUserKeys', + 'deleteUserValue', + 'deleteAllUserValues', + 'deleteAppFromAllUsers', + 'getUsersForUserValue' + ] + ) + ->getMock() + ; + $mockedConfig->expects($this->any())->method('getSystemValue')->will( + $this->returnValue($returnValue) + ); + + return $mockedConfig; + } +} diff --git a/config/config.sample.php b/config/config.sample.php index a841831f01..a2889a57a7 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -434,6 +434,34 @@ $CONFIG = array( 'trashbin_retention_obligation' => 'auto', +/** + * If the versions app is enabled (default), this setting defines the policy + * for when versions will be permanently deleted. + * The app allows for two settings, a minimum time for version retention, + * and a maximum time for version retention. + * Minimum time is the number of days a version will be kept, after which it + * may be deleted. Maximum time is the number of days at which it is guaranteed + * to be deleted. + * Both minimum and maximum times can be set together to explicitly define + * version deletion. For migration purposes, this setting is installed + * initially set to "auto", which is equivalent to the default setting in + * ownCloud 8.1 and before. + * + * Available values: + * ``auto`` default setting. keeps versions for 30 days and automatically + * deletes anytime after that if space is needed (note: files + * may not be deleted if space is not needed). + * ``D, auto`` keeps versions for D+ days, delete anytime if space needed + * (note: files may not be deleted + * if space is not needed) + * * ``auto, D`` delete all versions that are older than D days automatically, + * delete other files anytime if space needed + * * ``D1, D2`` keep versions for at least D1 days and delete when exceeds D2 days + * ``disabled`` versions auto clean disabled, versions will be kept forever + */ +'versions_retention_obligation' => 'auto', + + /** * ownCloud Verifications *