diff --git a/lib/private/files/cache/storage.php b/lib/private/files/cache/storage.php index 9ad31a272e..a38656d849 100644 --- a/lib/private/files/cache/storage.php +++ b/lib/private/files/cache/storage.php @@ -28,9 +28,7 @@ class Storage { } else { $this->storageId = $storage; } - if (strlen($this->storageId) > 64) { - $this->storageId = md5($this->storageId); - } + $this->storageId = self::adjustStorageId($this->storageId); $sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?'; $result = \OC_DB::executeAudited($sql, array($this->storageId)); @@ -43,6 +41,19 @@ class Storage { } } + /** + * Adjusts the storage id to use md5 if too long + * @param string $storageId storage id + * @return unchanged $storageId if its length is less than 64 characters, + * else returns the md5 of $storageId + */ + public static function adjustStorageId($storageId) { + if (strlen($storageId) > 64) { + return md5($storageId); + } + return $storageId; + } + /** * @return string */ @@ -68,9 +79,7 @@ class Storage { * @return string|null */ public static function getNumericStorageId($storageId) { - if (strlen($storageId) > 64) { - $storageId = md5($storageId); - } + $storageId = self::adjustStorageId($storageId); $sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?'; $result = \OC_DB::executeAudited($sql, array($storageId)); @@ -95,9 +104,7 @@ class Storage { * @param string $storageId */ public static function remove($storageId) { - if (strlen($storageId) > 64) { - $storageId = md5($storageId); - } + $storageId = self::adjustStorageId($storageId); $sql = 'DELETE FROM `*PREFIX*storages` WHERE `id` = ?'; \OC_DB::executeAudited($sql, array($storageId)); diff --git a/lib/private/repair.php b/lib/private/repair.php index e6943c5d05..46b5ae4639 100644 --- a/lib/private/repair.php +++ b/lib/private/repair.php @@ -69,7 +69,8 @@ class Repair extends BasicEmitter { */ public static function getRepairSteps() { return array( - new \OC\Repair\RepairMimeTypes() + new \OC\Repair\RepairMimeTypes(), + new \OC\Repair\RepairLegacyStorages(\OC::$server->getConfig(), \OC_DB::getConnection()), ); } diff --git a/lib/private/repair/repairlegacystorages.php b/lib/private/repair/repairlegacystorages.php new file mode 100644 index 0000000000..9d38b256c8 --- /dev/null +++ b/lib/private/repair/repairlegacystorages.php @@ -0,0 +1,219 @@ + + * Copyright (c) 2014 Vincent Petry + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\Repair; + +use OC\Hooks\BasicEmitter; + +class RepairLegacyStorages extends BasicEmitter { + /** + * @var \OCP\IConfig + */ + protected $config; + + /** + * @var \OC\DB\Connection + */ + protected $connection; + + protected $findStorageInCacheStatement; + protected $renameStorageStatement; + + /** + * @param \OCP\IConfig $config + * @param \OC\DB\Connection $connection + */ + public function __construct($config, $connection) { + $this->connection = $connection; + $this->config = $config; + + $this->findStorageInCacheStatement = $this->connection->prepare( + 'SELECT DISTINCT `storage` FROM `*PREFIX*filecache`' + . ' WHERE `storage` in (?, ?)' + ); + $this->renameStorageStatement = $this->connection->prepare( + 'UPDATE `*PREFIX*storages`' + . ' SET `id` = ?' + . ' WHERE `id` = ?' + ); + } + + public function getName() { + return 'Repair legacy storages'; + } + + /** + * Extracts the user id from a legacy storage id + * + * @param string $storageId legacy storage id in the + * format "local::/path/to/datadir/userid" + * @return string user id extracted from the storage id + */ + private function extractUserId($storageId) { + $storageId = rtrim($storageId, '/'); + $pos = strrpos($storageId, '/'); + return substr($storageId, $pos + 1); + } + + /** + * Fix the given legacy storage by renaming the old id + * to the new id. If the new id already exists, whichever + * storage that has data in the file cache will be used. + * If both have data, nothing will be done and false is + * returned. + * + * @param string $oldId old storage id + * @param int $oldNumericId old storage numeric id + * + * @return bool true if fixed, false otherwise + */ + private function fixLegacyStorage($oldId, $oldNumericId, $userId = null) { + // check whether the new storage already exists + if (is_null($userId)) { + $userId = $this->extractUserId($oldId); + } + $newId = 'home::' . $userId; + + // check if target id already exists + $newNumericId = \OC\Files\Cache\Storage::getNumericStorageId($newId); + if (!is_null($newNumericId)) { + $newNumericId = (int)$newNumericId; + // try and resolve the conflict + // check which one of "local::" or "home::" needs to be kept + $result = $this->findStorageInCacheStatement->execute(array($oldNumericId, $newNumericId)); + $row1 = $this->findStorageInCacheStatement->fetch(); + $row2 = $this->findStorageInCacheStatement->fetch(); + if ($row2 !== false) { + // two results means both storages have data, not auto-fixable + throw new \OC\RepairException( + 'Could not automatically fix legacy storage ' + . '"' . $oldId . '" => "' . $newId . '"' + . ' because they both have data.' + ); + } + if ($row1 === false || (int)$row1['storage'] === $oldNumericId) { + // old storage has data, then delete the empty new id + $toDelete = $newId; + } else if ((int)$row1['storage'] === $newNumericId) { + // new storage has data, then delete the empty old id + $toDelete = $oldId; + } else { + // unknown case, do not continue + return false; + } + + // delete storage including file cache + \OC\Files\Cache\Storage::remove($toDelete); + + // if we deleted the old id, the new id will be used + // automatically + if ($toDelete === $oldId) { + // nothing more to do + return true; + } + } + + // rename old id to new id + $newId = \OC\Files\Cache\Storage::adjustStorageId($newId); + $oldId = \OC\Files\Cache\Storage::adjustStorageId($oldId); + $rowCount = $this->renameStorageStatement->execute(array($newId, $oldId)); + return ($rowCount === 1); + } + + /** + * Converts legacy home storage ids in the format + * "local::/data/dir/path/userid/" to the new format "home::userid" + */ + public function run() { + // only run once + if ($this->config->getAppValue('core', 'repairlegacystoragesdone') === 'yes') { + return; + } + + $dataDir = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/'); + $dataDir = rtrim($dataDir, '/') . '/'; + $dataDirId = 'local::' . $dataDir; + + $count = 0; + + $this->connection->beginTransaction(); + + try { + // note: not doing a direct UPDATE with the REPLACE function + // because regexp search/extract is needed and it is not guaranteed + // to work on all database types + $sql = 'SELECT `id`, `numeric_id` FROM `*PREFIX*storages`' + . ' WHERE `id` LIKE ?' + . ' ORDER BY `id`'; + $result = $this->connection->executeQuery($sql, array($dataDirId . '%')); + while ($row = $result->fetch()) { + $currentId = $row['id']; + // one entry is the datadir itself + if ($currentId === $dataDirId) { + continue; + } + + if ($this->fixLegacyStorage($currentId, (int)$row['numeric_id'])) { + $count++; + } + } + + // check for md5 ids, not in the format "prefix::" + $sql = 'SELECT COUNT(*) AS "c" FROM `*PREFIX*storages`' + . ' WHERE `id` NOT LIKE \'%::%\''; + $result = $this->connection->executeQuery($sql); + $row = $result->fetch(); + // find at least one to make sure it's worth + // querying the user list + if ((int)$row['c'] > 0) { + $userManager = \OC_User::getManager(); + + // use chunks to avoid caching too many users in memory + $limit = 30; + $offset = 0; + + do { + // query the next page of users + $results = $userManager->search('', $limit, $offset); + $storageIds = array(); + $userIds = array(); + foreach ($results as $uid => $userObject) { + $storageId = $dataDirId . $uid . '/'; + if (strlen($storageId) <= 64) { + // skip short storage ids as they were handled in the previous section + continue; + } + $storageIds[$uid] = $storageId; + } + + if (count($storageIds) > 0) { + // update the storages of these users + foreach ($storageIds as $uid => $storageId) { + $numericId = \OC\Files\Cache\Storage::getNumericStorageId($storageId); + if (!is_null($numericId) && $this->fixLegacyStorage($storageId, (int)$numericId)) { + $count++; + } + } + } + $offset += $limit; + } while (count($results) >= $limit); + } + + $this->emit('\OC\Repair', 'info', array('Updated ' . $count . ' legacy home storage ids')); + + $this->connection->commit(); + } + catch (\OC\RepairException $e) { + $this->connection->rollback(); + throw $e; + } + + $this->config->setAppValue('core', 'repairlegacystoragesdone', 'yes'); + } +} diff --git a/lib/private/repairexception.php b/lib/private/repairexception.php new file mode 100644 index 0000000000..642bf90ee4 --- /dev/null +++ b/lib/private/repairexception.php @@ -0,0 +1,16 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC; + +/** + * Exception thrown whenever a database/migration repair + * could not be done. + */ +class RepairException extends \Exception { +} diff --git a/tests/lib/repair/repairlegacystorage.php b/tests/lib/repair/repairlegacystorage.php new file mode 100644 index 0000000000..4528c5288d --- /dev/null +++ b/tests/lib/repair/repairlegacystorage.php @@ -0,0 +1,282 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +/** + * Tests for the converting of legacy storages to home storages. + * + * @see \OC\Repair\RepairLegacyStorages + */ +class TestRepairLegacyStorages extends PHPUnit_Framework_TestCase { + + private $user; + private $repair; + + private $dataDir; + private $oldDataDir; + + private $legacyStorageId; + private $newStorageId; + + public function setUp() { + $this->config = \OC::$server->getConfig(); + $this->connection = \OC_DB::getConnection(); + $this->oldDataDir = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/'); + + $this->repair = new \OC\Repair\RepairLegacyStorages($this->config, $this->connection); + } + + public function tearDown() { + \OC_User::deleteUser($this->user); + + $sql = 'DELETE FROM `*PREFIX*storages`'; + $this->connection->executeQuery($sql); + $sql = 'DELETE FROM `*PREFIX*filecache`'; + $this->connection->executeQuery($sql); + \OCP\Config::setSystemValue('datadirectory', $this->oldDataDir); + $this->config->setAppValue('core', 'repairlegacystoragesdone', 'no'); + } + + function prepareSettings($dataDir, $userId) { + // hard-coded string as we want a predictable fixed length + // no data will be written there + $this->dataDir = $dataDir; + \OCP\Config::setSystemValue('datadirectory', $this->dataDir); + + $this->user = $userId; + $this->legacyStorageId = 'local::' . $this->dataDir . $this->user . '/'; + $this->newStorageId = 'home::' . $this->user; + \OC_User::createUser($this->user, $this->user); + } + + /** + * Create a storage entry + * + * @param string $storageId + */ + private function createStorage($storageId) { + $sql = 'INSERT INTO `*PREFIX*storages` (`id`)' + . ' VALUES (?)'; + + $storageId = \OC\Files\Cache\Storage::adjustStorageId($storageId); + $numRows = $this->connection->executeUpdate($sql, array($storageId)); + $this->assertEquals(1, $numRows); + + return \OC_DB::insertid('*PREFIX*storages'); + } + + /** + * Returns the storage id based on the numeric id + * + * @param int $numericId numeric id of the storage + * @return string storage id or null if not found + */ + private function getStorageId($storageId) { + $numericId = \OC\Files\Cache\Storage::getNumericStorageId($storageId); + if (!is_null($numericId)) { + return (int)$numericId; + } + return null; + } + + /** + * Create dummy data in the filecache for the given storage numeric id + * + * @param string $storageId storage id + */ + private function createData($storageId) { + $cache = new \OC\Files\Cache\Cache($storageId); + $cache->put( + 'dummyfile.txt', + array('size' => 5, 'mtime' => 12, 'mimetype' => 'text/plain') + ); + } + + /** + * Test that existing home storages are left alone when valid. + * @dataProvider settingsProvider + */ + public function testNoopWithExistingHomeStorage($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $newStorageNumId = $this->createStorage($this->newStorageId); + + $this->repair->run(); + + $this->assertNull($this->getStorageId($this->legacyStorageId)); + $this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId)); + } + + /** + * Test that legacy storages are converted to home storages when + * the latter does not exist. + * @dataProvider settingsProvider + */ + public function testConvertLegacyToHomeStorage($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $legacyStorageNumId = $this->createStorage($this->legacyStorageId); + + $this->repair->run(); + + $this->assertNull($this->getStorageId($this->legacyStorageId)); + $this->assertEquals($legacyStorageNumId, $this->getStorageId($this->newStorageId)); + } + + /** + * Test that legacy storages are converted to home storages + * when home storage already exists but has no data. + * @dataProvider settingsProvider + */ + public function testConvertLegacyToExistingEmptyHomeStorage($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $legacyStorageNumId = $this->createStorage($this->legacyStorageId); + $newStorageNumId = $this->createStorage($this->newStorageId); + + $this->createData($this->legacyStorageId); + + $this->repair->run(); + + $this->assertNull($this->getStorageId($this->legacyStorageId)); + $this->assertEquals($legacyStorageNumId, $this->getStorageId($this->newStorageId)); + } + + /** + * Test that legacy storages are converted to home storages + * when home storage already exists and the legacy storage + * has no data. + * @dataProvider settingsProvider + */ + public function testConvertEmptyLegacyToHomeStorage($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $legacyStorageNumId = $this->createStorage($this->legacyStorageId); + $newStorageNumId = $this->createStorage($this->newStorageId); + + $this->createData($this->newStorageId); + + $this->repair->run(); + + $this->assertNull($this->getStorageId($this->legacyStorageId)); + $this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId)); + } + + /** + * Test that nothing is done when both conflicting legacy + * and home storage have data. + * @dataProvider settingsProvider + */ + public function testConflictNoop($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $legacyStorageNumId = $this->createStorage($this->legacyStorageId); + $newStorageNumId = $this->createStorage($this->newStorageId); + + $this->createData($this->legacyStorageId); + $this->createData($this->newStorageId); + + try { + $thrown = false; + $this->repair->run(); + } + catch (\OC\RepairException $e) { + $thrown = true; + } + + $this->assertTrue($thrown); + + // storages left alone + $this->assertEquals($legacyStorageNumId, $this->getStorageId($this->legacyStorageId)); + $this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId)); + + // did not set the done flag + $this->assertNotEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone')); + } + + /** + * Test that the data dir local entry is left alone + * @dataProvider settingsProvider + */ + public function testDataDirEntryNoop($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $storageId = 'local::' . $this->dataDir; + $numId = $this->createStorage($storageId); + + $this->repair->run(); + + $this->assertEquals($numId, $this->getStorageId($storageId)); + } + + /** + * Test that external local storages are left alone + * @dataProvider settingsProvider + */ + public function testLocalExtStorageNoop($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $storageId = 'local::/tmp/somedir/' . $this->user; + $numId = $this->createStorage($storageId); + + $this->repair->run(); + + $this->assertEquals($numId, $this->getStorageId($storageId)); + } + + /** + * Test that other external storages are left alone + * @dataProvider settingsProvider + */ + public function testExtStorageNoop($dataDir, $userId) { + $this->prepareSettings($dataDir, $userId); + $storageId = 'smb::user@password/tmp/somedir/' . $this->user; + $numId = $this->createStorage($storageId); + + $this->repair->run(); + + $this->assertEquals($numId, $this->getStorageId($storageId)); + } + + /** + * Provides data dir and user name + */ + function settingsProvider() { + return array( + // regular data dir + array( + '/tmp/oc-autotest/datadir/', + uniqid('user_'), + ), + // long datadir / short user + array( + '/tmp/oc-autotest/datadir01234567890123456789012345678901234567890123456789END/', + uniqid('user_'), + ), + // short datadir / long user + array( + '/tmp/oc-autotest/datadir/', + 'u123456789012345678901234567890123456789012345678901234567890END', // 64 chars + ), + ); + } + + /** + * Only run the repair once + */ + public function testOnlyRunOnce() { + $output = array(); + $this->repair->listen('\OC\Repair', 'info', function ($description) use (&$output) { + $output[] = 'info: ' . $description; + }); + + $this->prepareSettings('/tmp/oc-autotest/datadir', uniqid('user_')); + $this->assertNotEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone')); + $this->repair->run(); + $this->assertEquals(1, count($output)); + $this->assertEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone')); + + $output = array(); + $this->repair->run(); + // no output which means it did not run + $this->assertEquals(0, count($output)); + $this->assertEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone')); + } +}