Merge pull request #1247 from nextcloud/storage-id-global-cache

more efficient querying of numeric storage ids

apps/files_external/lib/Config/ConfigAdapter.php

@@ -121,54 +121,63 @@ class ConfigAdapter implements IMountProvider {
 	public function getMountsForUser(IUser $user, IStorageFactory $loader) {
 		$this->migrator->migrateUser($user);
 
-		$mounts = [];
-
 		$this->userStoragesService->setUser($user);
 		$this->userGlobalStoragesService->setUser($user);
 
-		foreach ($this->userGlobalStoragesService->getAllStoragesForUser() as $storage) {
+		$storageConfigs = $this->userGlobalStoragesService->getAllStoragesForUser();
+
+		$storages = array_map(function(StorageConfig $storageConfig) use ($user) {
 			try {
-				$this->prepareStorageConfig($storage, $user);
+				$this->prepareStorageConfig($storageConfig, $user);
-				$impl = $this->constructStorage($storage);
+				return $this->constructStorage($storageConfig);
 			} catch (\Exception $e) {
 				// propagate exception into filesystem
-				$impl = new FailedStorage(['exception' => $e]);
+				return new FailedStorage(['exception' => $e]);
 			}
+		}, $storageConfigs);
 
+
+		\OC\Files\Cache\Storage::getGlobalCache()->loadForStorageIds(array_map(function(Storage\IStorage $storage) {
+			return $storage->getId();
+		}, $storages));
+
+		$availableStorages = array_map(function (Storage\IStorage $storage, StorageConfig $storageConfig) {
 			try {
-				$availability = $impl->getAvailability();
+				$availability = $storage->getAvailability();
 				if (!$availability['available'] && !Availability::shouldRecheck($availability)) {
-					$impl = new FailedStorage([
+					$storage = new FailedStorage([
-						'exception' => new StorageNotAvailableException('Storage with mount id ' . $storage->getId() . ' is not available')
+						'exception' => new StorageNotAvailableException('Storage with mount id ' . $storageConfig->getId() . ' is not available')
 					]);
 				}
 			} catch (\Exception $e) {
 				// propagate exception into filesystem
-				$impl = new FailedStorage(['exception' => $e]);
+				$storage = new FailedStorage(['exception' => $e]);
 			}
+			return $storage;
+		}, $storages, $storageConfigs);
 
-			if ($storage->getType() === StorageConfig::MOUNT_TYPE_PERSONAl) {
+		$mounts = array_map(function(StorageConfig $storageConfig, Storage\IStorage $storage) use ($user, $loader) {
-				$mount = new PersonalMount(
+			if ($storageConfig->getType() === StorageConfig::MOUNT_TYPE_PERSONAl) {
+				return new PersonalMount(
 					$this->userStoragesService,
-					$storage->getId(),
+					$storageConfig->getId(),
-					$impl,
+					$storage,
-					'/' . $user->getUID() . '/files' . $storage->getMountPoint(),
+					'/' . $user->getUID() . '/files' . $storageConfig->getMountPoint(),
 					null,
 					$loader,
-					$storage->getMountOptions()
+					$storageConfig->getMountOptions()
 				);
 			} else {
-				$mount = new MountPoint(
+				return new MountPoint(
-					$impl,
+					$storage,
-					'/' . $user->getUID() . '/files' . $storage->getMountPoint(),
+					'/' . $user->getUID() . '/files' . $storageConfig->getMountPoint(),
 					null,
 					$loader,
-					$storage->getMountOptions(),
+					$storageConfig->getMountOptions(),
-					$storage->getId()
+					$storageConfig->getId()
 				);
 			}
-			$mounts[$storage->getMountPoint()] = $mount;
+		}, $storageConfigs, $availableStorages);
-		}
 
 		$this->userStoragesService->resetUser();
 		$this->userGlobalStoragesService->resetUser();

apps/files_sharing/tests/TestCase.php

@@ -168,6 +168,7 @@ abstract class TestCase extends \Test\TestCase {
 		self::resetStorage();
 
 		\OC_Util::tearDownFS();
+		\OC\Files\Cache\Storage::getGlobalCache()->clearCache();
 		\OC::$server->getUserSession()->setUser(null);
 		\OC\Files\Filesystem::tearDown();
 		\OC::$server->getUserSession()->login($user, $password);

lib/composer/composer/autoload_classmap.php

@@ -466,6 +466,7 @@ return array(
     'OC\\Files\\Cache\\Propagator' => $baseDir . '/lib/private/Files/Cache/Propagator.php',
     'OC\\Files\\Cache\\Scanner' => $baseDir . '/lib/private/Files/Cache/Scanner.php',
     'OC\\Files\\Cache\\Storage' => $baseDir . '/lib/private/Files/Cache/Storage.php',
+    'OC\\Files\\Cache\\StorageGlobal' => $baseDir . '/lib/private/Files/Cache/StorageGlobal.php',
     'OC\\Files\\Cache\\Updater' => $baseDir . '/lib/private/Files/Cache/Updater.php',
     'OC\\Files\\Cache\\Watcher' => $baseDir . '/lib/private/Files/Cache/Watcher.php',
     'OC\\Files\\Cache\\Wrapper\\CacheJail' => $baseDir . '/lib/private/Files/Cache/Wrapper/CacheJail.php',

lib/composer/composer/autoload_static.php

@@ -496,6 +496,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OC\\Files\\Cache\\Propagator' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Propagator.php',
         'OC\\Files\\Cache\\Scanner' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Scanner.php',
         'OC\\Files\\Cache\\Storage' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Storage.php',
+        'OC\\Files\\Cache\\StorageGlobal' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/StorageGlobal.php',
         'OC\\Files\\Cache\\Updater' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Updater.php',
         'OC\\Files\\Cache\\Watcher' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Watcher.php',
         'OC\\Files\\Cache\\Wrapper\\CacheJail' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Wrapper/CacheJail.php',

lib/private/Files/Cache/Storage.php

@@ -40,9 +40,21 @@ namespace OC\Files\Cache;
  * @package OC\Files\Cache
  */
 class Storage {
+	/** @var StorageGlobal|null */
+	private static $globalCache = null;
 	private $storageId;
 	private $numericId;
 
+	/**
+	 * @return StorageGlobal
+	 */
+	public static function getGlobalCache() {
+		if (is_null(self::$globalCache)) {
+			self::$globalCache = new StorageGlobal(\OC::$server->getDatabaseConnection());
+		}
+		return self::$globalCache;
+	}
+
 	/**
 	 * @param \OC\Files\Storage\Storage|string $storage
 	 * @param bool $isAvailable
@@ -75,12 +87,10 @@ class Storage {
 
 	/**
 	 * @param string $storageId
-	 * @return array|null
+	 * @return array
 	 */
 	public static function getStorageById($storageId) {
-		$sql = 'SELECT * FROM `*PREFIX*storages` WHERE `id` = ?';
+		return self::getGlobalCache()->getStorageInfo($storageId);
-		$result = \OC_DB::executeAudited($sql, array($storageId));
-		return $result->fetchRow();
 	}
 
 	/**

lib/private/Files/Cache/StorageGlobal.php

@@ -0,0 +1,78 @@
+<?php
+/**
+ * @copyright Robin Appelman <robin@icewind.nl>
+ *
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Files\Cache;
+
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\Mount\IMountPoint;
+use OCP\IDBConnection;
+
+/**
+ * Handle the mapping between the string and numeric storage ids
+ *
+ * Each storage has 2 different ids
+ *    a string id which is generated by the storage backend and reflects the configuration of the storage (e.g. 'smb://user@host/share')
+ *    and a numeric storage id which is referenced in the file cache
+ *
+ * A mapping between the two storage ids is stored in the database and accessible trough this class
+ *
+ * @package OC\Files\Cache
+ */
+class StorageGlobal {
+	/** @var IDBConnection */
+	private $connection;
+
+	/** @var array[] */
+	private $cache = [];
+
+	public function __construct(IDBConnection $connection) {
+		$this->connection = $connection;
+	}
+
+	/**
+	 * @param string[] $storageIds
+	 */
+	public function loadForStorageIds(array $storageIds) {
+		$builder = $this->connection->getQueryBuilder();
+		$query = $builder->select(['id', 'numeric_id', 'available', 'last_checked'])
+			->from('storages')
+			->where($builder->expr()->in('id', $builder->createNamedParameter(array_values($storageIds), IQueryBuilder::PARAM_STR_ARRAY)));
+
+		$result = $query->execute();
+		while ($row = $result->fetch()) {
+			$this->cache[$row['id']] = $row;
+		}
+	}
+
+	/**
+	 * @param string $storageId
+	 * @return array|null
+	 */
+	public function getStorageInfo($storageId) {
+		if (!isset($this->cache[$storageId])) {
+			$this->loadForStorageIds([$storageId]);
+		}
+		return isset($this->cache[$storageId]) ? $this->cache[$storageId] : null;
+	}
+
+	public function clearCache() {
+		$this->cache = [];
+	}
+}

lib/private/Repair/RepairLegacyStorages.php

@@ -245,6 +245,8 @@ class RepairLegacyStorages implements IRepairStep{
 
 		$this->connection->commit();
 
+		Storage::getGlobalCache()->clearCache();
+
 		if ($hasWarnings) {
 			$out->warning('Some legacy storages could not be repaired. Please manually fix them then re-run ./occ maintenance:repair');
 		} else {

tests/lib/TestCase.php

@@ -148,6 +148,8 @@ abstract class TestCase extends \PHPUnit_Framework_TestCase {
 		libxml_clear_errors();
 		$this->assertEquals([], $errors);
 
+		\OC\Files\Cache\Storage::getGlobalCache()->clearCache();
+
 		// tearDown the traits
 		$traits = $this->getTestTraits();
 		foreach ($traits as $trait) {