Merge pull request #9378 from nextcloud/backport/9364/swift-v3

[swiftv3] Log mount and storage issues in getFileInfo

3rdparty

@@ -1 +1 @@
-Subproject commit 8168fc1d0f33445ec4158867421dfaa9a0e241d7
+Subproject commit 49a3f672142d047a901329a6e1ea2ad582b7d681

apps/dav/lib/Files/FileSearchBackend.php

@@ -113,7 +113,7 @@ class FileSearchBackend implements ISearchBackend {
 		//todo dynamically load all propfind properties that are supported
 		return [
 			// queryable properties
-			new SearchPropertyDefinition('{DAV:}displayname', true, false, true),
+			new SearchPropertyDefinition('{DAV:}displayname', true, true, true),
 			new SearchPropertyDefinition('{DAV:}getcontenttype', true, true, true),
 			new SearchPropertyDefinition('{DAV:}getlastmodified', true, true, true, SearchPropertyDefinition::DATATYPE_DATETIME),
 			new SearchPropertyDefinition(FilesPlugin::SIZE_PROPERTYNAME, true, true, true, SearchPropertyDefinition::DATATYPE_NONNEGATIVE_INTEGER),

apps/files_external/lib/Lib/Storage/Swift.php

@@ -194,7 +194,11 @@ class Swift extends \OC\Files\Storage\Common {
 		$this->params = $params;
 		// FIXME: private class...
 		$this->objectCache = new \OC\Cache\CappedMemoryCache();
-		$this->connectionFactory = new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift/'), $this->params);
+		$this->connectionFactory = new SwiftFactory(
+			\OC::$server->getMemCacheFactory()->createDistributed('swift/'),
+			$this->params,
+			\OC::$server->getLogger()
+		);
 		$this->objectStore = new \OC\Files\ObjectStore\Swift($this->params, $this->connectionFactory);
 		$this->bucket = $params['bucket'];
 	}

config/config.sample.php

@@ -1226,13 +1226,21 @@ $CONFIG = array(
 			'password' => 'swift',
 			'domain' => [
 				'name' => 'default',
-			]
+			],
+		],
+		'scope' => [
+			'project' => [
+				'name' => 'service',
+				'domain' => [
+					'name' => 'default',
+				],
+			],
 		],
 		'tenantName' => 'service',
 		'serviceName' => 'swift',
 		'region' => 'regionOne',
-		'url' => "http://yourswifthost:5000/v3",
+		'url' => 'http://yourswifthost:5000/v3',
-		'bucket' => 'nextcloud'
+		'bucket' => 'nextcloud',
 	],
 ],
 

lib/private/Files/ObjectStore/ObjectStoreStorage.php

@@ -261,6 +261,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
 	public function fopen($path, $mode) {
 		$path = $this->normalizePath($path);
 
+		if (strrpos($path, '.') !== false) {
+			$ext = substr($path, strrpos($path, '.'));
+		} else {
+			$ext = '';
+		}
+
 		switch ($mode) {
 			case 'r':
 			case 'rb':
@@ -280,21 +286,21 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
 				}
 			case 'w':
 			case 'wb':
+			case 'w+':
+			case 'wb+':
+				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
+				$handle = fopen($tmpFile, $mode);
+				return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
+					$this->writeBack($tmpFile, $path);
+				});
 			case 'a':
 			case 'ab':
 			case 'r+':
-			case 'w+':
-			case 'wb+':
 			case 'a+':
 			case 'x':
 			case 'x+':
 			case 'c':
 			case 'c+':
-				if (strrpos($path, '.') !== false) {
-					$ext = substr($path, strrpos($path, '.'));
-				} else {
-					$ext = '';
-				}
 				$tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
 				if ($this->file_exists($path)) {
 					$source = $this->fopen($path, 'r');
@@ -423,4 +429,8 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
 	public function hasUpdated($path, $time) {
 		return false;
 	}
+
+	public function needsPartFile() {
+		return false;
+	}
 }

lib/private/Files/ObjectStore/Swift.php

@@ -36,16 +36,15 @@ class Swift implements IObjectStore {
 	 */
 	private $params;
 
-	/**
-	 * @var \OpenStack\ObjectStore\v1\Models\Container|null
-	 */
-	private $container = null;
-
 	/** @var SwiftFactory */
 	private $swiftFactory;
 
 	public function __construct($params, SwiftFactory $connectionFactory = null) {
-		$this->swiftFactory = $connectionFactory ?: new SwiftFactory(\OC::$server->getMemCacheFactory()->createDistributed('swift::'), $params);
+		$this->swiftFactory = $connectionFactory ?: new SwiftFactory(
+			\OC::$server->getMemCacheFactory()->createDistributed('swift::'),
+			$params,
+			\OC::$server->getLogger()
+		);
 		$this->params = $params;
 	}
 
@@ -62,6 +61,10 @@ class Swift implements IObjectStore {
 	 * @return string the container name where objects are stored
 	 */
 	public function getStorageId() {
+		if (isset($this->params['bucket'])) {
+			return $this->params['bucket'];
+		}
+
 		return $this->params['container'];
 	}
 

lib/private/Files/ObjectStore/SwiftFactory.php

@@ -30,6 +30,7 @@ use GuzzleHttp\HandlerStack;
 use OCP\Files\StorageAuthException;
 use OCP\Files\StorageNotAvailableException;
 use OCP\ICache;
+use OCP\ILogger;
 use OpenStack\Common\Error\BadResponseError;
 use OpenStack\Common\Auth\Token;
 use OpenStack\Identity\v2\Service as IdentityV2Service;
@@ -44,23 +45,30 @@ class SwiftFactory {
 	private $params;
 	/** @var Container|null */
 	private $container = null;
+	private $logger;
 
-	public function __construct(ICache $cache, array $params) {
+	public function __construct(ICache $cache, array $params, ILogger $logger) {
 		$this->cache = $cache;
 		$this->params = $params;
+		$this->logger = $logger;
 	}
 
 	private function getCachedToken(string $cacheKey) {
 		$cachedTokenString = $this->cache->get($cacheKey . '/token');
 		if ($cachedTokenString) {
-			return json_decode($cachedTokenString);
+			return json_decode($cachedTokenString, true);
 		} else {
 			return null;
 		}
 	}
 
 	private function cacheToken(Token $token, string $cacheKey) {
-		$this->cache->set($cacheKey . '/token', json_encode($token));
+		if ($token instanceof \OpenStack\Identity\v3\Models\Token) {
+			$value = json_encode($token->export());
+		} else {
+			$value = json_encode($token);
+		}
+		$this->cache->set($cacheKey . '/token', $value);
 	}
 
 	/**
@@ -72,7 +80,7 @@ class SwiftFactory {
 			$this->params['container'] = $this->params['bucket'];
 		}
 		if (!isset($this->params['container'])) {
-			$this->params['container'] = 'owncloud';
+			$this->params['container'] = 'nextcloud';
 		}
 		if (!isset($this->params['autocreate'])) {
 			// should only be true for tests
@@ -90,12 +98,9 @@ class SwiftFactory {
 			$this->params['tenantName'] = $this->params['tenant'];
 		}
 
-		$cacheKey = $userName . '@' . $this->params['url'] . '/' . $this->params['bucket'];
+		$cacheKey = $userName . '@' . $this->params['url'] . '/' . $this->params['container'];
 		$token = $this->getCachedToken($cacheKey);
-		$hasToken = is_array($token) && (new \DateTimeImmutable($token['expires_at'])) > (new \DateTimeImmutable('now'));
-		if ($hasToken) {
 		$this->params['cachedToken'] = $token;
-		}
 
 		$httpClient = new Client([
 			'base_uri' => TransportUtils::normalizeUrl($this->params['url']),
@@ -103,6 +108,10 @@ class SwiftFactory {
 		]);
 
 		if (isset($this->params['user']) && isset($this->params['user']['name'])) {
+			if (!isset($this->params['scope'])) {
+				throw new StorageAuthException('Scope has to be defined for V3 requests');
+			}
+
 			return $this->auth(IdentityV3Service::factory($httpClient), $cacheKey);
 		} else {
 			return $this->auth(IdentityV2Service::factory($httpClient), $cacheKey);
@@ -120,7 +129,20 @@ class SwiftFactory {
 		$this->params['authUrl'] = $this->params['url'];
 		$client = new OpenStack($this->params);
 
-		if (!isset($this->params['cachedToken'])) {
+		$cachedToken = $this->params['cachedToken'];
+		$hasValidCachedToken = false;
+		if (\is_array($cachedToken) && ($authService instanceof IdentityV3Service)) {
+			$token = $authService->generateTokenFromCache($cachedToken);
+			if (\is_null($token->catalog)) {
+				$this->logger->warning('Invalid cached token for swift, no catalog set: ' . json_encode($cachedToken));
+			} else if ($token->hasExpired()) {
+				$this->logger->debug('Cached token for swift expired');
+			} else {
+				$hasValidCachedToken = true;
+			}
+		}
+
+		if (!$hasValidCachedToken) {
 			try {
 				$token = $authService->generateToken($this->params);
 				$this->cacheToken($token, $cacheKey);

lib/private/Files/View.php

@@ -1360,6 +1360,7 @@ class View {
 
 		$mount = Filesystem::getMountManager()->find($path);
 		if (!$mount) {
+			\OC::$server->getLogger()->warning('Mountpoint not found for path: ' . $path);
 			return false;
 		}
 		$storage = $mount->getStorage();
@@ -1391,6 +1392,8 @@ class View {
 			}
 
 			return $info;
+		} else {
+			\OC::$server->getLogger()->warning('Storage not valid for mountpoint: ' . $mount->getMountPoint());
 		}
 
 		return false;