Merge pull request #17641 from owncloud/fix_objectstore_rename

don't move files in cache twice, fixes renaming for objectstores

apps/dav/tests/unit/connector/sabre/file.php

@@ -8,6 +8,7 @@
 
 namespace Test\Connector\Sabre;
 
+use OC\Files\Storage\Local;
 use Test\HookHelper;
 use OC\Files\Filesystem;
 use OCP\Lock\ILockingProvider;
@@ -798,14 +799,16 @@ class File extends \Test\TestCase {
 		}
 		$files = [];
 		list($storage, $internalPath) = $userView->resolvePath($path);
-		$realPath = $storage->getSourcePath($internalPath);
+		if($storage instanceof Local) {
-		$dh = opendir($realPath);
+			$realPath = $storage->getSourcePath($internalPath);
-		while (($file = readdir($dh)) !== false) {
+			$dh = opendir($realPath);
-			if (substr($file, strlen($file) - 5, 5) === '.part') {
+			while (($file = readdir($dh)) !== false) {
-				$files[] = $file;
+				if (substr($file, strlen($file) - 5, 5) === '.part') {
+					$files[] = $file;
+				}
 			}
+			closedir($dh);
 		}
-		closedir($dh);
 		return $files;
 	}
 

apps/encryption/tests/lib/MigrationTest.php

@@ -62,6 +62,8 @@ class MigrationTest extends \Test\TestCase {
 	}
 
 	protected function createDummyShareKeys($uid) {
+		$this->loginAsUser($uid);
+
 		$this->view->mkdir($uid . '/files_encryption/keys/folder1/folder2/folder3/file3');
 		$this->view->mkdir($uid . '/files_encryption/keys/folder1/folder2/file2');
 		$this->view->mkdir($uid . '/files_encryption/keys/folder1/file.1');
@@ -87,6 +89,8 @@ class MigrationTest extends \Test\TestCase {
 	}
 
 	protected function createDummyUserKeys($uid) {
+		$this->loginAsUser($uid);
+
 		$this->view->mkdir($uid . '/files_encryption/');
 		$this->view->mkdir('/files_encryption/public_keys');
 		$this->view->file_put_contents($uid . '/files_encryption/' . $uid . '.privateKey', 'privateKey');
@@ -94,6 +98,8 @@ class MigrationTest extends \Test\TestCase {
 	}
 
 	protected function createDummyFileKeys($uid) {
+		$this->loginAsUser($uid);
+
 		$this->view->mkdir($uid . '/files_encryption/keys/folder1/folder2/folder3/file3');
 		$this->view->mkdir($uid . '/files_encryption/keys/folder1/folder2/file2');
 		$this->view->mkdir($uid . '/files_encryption/keys/folder1/file.1');
@@ -105,6 +111,8 @@ class MigrationTest extends \Test\TestCase {
 	}
 
 	protected function createDummyFiles($uid) {
+		$this->loginAsUser($uid);
+
 		$this->view->mkdir($uid . '/files/folder1/folder2/folder3/file3');
 		$this->view->mkdir($uid . '/files/folder1/folder2/file2');
 		$this->view->mkdir($uid . '/files/folder1/file.1');
@@ -116,6 +124,8 @@ class MigrationTest extends \Test\TestCase {
 	}
 
 	protected function createDummyFilesInTrash($uid) {
+		$this->loginAsUser($uid);
+
 		$this->view->mkdir($uid . '/files_trashbin/keys/file1.d5457864');
 		$this->view->mkdir($uid . '/files_trashbin/keys/folder1.d7437648723/file2');
 		$this->view->file_put_contents($uid . '/files_trashbin/keys/file1.d5457864/' . self::TEST_ENCRYPTION_MIGRATION_USER1 . '.shareKey' , 'data');
@@ -165,6 +175,7 @@ class MigrationTest extends \Test\TestCase {
 
 		$this->createDummySystemWideKeys();
 
+		/** @var \PHPUnit_Framework_MockObject_MockObject|\OCA\Encryption\Migration $m */
 		$m = $this->getMockBuilder('OCA\Encryption\Migration')
 			->setConstructorArgs(
 				[
@@ -176,27 +187,38 @@ class MigrationTest extends \Test\TestCase {
 			)->setMethods(['getSystemMountPoints'])->getMock();
 
 		$m->expects($this->any())->method('getSystemMountPoints')
-			->willReturn([['mountpoint' => 'folder1'], ['mountpoint' => 'folder2']]);
+			->will($this->returnValue([['mountpoint' => 'folder1'], ['mountpoint' => 'folder2']]));
 
 		$m->reorganizeFolderStructure();
 		// even if it runs twice folder should always move only once
 		$m->reorganizeFolderStructure();
 
+		$this->loginAsUser(self::TEST_ENCRYPTION_MIGRATION_USER1);
+
 		$this->assertTrue(
 			$this->view->file_exists(
 				self::TEST_ENCRYPTION_MIGRATION_USER1 . '/files_encryption/' .
 				$this->moduleId . '/' . self::TEST_ENCRYPTION_MIGRATION_USER1 . '.publicKey')
 		);
+
+		$this->loginAsUser(self::TEST_ENCRYPTION_MIGRATION_USER2);
+
 		$this->assertTrue(
 			$this->view->file_exists(
 				self::TEST_ENCRYPTION_MIGRATION_USER2 . '/files_encryption/' .
 				$this->moduleId . '/' . self::TEST_ENCRYPTION_MIGRATION_USER2 . '.publicKey')
 		);
+
+		$this->loginAsUser(self::TEST_ENCRYPTION_MIGRATION_USER3);
+
 		$this->assertTrue(
 			$this->view->file_exists(
 				self::TEST_ENCRYPTION_MIGRATION_USER3 . '/files_encryption/' .
 				$this->moduleId . '/' . self::TEST_ENCRYPTION_MIGRATION_USER3 . '.publicKey')
 		);
+
+		$this->loginAsUser(self::TEST_ENCRYPTION_MIGRATION_USER1);
+
 		$this->assertTrue(
 			$this->view->file_exists(
 			    '/files_encryption/' . $this->moduleId . '/systemwide_1.publicKey')
@@ -217,6 +239,8 @@ class MigrationTest extends \Test\TestCase {
 	}
 
 	protected function verifyFilesInTrash($uid) {
+		$this->loginAsUser($uid);
+
 		// share keys
 		$this->assertTrue(
 			$this->view->file_exists($uid . '/files_encryption/keys/files_trashbin/file1.d5457864/' . $this->moduleId . '/' . self::TEST_ENCRYPTION_MIGRATION_USER1 . '.shareKey')
@@ -244,6 +268,7 @@ class MigrationTest extends \Test\TestCase {
 	protected function verifyNewKeyPath($uid) {
 		// private key
 		if ($uid !== '') {
+			$this->loginAsUser($uid);
 			$this->assertTrue($this->view->file_exists($uid . '/files_encryption/' . $this->moduleId . '/'. $uid . '.privateKey'));
 		}
 		// file keys

apps/files/tests/command/deleteorphanedfilestest.php

@@ -22,6 +22,7 @@
 namespace OCA\Files\Tests\Command;
 
 use OCA\Files\Command\DeleteOrphanedFiles;
+use OCP\Files\StorageNotAvailableException;
 
 class DeleteOrphanedFilesTest extends \Test\TestCase {
 
@@ -110,7 +111,11 @@ class DeleteOrphanedFilesTest extends \Test\TestCase {
 
 		$this->assertCount(0, $this->getFile($fileInfo->getId()), 'Asserts that file gets cleaned up');
 
-		$view->unlink('files/test');
+		// since we deleted the storage it might throw a (valid) StorageNotAvailableException
+		try {
+			$view->unlink('files/test');
+		} catch (StorageNotAvailableException $e) {
+		}
 	}
 }
 

apps/files_sharing/tests/sharedstorage.php

@@ -47,8 +47,10 @@ class Test_Files_Sharing_Storage extends OCA\Files_sharing\Tests\TestCase {
 	}
 
 	protected function tearDown() {
-		$this->view->unlink($this->folder);
+		if ($this->view) {
-		$this->view->unlink($this->filename);
+			$this->view->unlink($this->folder);
+			$this->view->unlink($this->filename);
+		}
 
 		\OC\Files\Filesystem::getLoader()->removeStorageWrapper('oc_trashbin');
 
@@ -85,8 +87,9 @@ class Test_Files_Sharing_Storage extends OCA\Files_sharing\Tests\TestCase {
 		$this->assertFalse($user2View->is_dir($this->folder));
 
 		// delete the local folder
-		$fullPath = \OC_Config::getValue('datadirectory') . '/' . self::TEST_FILES_SHARING_API_USER2 . '/files/localfolder';
+		/** @var \OC\Files\Storage\Storage $storage */
-		rmdir($fullPath);
+		list($storage, $internalPath)  = \OC\Files\Filesystem::resolvePath('/' . self::TEST_FILES_SHARING_API_USER2 . '/files/localfolder');
+		$storage->rmdir($internalPath);
 
 		//enforce reload of the mount points
 		self::loginHelper(self::TEST_FILES_SHARING_API_USER2);

apps/files_sharing/tests/watcher.php

@@ -108,9 +108,8 @@ class Test_Files_Sharing_Watcher extends OCA\Files_sharing\Tests\TestCase {
 		$this->sharedCache->put('', array('mtime' => 10, 'storage_mtime' => 10, 'size' => '-1', 'mimetype' => 'httpd/unix-directory'));
 
 		// run the propagation code
-		$result = $this->sharedStorage->getWatcher()->checkUpdate('');
+		$this->sharedStorage->getWatcher()->checkUpdate('');
-
+		$this->sharedStorage->getCache()->correctFolderSize('');
-		$this->assertTrue($result);
 
 		// the owner's parent dirs must have increase size
 		$newSizes = self::getOwnerDirSizes('files/container/shareddir');
@@ -139,9 +138,8 @@ class Test_Files_Sharing_Watcher extends OCA\Files_sharing\Tests\TestCase {
 		$this->sharedCache->put('subdir', array('mtime' => 10, 'storage_mtime' => 10, 'size' => $dataLen, 'mimetype' => 'text/plain'));
 
 		// run the propagation code
-		$result = $this->sharedStorage->getWatcher()->checkUpdate('subdir');
+		$this->sharedStorage->getWatcher()->checkUpdate('subdir');
-
+		$this->sharedStorage->getCache()->correctFolderSize('subdir');
-		$this->assertTrue($result);
 
 		// the owner's parent dirs must have increase size
 		$newSizes = self::getOwnerDirSizes('files/container/shareddir/subdir');

apps/files_trashbin/lib/trashbin.php

@@ -581,8 +581,9 @@ class Trashbin {
 		if ($quota === null || $quota === 'none') {
 			$quota = \OC\Files\Filesystem::free_space('/');
 			$softQuota = false;
-			if ($quota === \OCP\Files\FileInfo::SPACE_UNKNOWN) {
+			// inf or unknown free space
-				$quota = 0;
+			if ($quota < 0) {
+				$quota = PHP_INT_MAX;
 			}
 		} else {
 			$quota = \OCP\Util::computerFileSize($quota);

apps/files_trashbin/tests/trashbin.php

@@ -236,6 +236,8 @@ class Test_Trashbin extends \Test\TestCase {
 		// user2-1.txt should have been expired
 		$this->verifyArray($filesInTrashUser2AfterDelete, array('user2-2.txt', 'user1-4.txt'));
 
+		self::loginHelper(self::TEST_TRASHBIN_USER1);
+
 		// user1-1.txt and user1-3.txt should have been expired
 		$filesInTrashUser1AfterDelete = OCA\Files_Trashbin\Helper::getTrashFiles('/', self::TEST_TRASHBIN_USER1);
 
@@ -600,22 +602,24 @@ class Test_Trashbin extends \Test\TestCase {
 
 		// delete source folder
 		list($storage, $internalPath) = $this->rootView->resolvePath('/' . self::TEST_TRASHBIN_USER1 . '/files/folder');
-		$folderAbsPath = $storage->getSourcePath($internalPath);
+		if ($storage instanceof \OC\Files\Storage\Local) {
-		// make folder read-only
+			$folderAbsPath = $storage->getSourcePath($internalPath);
-		chmod($folderAbsPath, 0555);
+			// make folder read-only
+			chmod($folderAbsPath, 0555);
 
-		$this->assertTrue(
+			$this->assertTrue(
-			OCA\Files_Trashbin\Trashbin::restore(
+				OCA\Files_Trashbin\Trashbin::restore(
-				'file1.txt.d' . $trashedFile->getMtime(),
+					'file1.txt.d' . $trashedFile->getMtime(),
-				$trashedFile->getName(),
+					$trashedFile->getName(),
-				$trashedFile->getMtime()
+					$trashedFile->getMtime()
-			)
+				)
-		);
+			);
 
-		$file = $userFolder->get('file1.txt');
+			$file = $userFolder->get('file1.txt');
-		$this->assertEquals('foo', $file->getContent());
+			$this->assertEquals('foo', $file->getContent());
 
-		chmod($folderAbsPath, 0755);
+			chmod($folderAbsPath, 0755);
+		}
 	}
 
 	/**

apps/files_versions/lib/storage.php

@@ -347,7 +347,20 @@ class Storage {
 		$view->lockFile($path1, ILockingProvider::LOCK_EXCLUSIVE);
 		$view->lockFile($path2, ILockingProvider::LOCK_EXCLUSIVE);
 
-		$result = $storage2->moveFromStorage($storage1, $internalPath1, $internalPath2);
+		// TODO add a proper way of overwriting a file while maintaining file ids
+		if ($storage1->instanceOfStorage('\OC\Files\ObjectStore\ObjectStoreStorage') || $storage2->instanceOfStorage('\OC\Files\ObjectStore\ObjectStoreStorage')) {
+			$source = $storage1->fopen($internalPath1, 'r');
+			$target = $storage2->fopen($internalPath2, 'w');
+			list(, $result) = \OC_Helper::streamCopy($source, $target);
+			fclose($source);
+			fclose($target);
+
+			if ($result !== false) {
+				$storage1->unlink($internalPath1);
+			}
+		} else {
+			$result = $storage2->moveFromStorage($storage1, $internalPath1, $internalPath2);
+		}
 
 		$view->unlockFile($path1, ILockingProvider::LOCK_EXCLUSIVE);
 		$view->unlockFile($path2, ILockingProvider::LOCK_EXCLUSIVE);
@@ -663,17 +676,21 @@ class Storage {
 
 			// calculate available space for version history
 			// subtract size of files and current versions size from quota
-			if ($softQuota) {
+			if ($quota >= 0) {
-				$files_view = new \OC\Files\View('/'.$uid.'/files');
+				if ($softQuota) {
-				$rootInfo = $files_view->getFileInfo('/', false);
+					$files_view = new \OC\Files\View('/' . $uid . '/files');
-				$free = $quota-$rootInfo['size']; // remaining free space for user
+					$rootInfo = $files_view->getFileInfo('/', false);
-				if ( $free > 0 ) {
+					$free = $quota - $rootInfo['size']; // remaining free space for user
-					$availableSpace = ($free * self::DEFAULTMAXSIZE / 100) - ($versionsSize + $offset); // how much space can be used for versions
+					if ($free > 0) {
+						$availableSpace = ($free * self::DEFAULTMAXSIZE / 100) - ($versionsSize + $offset); // how much space can be used for versions
+					} else {
+						$availableSpace = $free - $versionsSize - $offset;
+					}
 				} else {
-					$availableSpace = $free - $versionsSize - $offset;
+					$availableSpace = $quota - $offset;
 				}
 			} else {
-				$availableSpace = $quota - $offset;
+				$availableSpace = PHP_INT_MAX;
 			}
 
 			$allVersions = Storage::getVersions($uid, $filename);

apps/files_versions/tests/versions.php

@@ -301,11 +301,10 @@ class Test_Files_Versioning extends \Test\TestCase {
 		// execute rename hook of versions app
 		\OC\Files\Filesystem::rename('/folder1/test.txt', '/folder1/folder2/test.txt');
 
-
-		self::loginHelper(self::TEST_VERSIONS_USER2);
-
 		$this->runCommands();
 
+		self::loginHelper(self::TEST_VERSIONS_USER);
+
 		$this->assertFalse($this->rootView->file_exists($v1));
 		$this->assertFalse($this->rootView->file_exists($v2));
 

autotest.sh

@@ -270,6 +270,7 @@ function execute_tests {
 	fi
 
 	if [ "$PRIMARY_STORAGE_CONFIG" == "swift" ] ; then
+		cd ..
 		echo "Kill the swift docker"
 		tests/objectstore/stop-swift-ceph.sh
 	fi

lib/private/files/cache/updater.php

@@ -171,13 +171,15 @@ class Updater {
 
 		if ($sourceStorage && $targetStorage) {
 			$targetCache = $targetStorage->getCache($sourceInternalPath);
-			if ($targetCache->inCache($targetInternalPath)) {
+			if ($sourceStorage->getCache($sourceInternalPath)->inCache($sourceInternalPath)) {
-				$targetCache->remove($targetInternalPath);
+				if ($targetCache->inCache($targetInternalPath)) {
-			}
+					$targetCache->remove($targetInternalPath);
-			if ($sourceStorage === $targetStorage) {
+				}
-				$targetCache->move($sourceInternalPath, $targetInternalPath);
+				if ($sourceStorage === $targetStorage) {
-			} else {
+					$targetCache->move($sourceInternalPath, $targetInternalPath);
-				$targetCache->moveFromCache($sourceStorage->getCache(), $sourceInternalPath, $targetInternalPath);
+				} else {
+					$targetCache->moveFromCache($sourceStorage->getCache(), $sourceInternalPath, $targetInternalPath);
+				}
 			}
 
 			if (pathinfo($sourceInternalPath, PATHINFO_EXTENSION) !== pathinfo($targetInternalPath, PATHINFO_EXTENSION)) {

lib/private/files/objectstore/objectstorestorage.php

@@ -337,7 +337,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
 				'size' => 0,
 				'mtime' => $mtime,
 				'storage_mtime' => $mtime,
-				'permissions' => \OCP\Constants::PERMISSION_ALL,
+				'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
 			);
 			$fileId = $this->getCache()->put($path, $stat);
 			try {
@@ -362,7 +362,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
 		if (empty($stat)) {
 			// create new file
 			$stat = array(
-				'permissions' => \OCP\Constants::PERMISSION_ALL,
+				'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
 			);
 		}
 		// update stat with new data

lib/private/share/share.php

@@ -2636,7 +2636,9 @@ class Share extends Constants {
 	 */
 	private static function isFileReachable($path, $ownerStorageId) {
 		// if outside the home storage, file is always considered reachable
-		if (!(substr($ownerStorageId, 0, 6) === 'home::')) {
+		if (!(substr($ownerStorageId, 0, 6) === 'home::' ||
+			substr($ownerStorageId, 0, 13) === 'object::user:'
+		)) {
 			return true;
 		}
 

tests/lib/files/cache/updater.php

@@ -143,6 +143,24 @@ class Updater extends \Test\TestCase {
 		$this->assertEquals($cached['fileid'], $cachedTarget['fileid']);
 	}
 
+	public function testMoveNonExistingOverwrite() {
+		$this->storage->file_put_contents('bar.txt', 'qwerty');
+		$this->updater->update('bar.txt');
+
+		$cached = $this->cache->get('bar.txt');
+
+		$this->updater->rename('foo.txt', 'bar.txt');
+
+		$this->assertFalse($this->cache->inCache('foo.txt'));
+		$this->assertTrue($this->cache->inCache('bar.txt'));
+
+		$cachedTarget = $this->cache->get('bar.txt');
+		$this->assertEquals($cached['etag'], $cachedTarget['etag']);
+		$this->assertEquals($cached['mtime'], $cachedTarget['mtime']);
+		$this->assertEquals($cached['size'], $cachedTarget['size']);
+		$this->assertEquals($cached['fileid'], $cachedTarget['fileid']);
+	}
+
 	public function testNewFileDisabled() {
 		$this->storage->file_put_contents('foo.txt', 'bar');
 		$this->assertFalse($this->cache->inCache('foo.txt'));

tests/lib/files/objectstore/swift.php

@@ -99,6 +99,10 @@ class Swift extends \Test\Files\Storage\Storage {
 		}
 	}
 
+	public function testCheckUpdate() {
+		$this->markTestSkipped('Detecting external changes is not supported on object storages');
+	}
+
 	/**
 	 * @dataProvider copyAndMoveProvider
 	 */

tests/objectstore/stop-swift-ceph.sh

@@ -23,16 +23,18 @@ if [ -z "$thisFolder" ]; then
     thisFolder="."
 fi;
 
-# stopping and removing docker containers
+if [ -e $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift ]; then
-for container in `cat $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift`; do
+    # stopping and removing docker containers
-    if [ -n "$DEBUG" ]; then
+    for container in `cat $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift`; do
-        docker logs $container
+        if [ -n "$DEBUG" ]; then
-    fi
+            docker logs $container
-    echo "Stopping and removing docker container $container"
+        fi
-    # kills running container and removes it
+        echo "Stopping and removing docker container $container"
-    docker rm -f $container
+        # kills running container and removes it
-done;
+        docker rm -f $container
+    done;
+fi;
 
 # cleanup
-rm $thisFolder/swift.config.php
+rm -rf $thisFolder/swift.config.php
-rm $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift
+rm -rf $thisFolder/dockerContainerCeph.$EXECUTOR_NUMBER.swift