Merge pull request #12227 from nextcloud/bugfix/cleanu_upload_folders

Background job to cleanup leftover chunked uploads

apps/dav/appinfo/info.xml

@@ -5,7 +5,7 @@
 	<name>WebDAV</name>
 	<summary>WebDAV endpoint</summary>
 	<description>WebDAV endpoint</description>
-	<version>1.9.0</version>
+	<version>1.9.1</version>
 	<licence>agpl</licence>
 	<author>owncloud.org</author>
 	<namespace>DAV</namespace>
@@ -32,6 +32,9 @@
 			<step>OCA\DAV\Migration\BuildCalendarSearchIndex</step>
 			<step>OCA\DAV\Migration\RefreshWebcalJobRegistrar</step>
 		</post-migration>
+		<live-migration>
+			<step>OCA\DAV\Migration\ChunkCleanup</step>
+		</live-migration>
 	</repair-steps>
 
 	<commands>

apps/dav/composer/composer/autoload_classmap.php

@@ -16,6 +16,7 @@ return array(
     'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
     'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => $baseDir . '/../lib/BackgroundJob/RefreshWebcalJob.php',
     'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => $baseDir . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
+    'OCA\\DAV\\BackgroundJob\\UploadCleanup' => $baseDir . '/../lib/BackgroundJob/UploadCleanup.php',
     'OCA\\DAV\\CalDAV\\Activity\\Backend' => $baseDir . '/../lib/CalDAV/Activity/Backend.php',
     'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => $baseDir . '/../lib/CalDAV/Activity/Filter/Calendar.php',
     'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => $baseDir . '/../lib/CalDAV/Activity/Filter/Todo.php',
@@ -153,6 +154,7 @@ return array(
     'OCA\\DAV\\Migration\\BuildCalendarSearchIndex' => $baseDir . '/../lib/Migration/BuildCalendarSearchIndex.php',
     'OCA\\DAV\\Migration\\BuildCalendarSearchIndexBackgroundJob' => $baseDir . '/../lib/Migration/BuildCalendarSearchIndexBackgroundJob.php',
     'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => $baseDir . '/../lib/Migration/CalDAVRemoveEmptyValue.php',
+    'OCA\\DAV\\Migration\\ChunkCleanup' => $baseDir . '/../lib/Migration/ChunkCleanup.php',
     'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => $baseDir . '/../lib/Migration/FixBirthdayCalendarComponent.php',
     'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => $baseDir . '/../lib/Migration/RefreshWebcalJobRegistrar.php',
     'OCA\\DAV\\Migration\\Version1004Date20170825134824' => $baseDir . '/../lib/Migration/Version1004Date20170825134824.php',
@@ -181,6 +183,7 @@ return array(
     'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => $baseDir . '/../lib/SystemTag/SystemTagsRelationsCollection.php',
     'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
     'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
+    'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
     'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
     'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',
     'OCA\\DAV\\Upload\\UploadFolder' => $baseDir . '/../lib/Upload/UploadFolder.php',

apps/dav/composer/composer/autoload_static.php

@@ -31,6 +31,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
         'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/RefreshWebcalJob.php',
         'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
+        'OCA\\DAV\\BackgroundJob\\UploadCleanup' => __DIR__ . '/..' . '/../lib/BackgroundJob/UploadCleanup.php',
         'OCA\\DAV\\CalDAV\\Activity\\Backend' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Backend.php',
         'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Calendar.php',
         'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Todo.php',
@@ -168,6 +169,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Migration\\BuildCalendarSearchIndex' => __DIR__ . '/..' . '/../lib/Migration/BuildCalendarSearchIndex.php',
         'OCA\\DAV\\Migration\\BuildCalendarSearchIndexBackgroundJob' => __DIR__ . '/..' . '/../lib/Migration/BuildCalendarSearchIndexBackgroundJob.php',
         'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => __DIR__ . '/..' . '/../lib/Migration/CalDAVRemoveEmptyValue.php',
+        'OCA\\DAV\\Migration\\ChunkCleanup' => __DIR__ . '/..' . '/../lib/Migration/ChunkCleanup.php',
         'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => __DIR__ . '/..' . '/../lib/Migration/FixBirthdayCalendarComponent.php',
         'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => __DIR__ . '/..' . '/../lib/Migration/RefreshWebcalJobRegistrar.php',
         'OCA\\DAV\\Migration\\Version1004Date20170825134824' => __DIR__ . '/..' . '/../lib/Migration/Version1004Date20170825134824.php',
@@ -196,6 +198,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagsRelationsCollection.php',
         'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
         'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
+        'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
         'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
         'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',
         'OCA\\DAV\\Upload\\UploadFolder' => __DIR__ . '/..' . '/../lib/Upload/UploadFolder.php',

apps/dav/lib/BackgroundJob/UploadCleanup.php

@@ -0,0 +1,87 @@
+<?php
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\DAV\BackgroundJob;
+
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJobList;
+use OCP\BackgroundJob\TimedJob;
+use OCP\Files\File;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+
+class UploadCleanup extends TimedJob {
+
+	/** @var IRootFolder */
+	private $rootFolder;
+
+	/** @var IJobList */
+	private $jobList;
+
+	public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) {
+		parent::__construct($time);
+		$this->rootFolder = $rootFolder;
+		$this->jobList = $jobList;
+
+		// Run once a day
+		$this->setInterval(60*60*24);
+	}
+
+	protected function run($argument) {
+		$uid = $argument['uid'];
+		$folder = $argument['folder'];
+
+		$userFolder = $this->rootFolder->getUserFolder($uid);
+		$userRoot = $userFolder->getParent();
+
+		try {
+			/** @var Folder $uploads */
+			$uploads = $userRoot->get('uploads');
+			/** @var Folder $uploadFolder */
+			$uploadFolder = $uploads->get($folder);
+		} catch (NotFoundException $e) {
+			$this->jobList->remove(self::class, $argument);
+			return;
+		}
+
+		$files = $uploadFolder->getDirectoryListing();
+
+		// Remove if all files have an mtime of more than a day
+		$time = $this->time->getTime() - 60 * 60 * 24;
+
+		// The folder has to be more than a day old
+		$initial = $uploadFolder->getMTime() < $time;
+
+		$expire = array_reduce($files, function(bool $carry, File $file) use ($time) {
+			return $carry && $file->getMTime() < $time;
+		}, $initial);
+
+		if ($expire) {
+			$uploadFolder->delete();
+			$this->jobList->remove(self::class, $argument);
+		}
+	}
+
+}

apps/dav/lib/Migration/ChunkCleanup.php

@@ -0,0 +1,95 @@
+<?php
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\DAV\Migration;
+
+use OCA\DAV\BackgroundJob\UploadCleanup;
+use OCP\BackgroundJob\IJobList;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use OCP\IConfig;
+use OCP\IUser;
+use OCP\IUserManager;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+
+class ChunkCleanup implements IRepairStep {
+
+	/** @var IConfig */
+	private $config;
+	/** @var IUserManager */
+	private $userManager;
+	/** @var IRootFolder */
+	private $rootFolder;
+	/** @var IJobList */
+	private $jobList;
+
+	public function __construct(IConfig $config,
+								IUserManager $userManager,
+								IRootFolder $rootFolder,
+								IJobList $jobList) {
+		$this->config = $config;
+		$this->userManager = $userManager;
+		$this->rootFolder = $rootFolder;
+		$this->jobList = $jobList;
+	}
+
+	public function getName(): string {
+		return 'Chunk cleanup scheduler';
+	}
+
+	public function run(IOutput $output) {
+		// If we already ran this onec there is no need to run it again
+		if ($this->config->getAppValue('dav', 'chunks_migrated', '0') === '1') {
+			$output->info('Cleanup not required');
+		}
+
+		$output->startProgress();
+		// Loop over all seen users
+		$this->userManager->callForSeenUsers(function (IUser $user) use ($output) {
+			try {
+				$userFolder = $this->rootFolder->getUserFolder($user->getUID());
+				$userRoot = $userFolder->getParent();
+				/** @var Folder $uploadFolder */
+				$uploadFolder = $userRoot->get('uploads');
+			} catch (NotFoundException $e) {
+				// No folder so skipping
+				return;
+			}
+
+			// Insert a cleanup job for each folder we find
+			$uploads = $uploadFolder->getDirectoryListing();
+			foreach ($uploads as $upload) {
+				$this->jobList->add(UploadCleanup::class, ['uid' => $user->getUID(), 'folder' => $upload->getName()]);
+			}
+			$output->advance();
+		});
+		$output->finishProgress();
+
+
+		$this->config->setAppValue('dav', 'chunks_migrated', '1');
+	}
+
+}

apps/dav/lib/RootCollection.php

@@ -35,6 +35,7 @@ use OCA\DAV\Connector\Sabre\Principal;
 use OCA\DAV\DAV\GroupPrincipalBackend;
 use OCA\DAV\DAV\SystemPrincipalBackend;
 use OCA\DAV\CalDAV\Principal\Collection;
+use OCA\DAV\Upload\CleanupService;
 use Sabre\DAV\SimpleCollection;
 
 class RootCollection extends SimpleCollection {
@@ -120,7 +121,10 @@ class RootCollection extends SimpleCollection {
 		$systemAddressBookRoot = new AddressBookRoot(new SystemPrincipalBackend(), $systemCardDavBackend, 'principals/system');
 		$systemAddressBookRoot->disableListing = $disableListing;
 
-		$uploadCollection = new Upload\RootCollection($userPrincipalBackend, 'principals/users');
+		$uploadCollection = new Upload\RootCollection(
+			$userPrincipalBackend,
+			'principals/users',
+			\OC::$server->query(CleanupService::class));
 		$uploadCollection->disableListing = $disableListing;
 
 		$avatarCollection = new Avatars\RootCollection($userPrincipalBackend, 'principals/users');

apps/dav/lib/Upload/CleanupService.php

@@ -0,0 +1,49 @@
+<?php
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @author Roeland Jago Douma <roeland@famdouma.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\DAV\Upload;
+
+use OCA\DAV\BackgroundJob\UploadCleanup;
+use OCP\BackgroundJob\IJobList;
+use OCP\IUserSession;
+
+class CleanupService {
+	/** @var IUserSession */
+	private $userSession;
+	/** @var IJobList */
+	private $jobList;
+
+	public function __construct(IUserSession $userSession, IJobList $jobList) {
+		$this->userSession = $userSession;
+		$this->jobList = $jobList;
+	}
+
+	public function addJob(string $folder) {
+		$this->jobList->add(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]);
+	}
+
+	public function removeJob(string $folder) {
+		$this->jobList->remove(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]);
+	}
+}

apps/dav/lib/Upload/RootCollection.php

@@ -1,4 +1,5 @@
 <?php
+declare(strict_types=1);
 /**
  * @copyright Copyright (c) 2016, ownCloud, Inc.
  *
@@ -23,20 +24,31 @@
 namespace OCA\DAV\Upload;
 
 use Sabre\DAVACL\AbstractPrincipalCollection;
+use Sabre\DAVACL\PrincipalBackend;
 
 class RootCollection extends AbstractPrincipalCollection {
 
-	/**
+	/** @var CleanupService */
-	 * @inheritdoc
+	private $cleanupService;
-	 */
+
-	function getChildForPrincipal(array $principalInfo) {
+	public function __construct(PrincipalBackend\BackendInterface $principalBackend,
-		return new UploadHome($principalInfo);
+								string $principalPrefix,
+								CleanupService $cleanupService) {
+		parent::__construct($principalBackend, $principalPrefix);
+		$this->cleanupService = $cleanupService;
 	}
 
 	/**
 	 * @inheritdoc
 	 */
-	function getName() {
+	public function getChildForPrincipal(array $principalInfo): UploadHome {
+		return new UploadHome($principalInfo, $this->cleanupService);
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	public function getName(): string {
 		return 'uploads';
 	}
 

apps/dav/lib/Upload/UploadFolder.php

@@ -22,16 +22,22 @@
  */
 namespace OCA\DAV\Upload;
 
+use OCA\DAV\BackgroundJob\UploadCleanup;
 use OCA\DAV\Connector\Sabre\Directory;
+use OCP\BackgroundJob\IJobList;
 use Sabre\DAV\Exception\Forbidden;
 use Sabre\DAV\ICollection;
 
 class UploadFolder implements ICollection {
 
+	/** @var Directory */
 	private $node;
+	/** @var CleanupService */
+	private $cleanupService;
 
-	function __construct(Directory $node) {
+	function __construct(Directory $node, CleanupService $cleanupService) {
 		$this->node = $node;
+		$this->cleanupService = $cleanupService;
 	}
 
 	function createFile($name, $data = null) {
@@ -65,6 +71,9 @@ class UploadFolder implements ICollection {
 
 	function delete() {
 		$this->node->delete();
+
+		// Background cleanup job is not needed anymore
+		$this->cleanupService->removeJob($this->getName());
 	}
 
 	function getName() {

apps/dav/lib/Upload/UploadHome.php

@@ -30,51 +30,56 @@ use Sabre\DAV\Exception\Forbidden;
 use Sabre\DAV\ICollection;
 
 class UploadHome implements ICollection {
-	/**
+
-	 * UploadHome constructor.
+	/** @var array */
-	 *
+	private $principalInfo;
-	 * @param array $principalInfo
+	/** @var CleanupService */
-	 */
+	private $cleanupService;
-	public function __construct($principalInfo) {
+
+	public function __construct(array $principalInfo, CleanupService $cleanupService) {
 		$this->principalInfo = $principalInfo;
+		$this->cleanupService = $cleanupService;
 	}
 
-	function createFile($name, $data = null) {
+	public function createFile($name, $data = null) {
 		throw new Forbidden('Permission denied to create file (filename ' . $name . ')');
 	}
 
-	function createDirectory($name) {
+	public function createDirectory($name) {
 		$this->impl()->createDirectory($name);
+
+		// Add a cleanup job
+		$this->cleanupService->addJob($name);
 	}
 
-	function getChild($name) {
+	public function getChild($name): UploadFolder {
-		return new UploadFolder($this->impl()->getChild($name));
+		return new UploadFolder($this->impl()->getChild($name), $this->cleanupService);
 	}
 
-	function getChildren() {
+	public function getChildren(): array {
 		return array_map(function($node) {
-			return new UploadFolder($node);
+			return new UploadFolder($node, $this->cleanupService);
 		}, $this->impl()->getChildren());
 	}
 
-	function childExists($name) {
+	public function childExists($name): bool {
 		return !is_null($this->getChild($name));
 	}
 
-	function delete() {
+	public function delete() {
 		$this->impl()->delete();
 	}
 
-	function getName() {
+	public function getName() {
 		list(,$name) = \Sabre\Uri\split($this->principalInfo['uri']);
 		return $name;
 	}
 
-	function setName($name) {
+	public function setName($name) {
 		throw new Forbidden('Permission denied to rename this folder');
 	}
 
-	function getLastModified() {
+	public function getLastModified() {
 		return $this->impl()->getLastModified();
 	}