mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #12227 from nextcloud/bugfix/cleanu_upload_folders 

Background job to cleanup leftover chunked uploads
This commit is contained in:
Morris Jobke 2019-01-07 11:19:43 +01:00 committed by GitHub
parent 8f4e6fcea1 adb2b9a2f8
commit 2bd98daaa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 296 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/dav/appinfo/info.xml
View File

@ -5,7 +5,7 @@
<name>WebDAV</name>
<summary>WebDAV endpoint</summary>
<description>WebDAV endpoint</description>
<version>1.9.0</version>
<version>1.9.1</version>
<licence>agpl</licence>
<author>owncloud.org</author>
<namespace>DAV</namespace>
@ -32,6 +32,9 @@
<step>OCA\DAV\Migration\BuildCalendarSearchIndex</step>
<step>OCA\DAV\Migration\RefreshWebcalJobRegistrar</step>
</post-migration>
<live-migration>
<step>OCA\DAV\Migration\ChunkCleanup</step>
</live-migration>
</repair-steps>
<commands>

3
apps/dav/composer/composer/autoload_classmap.php
View File

@ -16,6 +16,7 @@ return array(
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => $baseDir . '/../lib/BackgroundJob/RefreshWebcalJob.php',
'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => $baseDir . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\UploadCleanup' => $baseDir . '/../lib/BackgroundJob/UploadCleanup.php',
'OCA\\DAV\\CalDAV\\Activity\\Backend' => $baseDir . '/../lib/CalDAV/Activity/Backend.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => $baseDir . '/../lib/CalDAV/Activity/Filter/Calendar.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => $baseDir . '/../lib/CalDAV/Activity/Filter/Todo.php',
@ -153,6 +154,7 @@ return array(
'OCA\\DAV\\Migration\\BuildCalendarSearchIndex' => $baseDir . '/../lib/Migration/BuildCalendarSearchIndex.php',
'OCA\\DAV\\Migration\\BuildCalendarSearchIndexBackgroundJob' => $baseDir . '/../lib/Migration/BuildCalendarSearchIndexBackgroundJob.php',
'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => $baseDir . '/../lib/Migration/CalDAVRemoveEmptyValue.php',
'OCA\\DAV\\Migration\\ChunkCleanup' => $baseDir . '/../lib/Migration/ChunkCleanup.php',
'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => $baseDir . '/../lib/Migration/FixBirthdayCalendarComponent.php',
'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => $baseDir . '/../lib/Migration/RefreshWebcalJobRegistrar.php',
'OCA\\DAV\\Migration\\Version1004Date20170825134824' => $baseDir . '/../lib/Migration/Version1004Date20170825134824.php',
@ -181,6 +183,7 @@ return array(
'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => $baseDir . '/../lib/SystemTag/SystemTagsRelationsCollection.php',
'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',
'OCA\\DAV\\Upload\\UploadFolder' => $baseDir . '/../lib/Upload/UploadFolder.php',

3
apps/dav/composer/composer/autoload_static.php
View File

@ -31,6 +31,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/RefreshWebcalJob.php',
'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php',
'OCA\\DAV\\BackgroundJob\\UploadCleanup' => __DIR__ . '/..' . '/../lib/BackgroundJob/UploadCleanup.php',
'OCA\\DAV\\CalDAV\\Activity\\Backend' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Backend.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Calendar.php',
'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Todo.php',
@ -168,6 +169,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\Migration\\BuildCalendarSearchIndex' => __DIR__ . '/..' . '/../lib/Migration/BuildCalendarSearchIndex.php',
'OCA\\DAV\\Migration\\BuildCalendarSearchIndexBackgroundJob' => __DIR__ . '/..' . '/../lib/Migration/BuildCalendarSearchIndexBackgroundJob.php',
'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => __DIR__ . '/..' . '/../lib/Migration/CalDAVRemoveEmptyValue.php',
'OCA\\DAV\\Migration\\ChunkCleanup' => __DIR__ . '/..' . '/../lib/Migration/ChunkCleanup.php',
'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => __DIR__ . '/..' . '/../lib/Migration/FixBirthdayCalendarComponent.php',
'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => __DIR__ . '/..' . '/../lib/Migration/RefreshWebcalJobRegistrar.php',
'OCA\\DAV\\Migration\\Version1004Date20170825134824' => __DIR__ . '/..' . '/../lib/Migration/Version1004Date20170825134824.php',
@ -196,6 +198,7 @@ class ComposerStaticInitDAV
'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagsRelationsCollection.php',
'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',
'OCA\\DAV\\Upload\\UploadFolder' => __DIR__ . '/..' . '/../lib/Upload/UploadFolder.php',

87
apps/dav/lib/BackgroundJob/UploadCleanup.php Normal file
View File

@ -0,0 +1,87 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\BackgroundJob;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\BackgroundJob\TimedJob;
use OCP\Files\File;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
class UploadCleanup extends TimedJob {
/** @var IRootFolder */
private $rootFolder;
/** @var IJobList */
private $jobList;
public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) {
parent::__construct($time);
$this->rootFolder = $rootFolder;
$this->jobList = $jobList;
// Run once a day
$this->setInterval(60*60*24);
}
protected function run($argument) {
$uid = $argument['uid'];
$folder = $argument['folder'];
$userFolder = $this->rootFolder->getUserFolder($uid);
$userRoot = $userFolder->getParent();
try {
/** @var Folder $uploads */
$uploads = $userRoot->get('uploads');
/** @var Folder $uploadFolder */
$uploadFolder = $uploads->get($folder);
} catch (NotFoundException $e) {
$this->jobList->remove(self::class, $argument);
return;
}
$files = $uploadFolder->getDirectoryListing();
// Remove if all files have an mtime of more than a day
$time = $this->time->getTime() - 60 * 60 * 24;
// The folder has to be more than a day old
$initial = $uploadFolder->getMTime() < $time;
$expire = array_reduce($files, function(bool $carry, File $file) use ($time) {
return $carry && $file->getMTime() < $time;
}, $initial);
if ($expire) {
$uploadFolder->delete();
$this->jobList->remove(self::class, $argument);
}
}
}

95
apps/dav/lib/Migration/ChunkCleanup.php Normal file
View File

@ -0,0 +1,95 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Migration;
use OCA\DAV\BackgroundJob\UploadCleanup;
use OCP\BackgroundJob\IJobList;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\IConfig;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Migration\IOutput;
use OCP\Migration\IRepairStep;
class ChunkCleanup implements IRepairStep {
/** @var IConfig */
private $config;
/** @var IUserManager */
private $userManager;
/** @var IRootFolder */
private $rootFolder;
/** @var IJobList */
private $jobList;
public function __construct(IConfig $config,
IUserManager $userManager,
IRootFolder $rootFolder,
IJobList $jobList) {
$this->config = $config;
$this->userManager = $userManager;
$this->rootFolder = $rootFolder;
$this->jobList = $jobList;
}
public function getName(): string {
return 'Chunk cleanup scheduler';
}
public function run(IOutput $output) {
// If we already ran this onec there is no need to run it again
if ($this->config->getAppValue('dav', 'chunks_migrated', '0') === '1') {
$output->info('Cleanup not required');
}
$output->startProgress();
// Loop over all seen users
$this->userManager->callForSeenUsers(function (IUser $user) use ($output) {
try {
$userFolder = $this->rootFolder->getUserFolder($user->getUID());
$userRoot = $userFolder->getParent();
/** @var Folder $uploadFolder */
$uploadFolder = $userRoot->get('uploads');
} catch (NotFoundException $e) {
// No folder so skipping
return;
}
// Insert a cleanup job for each folder we find
$uploads = $uploadFolder->getDirectoryListing();
foreach ($uploads as $upload) {
$this->jobList->add(UploadCleanup::class, ['uid' => $user->getUID(), 'folder' => $upload->getName()]);
}
$output->advance();
});
$output->finishProgress();
$this->config->setAppValue('dav', 'chunks_migrated', '1');
}
}

6
apps/dav/lib/RootCollection.php
View File

@ -35,6 +35,7 @@ use OCA\DAV\Connector\Sabre\Principal;
use OCA\DAV\DAV\GroupPrincipalBackend;
use OCA\DAV\DAV\SystemPrincipalBackend;
use OCA\DAV\CalDAV\Principal\Collection;
use OCA\DAV\Upload\CleanupService;
use Sabre\DAV\SimpleCollection;
class RootCollection extends SimpleCollection {
@ -120,7 +121,10 @@ class RootCollection extends SimpleCollection {
$systemAddressBookRoot = new AddressBookRoot(new SystemPrincipalBackend(), $systemCardDavBackend, 'principals/system');
$systemAddressBookRoot->disableListing = $disableListing;
$uploadCollection = new Upload\RootCollection($userPrincipalBackend, 'principals/users');
$uploadCollection = new Upload\RootCollection(
$userPrincipalBackend,
'principals/users',
\OC::$server->query(CleanupService::class));
$uploadCollection->disableListing = $disableListing;
$avatarCollection = new Avatars\RootCollection($userPrincipalBackend, 'principals/users');

49
apps/dav/lib/Upload/CleanupService.php Normal file
View File

@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2018, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Upload;
use OCA\DAV\BackgroundJob\UploadCleanup;
use OCP\BackgroundJob\IJobList;
use OCP\IUserSession;
class CleanupService {
/** @var IUserSession */
private $userSession;
/** @var IJobList */
private $jobList;
public function __construct(IUserSession $userSession, IJobList $jobList) {
$this->userSession = $userSession;
$this->jobList = $jobList;
}
public function addJob(string $folder) {
$this->jobList->add(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]);
}
public function removeJob(string $folder) {
$this->jobList->remove(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]);
}
}

24
apps/dav/lib/Upload/RootCollection.php
View File

@ -1,4 +1,5 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2016, ownCloud, Inc.
*
@ -23,20 +24,31 @@
namespace OCA\DAV\Upload;
use Sabre\DAVACL\AbstractPrincipalCollection;
use Sabre\DAVACL\PrincipalBackend;
class RootCollection extends AbstractPrincipalCollection {
/**
* @inheritdoc
*/
function getChildForPrincipal(array $principalInfo) {
return new UploadHome($principalInfo);
/** @var CleanupService */
private $cleanupService;
public function __construct(PrincipalBackend\BackendInterface $principalBackend,
string $principalPrefix,
CleanupService $cleanupService) {
parent::__construct($principalBackend, $principalPrefix);
$this->cleanupService = $cleanupService;
}
/**
* @inheritdoc
*/
function getName() {
public function getChildForPrincipal(array $principalInfo): UploadHome {
return new UploadHome($principalInfo, $this->cleanupService);
}
/**
* @inheritdoc
*/
public function getName(): string {
return 'uploads';
}

11
apps/dav/lib/Upload/UploadFolder.php
View File

@ -22,16 +22,22 @@
*/
namespace OCA\DAV\Upload;
use OCA\DAV\BackgroundJob\UploadCleanup;
use OCA\DAV\Connector\Sabre\Directory;
use OCP\BackgroundJob\IJobList;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\ICollection;
class UploadFolder implements ICollection {
/** @var Directory */
private $node;
/** @var CleanupService */
private $cleanupService;
function __construct(Directory $node) {
function __construct(Directory $node, CleanupService $cleanupService) {
$this->node = $node;
$this->cleanupService = $cleanupService;
}
function createFile($name, $data = null) {
@ -65,6 +71,9 @@ class UploadFolder implements ICollection {
function delete() {
$this->node->delete();
// Background cleanup job is not needed anymore
$this->cleanupService->removeJob($this->getName());
}
function getName() {

39
apps/dav/lib/Upload/UploadHome.php
View File

@ -30,51 +30,56 @@ use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\ICollection;
class UploadHome implements ICollection {
/**
* UploadHome constructor.
*
* @param array $principalInfo
*/
public function __construct($principalInfo) {
/** @var array */
private $principalInfo;
/** @var CleanupService */
private $cleanupService;
public function __construct(array $principalInfo, CleanupService $cleanupService) {
$this->principalInfo = $principalInfo;
$this->cleanupService = $cleanupService;
}
function createFile($name, $data = null) {
public function createFile($name, $data = null) {
throw new Forbidden('Permission denied to create file (filename ' . $name . ')');
}
function createDirectory($name) {
public function createDirectory($name) {
$this->impl()->createDirectory($name);
// Add a cleanup job
$this->cleanupService->addJob($name);
}
function getChild($name) {
return new UploadFolder($this->impl()->getChild($name));
public function getChild($name): UploadFolder {
return new UploadFolder($this->impl()->getChild($name), $this->cleanupService);
}
function getChildren() {
public function getChildren(): array {
return array_map(function($node) {
return new UploadFolder($node);
return new UploadFolder($node, $this->cleanupService);
}, $this->impl()->getChildren());
}
function childExists($name) {
public function childExists($name): bool {
return !is_null($this->getChild($name));
}
function delete() {
public function delete() {
$this->impl()->delete();
}
function getName() {
public function getName() {
list(,$name) = \Sabre\Uri\split($this->principalInfo['uri']);
return $name;
}
function setName($name) {
public function setName($name) {
throw new Forbidden('Permission denied to rename this folder');
}
function getLastModified() {
public function getLastModified() {
return $this->impl()->getLastModified();
}