Merge pull request #2152 from nextcloud/preview_cleanupjob

Adds background job to cleanup all previews.
This commit is contained in:
Morris Jobke 2016-11-30 10:39:21 +01:00 committed by GitHub
commit 62ec31eb7b
6 changed files with 583 additions and 0 deletions

View File

@ -36,6 +36,7 @@ use OC\Repair\CleanTags;
use OC\Repair\Collation;
use OC\Repair\DropOldJobs;
use OC\Repair\MoveUpdaterStepFile;
use OC\Repair\NC11\CleanPreviews;
use OC\Repair\NC11\MoveAvatars;
use OC\Repair\OldGroupMembershipShares;
use OC\Repair\RemoveGetETagEntries;
@ -155,6 +156,11 @@ class Repair implements IOutput{
\OC::$server->getJobList(),
\OC::$server->getSystemConfig()
),
new CleanPreviews(
\OC::$server->getJobList(),
\OC::$server->getUserManager(),
\OC::$server->getConfig()
),
];
}

View File

@ -0,0 +1,73 @@
<?php
/**
* @copyright 2016 Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Repair\NC11;
use OCP\BackgroundJob\IJobList;
use OCP\IConfig;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Migration\IOutput;
use OCP\Migration\IRepairStep;
class CleanPreviews implements IRepairStep {
/** @var IJobList */
private $jobList;
/** @var IUserManager */
private $userManager;
/** @var IConfig */
private $config;
/**
* MoveAvatars constructor.
*
* @param IJobList $jobList
* @param IUserManager $userManager
* @param IConfig $config
*/
public function __construct(IJobList $jobList,
IUserManager $userManager,
IConfig $config) {
$this->jobList = $jobList;
$this->userManager = $userManager;
$this->config = $config;
}
/**
* @return string
*/
public function getName() {
return 'Add preview cleanup background jobs';
}
public function run(IOutput $output) {
if (!$this->config->getAppValue('core', 'previewsCleanedUp', false)) {
$this->userManager->callForSeenUsers(function (IUser $user) {
$this->jobList->add(CleanPreviewsBackgroundJob::class, ['uid' => $user->getUID()]);
});
$this->config->setAppValue('core', 'previewsCleanedUp', 1);
}
}
}

View File

@ -0,0 +1,121 @@
<?php
/**
* @copyright 2016 Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Repair\NC11;
use OC\BackgroundJob\QueuedJob;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\ILogger;
class CleanPreviewsBackgroundJob extends QueuedJob {
/** @var IRootFolder */
private $rootFolder;
/** @var ILogger */
private $logger;
/** @var IJobList */
private $jobList;
/** @var ITimeFactory */
private $timeFactory;
/**
* CleanPreviewsBackgroundJob constructor.
*
* @param IRootFolder $rootFolder
* @param ILogger $logger
* @param IJobList $jobList
* @param ITimeFactory $timeFactory
*/
public function __construct(IRootFolder $rootFolder,
ILogger $logger,
IJobList $jobList,
ITimeFactory $timeFactory) {
$this->rootFolder = $rootFolder;
$this->logger = $logger;
$this->jobList = $jobList;
$this->timeFactory = $timeFactory;
}
public function run($arguments) {
$uid = $arguments['uid'];
$this->logger->info('Started preview cleanup for ' . $uid);
$empty = $this->cleanupPreviews($uid);
if (!$empty) {
$this->jobList->add(self::class, ['uid' => $uid]);
$this->logger->info('New preview cleanup scheduled for ' . $uid);
} else {
$this->logger->info('Preview cleanup done for ' . $uid);
}
}
/**
* @param $uid
* @return bool
*/
private function cleanupPreviews($uid) {
try {
$userFolder = $this->rootFolder->getUserFolder($uid);
} catch (NotFoundException $e) {
return true;
}
$userRoot = $userFolder->getParent();
try {
/** @var Folder $thumbnailFolder */
$thumbnailFolder = $userRoot->get('thumbnails');
} catch (NotFoundException $e) {
return true;
}
$thumbnails = $thumbnailFolder->getDirectoryListing();
$start = $this->timeFactory->getTime();
foreach ($thumbnails as $thumbnail) {
try {
$thumbnail->delete();
} catch (NotPermittedException $e) {
// Ignore
}
if (($this->timeFactory->getTime() - $start) > 15) {
return false;
}
}
try {
$thumbnailFolder->delete();
} catch (NotPermittedException $e) {
// Ignore
}
return true;
}
}

View File

@ -73,6 +73,7 @@ use OC\Lockdown\LockdownManager;
use OC\Mail\Mailer;
use OC\Memcache\ArrayCache;
use OC\Notification\Manager;
use OC\Repair\NC11\CleanPreviewsBackgroundJob;
use OC\RichObjectStrings\Validator;
use OC\Security\Bruteforce\Throttler;
use OC\Security\CertificateManager;
@ -803,9 +804,20 @@ class Server extends ServerContainer implements IServerContainer {
$c->getSystemConfig()
);
});
$this->registerService('LockdownManager', function (Server $c) {
return new LockdownManager();
});
/* To trick DI since we don't extend the DIContainer here */
$this->registerService(CleanPreviewsBackgroundJob::class, function (Server $c) {
return new CleanPreviewsBackgroundJob(
$c->getRootFolder(),
$c->getLogger(),
$c->getJobList(),
new TimeFactory()
);
});
}
/**

View File

@ -0,0 +1,236 @@
<?php
/**
* @copyright 2016, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Repair\NC11;
use OC\Repair\NC11\CleanPreviewsBackgroundJob;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\Files\Folder;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\ILogger;
use Test\TestCase;
class CleanPreviewsBackgroundJobTest extends TestCase {
/** @var IRootFolder|\PHPUnit_Framework_MockObject_MockObject */
private $rootFolder;
/** @var ILogger|\PHPUnit_Framework_MockObject_MockObject */
private $logger;
/** @var IJobList|\PHPUnit_Framework_MockObject_MockObject */
private $jobList;
/** @var ITimeFactory|\PHPUnit_Framework_MockObject_MockObject */
private $timeFactory;
/** @var CleanPreviewsBackgroundJob */
private $job;
public function setUp() {
parent::setUp();
$this->rootFolder = $this->createMock(IRootFolder::class);
$this->logger = $this->createMock(ILogger::class);
$this->jobList = $this->createMock(IJobList::class);
$this->timeFactory = $this->createMock(ITimeFactory::class);
$this->job = new CleanPreviewsBackgroundJob(
$this->rootFolder,
$this->logger,
$this->jobList,
$this->timeFactory);
}
public function testCleanupPreviewsUnfinished() {
$userFolder = $this->createMock(Folder::class);
$userRoot = $this->createMock(Folder::class);
$thumbnailFolder = $this->createMock(Folder::class);
$this->rootFolder->method('getUserFolder')
->with($this->equalTo('myuid'))
->willReturn($userFolder);
$userFolder->method('getParent')->willReturn($userRoot);
$userRoot->method('get')
->with($this->equalTo('thumbnails'))
->willReturn($thumbnailFolder);
$previewFolder1 = $this->createMock(Folder::class);
$previewFolder1->expects($this->once())
->method('delete');
$thumbnailFolder->method('getDirectoryListing')
->willReturn([$previewFolder1]);
$thumbnailFolder->expects($this->never())
->method('delete');
$this->timeFactory->method('getTime')
->will($this->onConsecutiveCalls(100, 200));
$this->jobList->expects($this->once())
->method('add')
->with(
$this->equalTo(CleanPreviewsBackgroundJob::class),
$this->equalTo(['uid' => 'myuid'])
);
$this->logger->expects($this->at(0))
->method('info')
->with($this->equalTo('Started preview cleanup for myuid'));
$this->logger->expects($this->at(1))
->method('info')
->with($this->equalTo('New preview cleanup scheduled for myuid'));
$this->job->run(['uid' => 'myuid']);
}
public function testCleanupPreviewsFinished() {
$userFolder = $this->createMock(Folder::class);
$userRoot = $this->createMock(Folder::class);
$thumbnailFolder = $this->createMock(Folder::class);
$this->rootFolder->method('getUserFolder')
->with($this->equalTo('myuid'))
->willReturn($userFolder);
$userFolder->method('getParent')->willReturn($userRoot);
$userRoot->method('get')
->with($this->equalTo('thumbnails'))
->willReturn($thumbnailFolder);
$previewFolder1 = $this->createMock(Folder::class);
$previewFolder1->expects($this->once())
->method('delete');
$thumbnailFolder->method('getDirectoryListing')
->willReturn([$previewFolder1]);
$this->timeFactory->method('getTime')
->will($this->onConsecutiveCalls(100, 101));
$this->jobList->expects($this->never())
->method('add');
$this->logger->expects($this->at(0))
->method('info')
->with($this->equalTo('Started preview cleanup for myuid'));
$this->logger->expects($this->at(1))
->method('info')
->with($this->equalTo('Preview cleanup done for myuid'));
$thumbnailFolder->expects($this->once())
->method('delete');
$this->job->run(['uid' => 'myuid']);
}
public function testNoUserFolder() {
$this->rootFolder->method('getUserFolder')
->with($this->equalTo('myuid'))
->willThrowException(new NotFoundException());
$this->logger->expects($this->at(0))
->method('info')
->with($this->equalTo('Started preview cleanup for myuid'));
$this->logger->expects($this->at(1))
->method('info')
->with($this->equalTo('Preview cleanup done for myuid'));
$this->job->run(['uid' => 'myuid']);
}
public function testNoThumbnailFolder() {
$userFolder = $this->createMock(Folder::class);
$userRoot = $this->createMock(Folder::class);
$this->rootFolder->method('getUserFolder')
->with($this->equalTo('myuid'))
->willReturn($userFolder);
$userFolder->method('getParent')->willReturn($userRoot);
$userRoot->method('get')
->with($this->equalTo('thumbnails'))
->willThrowException(new NotFoundException());
$this->logger->expects($this->at(0))
->method('info')
->with($this->equalTo('Started preview cleanup for myuid'));
$this->logger->expects($this->at(1))
->method('info')
->with($this->equalTo('Preview cleanup done for myuid'));
$this->job->run(['uid' => 'myuid']);
}
public function testNotPermittedToDelete() {
$userFolder = $this->createMock(Folder::class);
$userRoot = $this->createMock(Folder::class);
$thumbnailFolder = $this->createMock(Folder::class);
$this->rootFolder->method('getUserFolder')
->with($this->equalTo('myuid'))
->willReturn($userFolder);
$userFolder->method('getParent')->willReturn($userRoot);
$userRoot->method('get')
->with($this->equalTo('thumbnails'))
->willReturn($thumbnailFolder);
$previewFolder1 = $this->createMock(Folder::class);
$previewFolder1->expects($this->once())
->method('delete')
->willThrowException(new NotPermittedException());
$thumbnailFolder->method('getDirectoryListing')
->willReturn([$previewFolder1]);
$this->timeFactory->method('getTime')
->will($this->onConsecutiveCalls(100, 101));
$this->jobList->expects($this->never())
->method('add');
$this->logger->expects($this->at(0))
->method('info')
->with($this->equalTo('Started preview cleanup for myuid'));
$this->logger->expects($this->at(1))
->method('info')
->with($this->equalTo('Preview cleanup done for myuid'));
$thumbnailFolder->expects($this->once())
->method('delete')
->willThrowException(new NotPermittedException());
$this->job->run(['uid' => 'myuid']);
}
}

View File

@ -0,0 +1,135 @@
<?php
/**
* @copyright 2016, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace Test\Repair\NC11;
use OC\Repair\NC11\CleanPreviews;
use OC\Repair\NC11\CleanPreviewsBackgroundJob;
use OCP\BackgroundJob\IJobList;
use OCP\IConfig;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Migration\IOutput;
use Test\TestCase;
class CleanPreviewsTest extends TestCase {
/** @var IJobList|\PHPUnit_Framework_MockObject_MockObject */
private $jobList;
/** @var IUserManager|\PHPUnit_Framework_MockObject_MockObject */
private $userManager;
/** @var IConfig|\PHPUnit_Framework_MockObject_MockObject */
private $config;
/** @var CleanPreviews */
private $repair;
public function setUp() {
parent::setUp();
$this->jobList = $this->createMock(IJobList::class);
$this->userManager = $this->createMock(IUserManager::class);
$this->config = $this->createMock(IConfig::class);
$this->repair = new CleanPreviews(
$this->jobList,
$this->userManager,
$this->config
);
}
public function testGetName() {
$this->assertSame('Add preview cleanup background jobs', $this->repair->getName());
}
public function testRun() {
$user1 = $this->createMock(IUser::class);
$user1->method('getUID')
->willReturn('user1');
$user2 = $this->createMock(IUser::class);
$user2->method('getUID')
->willReturn('user2');
$this->userManager->expects($this->once())
->method('callForSeenUsers')
->will($this->returnCallback(function (\Closure $function) use ($user1, $user2) {
$function($user1);
$function($user2);
}));
$this->jobList->expects($this->at(0))
->method('add')
->with(
$this->equalTo(CleanPreviewsBackgroundJob::class),
$this->equalTo(['uid' => 'user1'])
);
$this->jobList->expects($this->at(1))
->method('add')
->with(
$this->equalTo(CleanPreviewsBackgroundJob::class),
$this->equalTo(['uid' => 'user2'])
);
$this->config->expects($this->once())
->method('getAppValue')
->with(
$this->equalTo('core'),
$this->equalTo('previewsCleanedUp'),
$this->equalTo(false)
)->willReturn(false);
$this->config->expects($this->once())
->method('setAppValue')
->with(
$this->equalTo('core'),
$this->equalTo('previewsCleanedUp'),
$this->equalTo(1)
);
$this->repair->run($this->createMock(IOutput::class));
}
public function testRunAlreadyDoone() {
$this->userManager->expects($this->never())
->method($this->anything());
$this->jobList->expects($this->never())
->method($this->anything());
$this->config->expects($this->once())
->method('getAppValue')
->with(
$this->equalTo('core'),
$this->equalTo('previewsCleanedUp'),
$this->equalTo(false)
)->willReturn('1');
$this->config->expects($this->never())
->method('setAppValue');
$this->repair->run($this->createMock(IOutput::class));
}
}