mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #19214 from nextcloud/enh/preview_folder_tree 

Move to subfolders for preview files
This commit is contained in:
Roeland Jago Douma 2020-04-19 11:22:25 +02:00 committed by GitHub
parent 399253a5cf 6c603e8e7d
commit 56a07009bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 224 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/composer/composer/autoload_classmap.php
View File

@ -1153,6 +1153,7 @@ return array(
'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php',
'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php',
'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php',
'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php',
'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php',
'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php',
'OC\\Preview\\Watcher' => $baseDir . '/lib/private/Preview/Watcher.php',

1
lib/composer/composer/autoload_static.php
View File

@ -1182,6 +1182,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php',
'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php',
'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php',
'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php',
'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php',
'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php',
'OC\\Preview\\Watcher' => __DIR__ . '/../../..' . '/lib/private/Preview/Watcher.php',

94
lib/private/Preview/BackgroundCleanupJob.php
View File

@ -27,8 +27,9 @@ declare(strict_types=1);
namespace OC\Preview;
use OC\BackgroundJob\TimedJob;
use OC\Files\AppData\Factory;
use OC\Preview\Storage\Root;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\IDBConnection;
@ -38,28 +39,47 @@ class BackgroundCleanupJob extends TimedJob {
/** @var IDBConnection */
private $connection;
/** @var Factory */
private $appDataFactory;
/** @var Root */
private $previewFolder;
/** @var bool */
private $isCLI;
/** @var IMimeTypeLoader */
private $mimeTypeLoader;
public function __construct(IDBConnection $connection,
Factory $appDataFactory,
Root $previewFolder,
IMimeTypeLoader $mimeTypeLoader,
bool $isCLI) {
// Run at most once an hour
$this->setInterval(3600);
$this->connection = $connection;
$this->appDataFactory = $appDataFactory;
$this->previewFolder = $previewFolder;
$this->isCLI = $isCLI;
$this->mimeTypeLoader = $mimeTypeLoader;
}
public function run($argument) {
$previews = $this->appDataFactory->get('preview');
foreach ($this->getDeletedFiles() as $fileId) {
try {
$preview = $this->previewFolder->getFolder((string)$fileId);
$preview->delete();
} catch (NotFoundException $e) {
// continue
} catch (NotPermittedException $e) {
// continue
}
}
}
$previewFodlerId = $previews->getId();
private function getDeletedFiles(): \Iterator {
yield from $this->getOldPreviewLocations();
yield from $this->getNewPreviewLocations();
}
private function getOldPreviewLocations(): \Iterator {
$qb = $this->connection->getQueryBuilder();
$qb->select('a.name')
->from('filecache', 'a')
@ -69,7 +89,9 @@ class BackgroundCleanupJob extends TimedJob {
->where(
$qb->expr()->isNull('b.fileid')
)->andWhere(
$qb->expr()->eq('a.parent', $qb->createNamedParameter($previewFodlerId))
$qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId()))
)->andWhere(
$qb->expr()->like('a.name', $qb->createNamedParameter('__%'))
);
if (!$this->isCLI) {
@ -79,14 +101,54 @@ class BackgroundCleanupJob extends TimedJob {
$cursor = $qb->execute();
while ($row = $cursor->fetch()) {
try {
$preview = $previews->getFolder($row['name']);
$preview->delete();
} catch (NotFoundException $e) {
// continue
} catch (NotPermittedException $e) {
// continue
}
yield $row['name'];
}
$cursor->closeCursor();
}
private function getNewPreviewLocations(): \Iterator {
$qb = $this->connection->getQueryBuilder();
$qb->select('path', 'mimetype')
->from('filecache')
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId())));
$cursor = $qb->execute();
$data = $cursor->fetch();
$cursor->closeCursor();
if ($data === null) {
return [];
}
/*
* This lovely like is the result of the way the new previews are stored
* We take the md5 of the name (fileid) and split the first 7 chars. That way
* there are not a gazillion files in the root of the preview appdata.
*/
$like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%';
$qb = $this->connection->getQueryBuilder();
$qb->select('a.name')
->from('filecache', 'a')
->leftJoin('a', 'filecache', 'b', $qb->expr()->eq(
$qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid'
))
->where(
$qb->expr()->andX(
$qb->expr()->isNull('b.fileid'),
$qb->expr()->like('a.path', $qb->createNamedParameter($like)),
$qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory')))
)
);
if (!$this->isCLI) {
$qb->setMaxResults(10);
}
$cursor = $qb->execute();
while ($row = $cursor->fetch()) {
yield $row['name'];
}
$cursor->closeCursor();

71
lib/private/Preview/Storage/Root.php Normal file
View File

@ -0,0 +1,71 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2020, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Preview\Storage;
use OC\Files\AppData\AppData;
use OC\SystemConfig;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\Files\SimpleFS\ISimpleFolder;
class Root extends AppData {
public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig) {
parent::__construct($rootFolder, $systemConfig, 'preview');
}
public function getFolder(string $name): ISimpleFolder {
$internalFolder = $this->getInternalFolder($name);
try {
return parent::getFolder($internalFolder);
} catch (NotFoundException $e) {
/*
* The new folder structure is not found.
* Lets try the old one
*/
}
return parent::getFolder($name);
}
public function newFolder(string $name): ISimpleFolder {
$internalFolder = $this->getInternalFolder($name);
return parent::newFolder($internalFolder);
}
/*
* Do not allow directory listing on this special root
* since it gets to big and time consuming
*/
public function getDirectoryListing(): array {
return [];
}
private function getInternalFolder(string $name): string {
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
}
}

4
lib/private/Server.php
View File

@ -281,7 +281,7 @@ class Server extends ServerContainer implements IServerContainer {
return new PreviewManager(
$c->getConfig(),
$c->getRootFolder(),
$c->getAppDataDir('preview'),
new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview'),
$c->getEventDispatcher(),
$c->getGeneratorHelper(),
$c->getSession()->get('user_id')
@ -291,7 +291,7 @@ class Server extends ServerContainer implements IServerContainer {
$this->registerService(\OC\Preview\Watcher::class, function (Server $c) {
return new \OC\Preview\Watcher(
$c->getAppDataDir('preview')
new \OC\Preview\Storage\Root($c->getRootFolder(), $c->getSystemConfig(), 'preview')
);
});

98
tests/lib/Preview/BackgroundCleanupJobTest.php
View File

@ -22,10 +22,13 @@
namespace Test\Preview;
use OC\Files\AppData\Factory;
use OC\Preview\BackgroundCleanupJob;
use OC\Preview\Storage\Root;
use OC\PreviewManager;
use OCP\Files\File;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\IDBConnection;
use Test\Traits\MountProviderTrait;
use Test\Traits\UserTrait;
@ -47,9 +50,6 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
/** @var bool */
private $trashEnabled;
/** @var Factory */
private $appDataFactory;
/** @var IDBConnection */
private $connection;
@ -59,6 +59,9 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
/** @var IRootFolder */
private $rootFolder;
/** @var IMimeTypeLoader */
private $mimeTypeLoader;
protected function setUp(): void {
parent::setUp();
@ -76,13 +79,10 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
$this->trashEnabled = $appManager->isEnabledForUser('files_trashbin', $this->userId);
$appManager->disableApp('files_trashbin');
$this->appDataFactory = new Factory(
\OC::$server->getRootFolder(),
\OC::$server->getSystemConfig()
);
$this->connection = \OC::$server->getDatabaseConnection();
$this->previewManager = \OC::$server->getPreviewManager();
$this->rootFolder = \OC::$server->getRootFolder();
$this->mimeTypeLoader = \OC::$server->getMimeTypeLoader();
}
protected function tearDown(): void {
@ -96,6 +96,13 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
parent::tearDown();
}
private function getRoot(): Root {
return new Root(
\OC::$server->getRootFolder(),
\OC::$server->getSystemConfig()
);
}
private function setup11Previews(): array {
$userFolder = $this->rootFolder->getUserFolder($this->userId);
@ -110,52 +117,89 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
return $files;
}
private function countPreviews(Root $previewRoot, array $fileIds): int {
$i = 0;
foreach ($fileIds as $fileId) {
try {
$previewRoot->getFolder((string)$fileId);
} catch (NotFoundException $e) {
continue;
}
$i++;
}
return $i;
}
public function testCleanupSystemCron() {
$files = $this->setup11Previews();
$fileIds = array_map(function (File $f) {
return $f->getId();
}, $files);
$preview = $this->appDataFactory->get('preview');
$root = $this->getRoot();
$previews = $preview->getDirectoryListing();
$this->assertCount(11, $previews);
$job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, true);
$this->assertSame(11, $this->countPreviews($root, $fileIds));
$job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, true);
$job->run([]);
foreach ($files as $file) {
$file->delete();
}
$this->assertCount(11, $previews);
$root = $this->getRoot();
$this->assertSame(11, $this->countPreviews($root, $fileIds));
$job->run([]);
$previews = $preview->getDirectoryListing();
$this->assertCount(0, $previews);
$root = $this->getRoot();
$this->assertSame(0, $this->countPreviews($root, $fileIds));
}
public function testCleanupAjax() {
$files = $this->setup11Previews();
$fileIds = array_map(function (File $f) {
return $f->getId();
}, $files);
$preview = $this->appDataFactory->get('preview');
$root = $this->getRoot();
$previews = $preview->getDirectoryListing();
$this->assertCount(11, $previews);
$job = new BackgroundCleanupJob($this->connection, $this->appDataFactory, false);
$this->assertSame(11, $this->countPreviews($root, $fileIds));
$job = new BackgroundCleanupJob($this->connection, $root, $this->mimeTypeLoader, false);
$job->run([]);
foreach ($files as $file) {
$file->delete();
}
$this->assertCount(11, $previews);
$root = $this->getRoot();
$this->assertSame(11, $this->countPreviews($root, $fileIds));
$job->run([]);
$previews = $preview->getDirectoryListing();
$this->assertCount(1, $previews);
$root = $this->getRoot();
$this->assertSame(1, $this->countPreviews($root, $fileIds));
$job->run([]);
$previews = $preview->getDirectoryListing();
$this->assertCount(0, $previews);
$root = $this->getRoot();
$this->assertSame(0, $this->countPreviews($root, $fileIds));
}
public function testOldPreviews() {
$appdata = \OC::$server->getAppDataDir('preview');
$f1 = $appdata->newFolder('123456781');
$f1->newFile('foo.jpg', 'foo');
$f2 = $appdata->newFolder('123456782');
$f2->newFile('foo.jpg', 'foo');
$appdata = \OC::$server->getAppDataDir('preview');
$this->assertSame(2, count($appdata->getDirectoryListing()));
$job = new BackgroundCleanupJob($this->connection, $this->getRoot(), $this->mimeTypeLoader, true);
$job->run([]);
$appdata = \OC::$server->getAppDataDir('preview');
$this->assertSame(0, count($appdata->getDirectoryListing()));
}
}