diff --git a/apps/files_external/appinfo/register_command.php b/apps/files_external/appinfo/register_command.php index a5163f96d5..629970722d 100644 --- a/apps/files_external/appinfo/register_command.php +++ b/apps/files_external/appinfo/register_command.php @@ -31,6 +31,7 @@ use OCA\Files_External\Command\Delete; use OCA\Files_External\Command\Create; use OCA\Files_External\Command\Backends; use OCA\Files_External\Command\Verify; +use OCA\Files_External\Command\Notify; $userManager = OC::$server->getUserManager(); $userSession = OC::$server->getUserSession(); @@ -42,6 +43,7 @@ $globalStorageService = $app->getContainer()->query('\OCA\Files_External\Service $userStorageService = $app->getContainer()->query('\OCA\Files_External\Service\UserStoragesService'); $importLegacyStorageService = $app->getContainer()->query('\OCA\Files_External\Service\ImportLegacyStoragesService'); $backendService = $app->getContainer()->query('OCA\Files_External\Service\BackendService'); +$connection = $app->getContainer()->getServer()->getDatabaseConnection(); /** @var Symfony\Component\Console\Application $application */ $application->add(new ListCommand($globalStorageService, $userStorageService, $userSession, $userManager)); @@ -54,3 +56,4 @@ $application->add(new Delete($globalStorageService, $userStorageService, $userSe $application->add(new Create($globalStorageService, $userStorageService, $userManager, $userSession, $backendService)); $application->add(new Backends($backendService)); $application->add(new Verify($globalStorageService)); +$application->add(new Notify($globalStorageService, $connection)); diff --git a/apps/files_external/lib/Command/Notify.php b/apps/files_external/lib/Command/Notify.php new file mode 100644 index 0000000000..63d027b677 --- /dev/null +++ b/apps/files_external/lib/Command/Notify.php @@ -0,0 +1,173 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Files_External\Command; + +use OC\Core\Command\Base; +use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException; +use OCA\Files_External\Lib\StorageConfig; +use OCA\Files_External\Service\GlobalStoragesService; +use OCP\Files\Storage\INotifyStorage; +use OCP\Files\StorageNotAvailableException; +use OCP\IDBConnection; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Notify extends Base { + /** @var GlobalStoragesService */ + private $globalService; + /** @var IDBConnection */ + private $connection; + /** @var \OCP\DB\QueryBuilder\IQueryBuilder */ + private $updateQuery; + + function __construct(GlobalStoragesService $globalService, IDBConnection $connection) { + parent::__construct(); + $this->globalService = $globalService; + $this->connection = $connection; + // the query builder doesn't really like subqueries with parameters + $this->updateQuery = $this->connection->prepare( + 'UPDATE *PREFIX*filecache SET size = -1 + WHERE `path` = ? + AND `storage` IN (SELECT storage_id FROM *PREFIX*mounts WHERE mount_id = ?)' + ); + } + + protected function configure() { + $this + ->setName('files_external:notify') + ->setDescription('Listen for active update notifications for a configured external mount') + ->addArgument( + 'mount_id', + InputArgument::REQUIRED, + 'the mount id of the mount to listen to' + )->addOption( + 'user', + 'u', + InputOption::VALUE_REQUIRED, + 'The username for the remote mount (required only for some mount configuration that don\'t store credentials)' + )->addOption( + 'password', + 'p', + InputOption::VALUE_REQUIRED, + 'The password for the remote mount (required only for some mount configuration that don\'t store credentials)' + )->addOption( + 'path', + null, + InputOption::VALUE_REQUIRED, + 'The directory in the storage to listen for updates in', + '/' + ); + parent::configure(); + } + + protected function execute(InputInterface $input, OutputInterface $output) { + $mount = $this->globalService->getStorage($input->getArgument('mount_id')); + if (is_null($mount)) { + $output->writeln('Mount not found'); + return 1; + } + $noAuth = false; + try { + $authBackend = $mount->getAuthMechanism(); + $authBackend->manipulateStorageConfig($mount); + } catch (InsufficientDataForMeaningfulAnswerException $e) { + $noAuth = true; + } catch (StorageNotAvailableException $e) { + $noAuth = true; + } + + if ($input->getOption('user')) { + $mount->setBackendOption('user', $input->getOption('user')); + } + if ($input->getOption('password')) { + $mount->setBackendOption('password', $input->getOption('password')); + } + + try { + $storage = $this->createStorage($mount); + } catch (\Exception $e) { + $output->writeln('Error while trying to create storage'); + if ($noAuth) { + $output->writeln('Username and/or password required'); + } + return 1; + } + if (!$storage instanceof INotifyStorage) { + $output->writeln('Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications'); + return 1; + } + + $verbose = $input->getOption('verbose'); + + $path = trim($input->getOption('path'), '/'); + $storage->listen($path, function ($type, $path, $renameTarget) use ($mount, $verbose, $output) { + if ($verbose) { + $this->logUpdate($type, $path, $renameTarget, $output); + } + if ($type == INotifyStorage::NOTIFY_RENAMED) { + $this->markParentAsOutdated($mount->getId(), $renameTarget); + } + $this->markParentAsOutdated($mount->getId(), $path); + }); + } + + private function createStorage(StorageConfig $mount) { + $class = $mount->getBackend()->getStorageClass(); + return new $class($mount->getBackendOptions()); + } + + private function markParentAsOutdated($mountId, $path) { + $parent = dirname($path); + if ($parent === '.') { + $parent = ''; + } + $this->updateQuery->execute([$parent, $mountId]); + } + + private function logUpdate($type, $path, $renameTarget, OutputInterface $output) { + switch ($type) { + case INotifyStorage::NOTIFY_ADDED: + $text = 'added'; + break; + case INotifyStorage::NOTIFY_MODIFIED: + $text = 'modified'; + break; + case INotifyStorage::NOTIFY_REMOVED: + $text = 'removed'; + break; + case INotifyStorage::NOTIFY_RENAMED: + $text = 'renamed'; + break; + default: + return; + } + + $text .= ' ' . $path; + if ($type === INotifyStorage::NOTIFY_RENAMED) { + $text .= ' to ' . $renameTarget; + } + + $output->writeln($text); + } +} diff --git a/apps/files_external/lib/Config/ConfigAdapter.php b/apps/files_external/lib/Config/ConfigAdapter.php index 48a521a76f..9375ff74c5 100644 --- a/apps/files_external/lib/Config/ConfigAdapter.php +++ b/apps/files_external/lib/Config/ConfigAdapter.php @@ -151,7 +151,8 @@ class ConfigAdapter implements IMountProvider { '/' . $user->getUID() . '/files' . $storage->getMountPoint(), null, $loader, - $storage->getMountOptions() + $storage->getMountOptions(), + $storage->getId() ); $mounts[$storage->getMountPoint()] = $mount; } diff --git a/apps/files_external/lib/Lib/Storage/SMB.php b/apps/files_external/lib/Lib/Storage/SMB.php index e677f8c9eb..6a24c6b17d 100644 --- a/apps/files_external/lib/Lib/Storage/SMB.php +++ b/apps/files_external/lib/Lib/Storage/SMB.php @@ -34,15 +34,18 @@ use Icewind\SMB\Exception\ConnectException; use Icewind\SMB\Exception\Exception; use Icewind\SMB\Exception\ForbiddenException; use Icewind\SMB\Exception\NotFoundException; +use Icewind\SMB\IShare; use Icewind\SMB\NativeServer; use Icewind\SMB\Server; use Icewind\Streams\CallbackWrapper; use Icewind\Streams\IteratorDirectory; use OC\Cache\CappedMemoryCache; use OC\Files\Filesystem; +use OC\Files\Storage\Common; +use OCP\Files\Storage\INotifyStorage; use OCP\Files\StorageNotAvailableException; -class SMB extends \OC\Files\Storage\Common { +class SMB extends Common implements INotifyStorage { /** * @var \Icewind\SMB\Server */ @@ -103,6 +106,16 @@ class SMB extends \OC\Files\Storage\Common { return Filesystem::normalizePath($this->root . '/' . $path, true, false, true); } + protected function relativePath($fullPath) { + if ($fullPath === $this->root) { + return ''; + } else if (substr($fullPath, 0, strlen($this->root)) === $this->root) { + return substr($fullPath, strlen($this->root)); + } else { + return null; + } + } + /** * @param string $path * @return \Icewind\SMB\IFileInfo @@ -413,4 +426,48 @@ class SMB extends \OC\Files\Storage\Common { return false; } } + + public function listen($path, callable $callback) { + $fullPath = $this->buildPath($path); + $oldRenamePath = null; + $this->share->notify($fullPath, function ($smbType, $fullPath) use (&$oldRenamePath, $callback) { + $path = $this->relativePath($fullPath); + if (is_null($path)) { + return true; + } + if ($smbType === IShare::NOTIFY_RENAMED_OLD) { + $oldRenamePath = $path; + return true; + } + $type = $this->mapNotifyType($smbType); + if (is_null($type)) { + return true; + } + if ($type === INotifyStorage::NOTIFY_RENAMED && !is_null($oldRenamePath)) { + $result = $callback($type, $path, $oldRenamePath); + $oldRenamePath = null; + } else { + $result = $callback($type, $path); + } + return $result; + }); + } + + private function mapNotifyType($smbType) { + switch ($smbType) { + case IShare::NOTIFY_ADDED: + return INotifyStorage::NOTIFY_ADDED; + case IShare::NOTIFY_REMOVED: + return INotifyStorage::NOTIFY_REMOVED; + case IShare::NOTIFY_MODIFIED: + case IShare::NOTIFY_ADDED_STREAM: + case IShare::NOTIFY_MODIFIED_STREAM: + case IShare::NOTIFY_REMOVED_STREAM: + return INotifyStorage::NOTIFY_MODIFIED; + case IShare::NOTIFY_RENAMED_NEW: + return INotifyStorage::NOTIFY_RENAMED; + default: + return null; + } + } } diff --git a/db_structure.xml b/db_structure.xml index 6b91c3c4c5..1127f0d82d 100644 --- a/db_structure.xml +++ b/db_structure.xml @@ -170,6 +170,11 @@ 4000 + + mount_id + integer + + mounts_user_index false @@ -197,6 +202,15 @@ + + mounts_mount_id_index + false + + mount_id + ascending + + + mounts_user_root_index true diff --git a/lib/private/Files/Config/CachedMountInfo.php b/lib/private/Files/Config/CachedMountInfo.php index ce75cb6689..b81cd11a1c 100644 --- a/lib/private/Files/Config/CachedMountInfo.php +++ b/lib/private/Files/Config/CachedMountInfo.php @@ -47,6 +47,11 @@ class CachedMountInfo implements ICachedMountInfo { */ protected $mountPoint; + /** + * @var int|null + */ + protected $mountId; + /** * CachedMountInfo constructor. * @@ -54,12 +59,14 @@ class CachedMountInfo implements ICachedMountInfo { * @param int $storageId * @param int $rootId * @param string $mountPoint + * @param int|null $mountId */ - public function __construct(IUser $user, $storageId, $rootId, $mountPoint) { + public function __construct(IUser $user, $storageId, $rootId, $mountPoint, $mountId = null) { $this->user = $user; $this->storageId = $storageId; $this->rootId = $rootId; $this->mountPoint = $mountPoint; + $this->mountId = $mountId; } /** @@ -104,4 +111,14 @@ class CachedMountInfo implements ICachedMountInfo { public function getMountPoint() { return $this->mountPoint; } + + /** + * Get the id of the configured mount + * + * @return int|null mount id or null if not applicable + * @since 9.1.0 + */ + public function getMountId() { + return $this->mountId; + } } diff --git a/lib/private/Files/Config/LazyStorageMountInfo.php b/lib/private/Files/Config/LazyStorageMountInfo.php index 5df04c4b78..58f288135e 100644 --- a/lib/private/Files/Config/LazyStorageMountInfo.php +++ b/lib/private/Files/Config/LazyStorageMountInfo.php @@ -75,4 +75,8 @@ class LazyStorageMountInfo extends CachedMountInfo { } return parent::getMountPoint(); } + + public function getMountId() { + return $this->mount->getMountId(); + } } diff --git a/lib/private/Files/Config/UserMountCache.php b/lib/private/Files/Config/UserMountCache.php index bc6ad1b34f..d9b538a002 100644 --- a/lib/private/Files/Config/UserMountCache.php +++ b/lib/private/Files/Config/UserMountCache.php @@ -112,13 +112,7 @@ class UserMountCache implements IUserMountCache { /** @var ICachedMountInfo[] $removedMounts */ $removedMounts = array_udiff($cachedMounts, $newMounts, $mountDiff); - $changedMounts = array_uintersect($newMounts, $cachedMounts, function (ICachedMountInfo $mount1, ICachedMountInfo $mount2) { - // filter mounts with the same root id and different mountpoints - if ($mount1->getRootId() !== $mount2->getRootId()) { - return -1; - } - return ($mount1->getMountPoint() !== $mount2->getMountPoint()) ? 0 : 1; - }); + $changedMounts = $this->findChangedMounts($newMounts, $cachedMounts); foreach ($addedMounts as $mount) { $this->addToCache($mount); @@ -130,28 +124,50 @@ class UserMountCache implements IUserMountCache { unset($this->mountsForUsers[$user->getUID()][$index]); } foreach ($changedMounts as $mount) { - $this->setMountPoint($mount); + $this->updateCachedMount($mount); } } + /** + * @param ICachedMountInfo[] $newMounts + * @param ICachedMountInfo[] $cachedMounts + * @return ICachedMountInfo[] + */ + private function findChangedMounts(array $newMounts, array $cachedMounts) { + $changed = []; + foreach ($newMounts as $newMount) { + foreach ($cachedMounts as $cachedMount) { + if ( + $newMount->getRootId() === $cachedMount->getRootId() && + ($newMount->getMountPoint() !== $cachedMount->getMountPoint() || $newMount->getMountId() !== $cachedMount->getMountId()) + ) { + $changed[] = $newMount; + } + } + } + return $changed; + } + private function addToCache(ICachedMountInfo $mount) { if ($mount->getStorageId() !== -1) { $this->connection->insertIfNotExist('*PREFIX*mounts', [ 'storage_id' => $mount->getStorageId(), 'root_id' => $mount->getRootId(), 'user_id' => $mount->getUser()->getUID(), - 'mount_point' => $mount->getMountPoint() + 'mount_point' => $mount->getMountPoint(), + 'mount_id' => $mount->getMountId() ], ['root_id', 'user_id']); } else { $this->logger->error('Error getting storage info for mount at ' . $mount->getMountPoint()); } } - private function setMountPoint(ICachedMountInfo $mount) { + private function updateCachedMount(ICachedMountInfo $mount) { $builder = $this->connection->getQueryBuilder(); $query = $builder->update('mounts') ->set('mount_point', $builder->createNamedParameter($mount->getMountPoint())) + ->set('mount_id', $builder->createNamedParameter($mount->getMountId(), IQueryBuilder::PARAM_INT)) ->where($builder->expr()->eq('user_id', $builder->createNamedParameter($mount->getUser()->getUID()))) ->andWhere($builder->expr()->eq('root_id', $builder->createNamedParameter($mount->getRootId(), IQueryBuilder::PARAM_INT))); @@ -169,7 +185,7 @@ class UserMountCache implements IUserMountCache { private function dbRowToMountInfo(array $row) { $user = $this->userManager->get($row['user_id']); - return new CachedMountInfo($user, (int)$row['storage_id'], (int)$row['root_id'], $row['mount_point']); + return new CachedMountInfo($user, (int)$row['storage_id'], (int)$row['root_id'], $row['mount_point'], $row['mount_id']); } /** @@ -179,7 +195,7 @@ class UserMountCache implements IUserMountCache { public function getMountsForUser(IUser $user) { if (!isset($this->mountsForUsers[$user->getUID()])) { $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point') + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id') ->from('mounts') ->where($builder->expr()->eq('user_id', $builder->createPositionalParameter($user->getUID()))); @@ -196,7 +212,7 @@ class UserMountCache implements IUserMountCache { */ public function getMountsForStorageId($numericStorageId) { $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point') + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id') ->from('mounts') ->where($builder->expr()->eq('storage_id', $builder->createPositionalParameter($numericStorageId, IQueryBuilder::PARAM_INT))); @@ -211,7 +227,7 @@ class UserMountCache implements IUserMountCache { */ public function getMountsForRootId($rootFileId) { $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point') + $query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id') ->from('mounts') ->where($builder->expr()->eq('root_id', $builder->createPositionalParameter($rootFileId, IQueryBuilder::PARAM_INT))); diff --git a/lib/private/Files/Mount/MountPoint.php b/lib/private/Files/Mount/MountPoint.php index e11da9e5c7..f76e815105 100644 --- a/lib/private/Files/Mount/MountPoint.php +++ b/lib/private/Files/Mount/MountPoint.php @@ -68,14 +68,19 @@ class MountPoint implements IMountPoint { */ private $invalidStorage = false; + /** @var int|null */ + protected $mountId; + /** * @param string|\OC\Files\Storage\Storage $storage * @param string $mountpoint * @param array $arguments (optional) configuration for the storage backend * @param \OCP\Files\Storage\IStorageFactory $loader * @param array $mountOptions mount specific options + * @param int|null $mountId + * @throws \Exception */ - public function __construct($storage, $mountpoint, $arguments = null, $loader = null, $mountOptions = null) { + public function __construct($storage, $mountpoint, $arguments = null, $loader = null, $mountOptions = null, $mountId = null) { if (is_null($arguments)) { $arguments = array(); } @@ -102,6 +107,7 @@ class MountPoint implements IMountPoint { $this->class = $storage; $this->arguments = $arguments; } + $this->mountId = $mountId; } /** @@ -249,4 +255,8 @@ class MountPoint implements IMountPoint { public function getStorageRootId() { return (int)$this->getStorage()->getCache()->getId(''); } + + public function getMountId() { + return $this->mountId; + } } diff --git a/lib/public/Files/Config/ICachedMountInfo.php b/lib/public/Files/Config/ICachedMountInfo.php index e09c1a7f01..24c0965421 100644 --- a/lib/public/Files/Config/ICachedMountInfo.php +++ b/lib/public/Files/Config/ICachedMountInfo.php @@ -59,4 +59,12 @@ interface ICachedMountInfo { * @since 9.0.0 */ public function getMountPoint(); + + /** + * Get the id of the configured mount + * + * @return int|null mount id or null if not applicable + * @since 9.1.0 + */ + public function getMountId(); } diff --git a/lib/public/Files/Mount/IMountPoint.php b/lib/public/Files/Mount/IMountPoint.php index bc7bf81709..824b60a102 100644 --- a/lib/public/Files/Mount/IMountPoint.php +++ b/lib/public/Files/Mount/IMountPoint.php @@ -102,4 +102,12 @@ interface IMountPoint { * @since 9.1.0 */ public function getStorageRootId(); + + /** + * Get the id of the configured mount + * + * @return int|null mount id or null if not applicable + * @since 9.1.0 + */ + public function getMountId(); } diff --git a/lib/public/Files/Storage/INotifyStorage.php b/lib/public/Files/Storage/INotifyStorage.php new file mode 100644 index 0000000000..0ebfd689b8 --- /dev/null +++ b/lib/public/Files/Storage/INotifyStorage.php @@ -0,0 +1,51 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\Files\Storage; + +/** + * Storage backend that support active notifications + * + * @since 9.1.0 + */ +interface INotifyStorage { + const NOTIFY_ADDED = 1; + const NOTIFY_REMOVED = 2; + const NOTIFY_MODIFIED = 3; + const NOTIFY_RENAMED = 4; + + /** + * Start listening for update notifications + * + * The provided callback will be called for every incoming notification with the following parameters + * - int $type the type of update, one of the INotifyStorage::NOTIFY_* constants + * - string $path the path of the update + * - string $renameTarget the target of the rename operation, only provided for rename updates + * + * Note that this call is blocking and will not exit on it's own, to stop listening for notifications return `false` from the callback + * + * @param string $path + * @param callable $callback + * + * @since 9.1.0 + */ + public function listen($path, callable $callback); +} diff --git a/tests/lib/Files/Config/UserMountCacheTest.php b/tests/lib/Files/Config/UserMountCacheTest.php index e7554fc36d..b9e09687c9 100644 --- a/tests/lib/Files/Config/UserMountCacheTest.php +++ b/tests/lib/Files/Config/UserMountCacheTest.php @@ -163,12 +163,14 @@ class UserMountCacheTest extends TestCase { $user = $this->userManager->get('u1'); $storage = $this->getStorage(10, 20); - $mount = new MountPoint($storage, '/foo/'); + $mount = new MountPoint($storage, '/bar/'); $this->cache->registerMounts($user, [$mount]); $this->clearCache(); + $mount = new MountPoint($storage, '/foo/'); + $this->cache->registerMounts($user, [$mount]); $this->clearCache(); @@ -180,6 +182,29 @@ class UserMountCacheTest extends TestCase { $this->assertEquals('/foo/', $cachedMount->getMountPoint()); } + public function testChangeMountId() { + $user = $this->userManager->get('u1'); + + $storage = $this->getStorage(10, 20); + $mount = new MountPoint($storage, '/foo/', null, null, null, null); + + $this->cache->registerMounts($user, [$mount]); + + $this->clearCache(); + + $mount = new MountPoint($storage, '/foo/', null, null, null, 1); + + $this->cache->registerMounts($user, [$mount]); + + $this->clearCache(); + + $cachedMounts = $this->cache->getMountsForUser($user); + + $this->assertCount(1, $cachedMounts); + $cachedMount = $cachedMounts[0]; + $this->assertEquals(1, $cachedMount->getMountId()); + } + public function testGetMountsForUser() { $user1 = $this->userManager->get('u1'); $user2 = $this->userManager->get('u2'); diff --git a/version.php b/version.php index 86fb9bbf54..15400c4f4e 100644 --- a/version.php +++ b/version.php @@ -25,7 +25,7 @@ // We only can count up. The 4. digit is only for the internal patchlevel to trigger DB upgrades // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version = array(9, 1, 0, 11); +$OC_Version = array(9, 1, 0, 12); // The human readable string $OC_VersionString = '9.1.0 RC1';