Merge pull request #402 from nextcloud/smb-notifications

smb update notifications
This commit is contained in:
Morris Jobke 2016-07-20 16:19:21 +02:00 committed by GitHub
commit 1264e9644f
14 changed files with 407 additions and 20 deletions

View File

@ -31,6 +31,7 @@ use OCA\Files_External\Command\Delete;
use OCA\Files_External\Command\Create;
use OCA\Files_External\Command\Backends;
use OCA\Files_External\Command\Verify;
use OCA\Files_External\Command\Notify;
$userManager = OC::$server->getUserManager();
$userSession = OC::$server->getUserSession();
@ -42,6 +43,7 @@ $globalStorageService = $app->getContainer()->query('\OCA\Files_External\Service
$userStorageService = $app->getContainer()->query('\OCA\Files_External\Service\UserStoragesService');
$importLegacyStorageService = $app->getContainer()->query('\OCA\Files_External\Service\ImportLegacyStoragesService');
$backendService = $app->getContainer()->query('OCA\Files_External\Service\BackendService');
$connection = $app->getContainer()->getServer()->getDatabaseConnection();
/** @var Symfony\Component\Console\Application $application */
$application->add(new ListCommand($globalStorageService, $userStorageService, $userSession, $userManager));
@ -54,3 +56,4 @@ $application->add(new Delete($globalStorageService, $userStorageService, $userSe
$application->add(new Create($globalStorageService, $userStorageService, $userManager, $userSession, $backendService));
$application->add(new Backends($backendService));
$application->add(new Verify($globalStorageService));
$application->add(new Notify($globalStorageService, $connection));

View File

@ -0,0 +1,173 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files_External\Command;
use OC\Core\Command\Base;
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
use OCA\Files_External\Lib\StorageConfig;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\Files\Storage\INotifyStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IDBConnection;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Notify extends Base {
/** @var GlobalStoragesService */
private $globalService;
/** @var IDBConnection */
private $connection;
/** @var \OCP\DB\QueryBuilder\IQueryBuilder */
private $updateQuery;
function __construct(GlobalStoragesService $globalService, IDBConnection $connection) {
parent::__construct();
$this->globalService = $globalService;
$this->connection = $connection;
// the query builder doesn't really like subqueries with parameters
$this->updateQuery = $this->connection->prepare(
'UPDATE *PREFIX*filecache SET size = -1
WHERE `path` = ?
AND `storage` IN (SELECT storage_id FROM *PREFIX*mounts WHERE mount_id = ?)'
);
}
protected function configure() {
$this
->setName('files_external:notify')
->setDescription('Listen for active update notifications for a configured external mount')
->addArgument(
'mount_id',
InputArgument::REQUIRED,
'the mount id of the mount to listen to'
)->addOption(
'user',
'u',
InputOption::VALUE_REQUIRED,
'The username for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'password',
'p',
InputOption::VALUE_REQUIRED,
'The password for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'path',
null,
InputOption::VALUE_REQUIRED,
'The directory in the storage to listen for updates in',
'/'
);
parent::configure();
}
protected function execute(InputInterface $input, OutputInterface $output) {
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
if (is_null($mount)) {
$output->writeln('<error>Mount not found</error>');
return 1;
}
$noAuth = false;
try {
$authBackend = $mount->getAuthMechanism();
$authBackend->manipulateStorageConfig($mount);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}
if ($input->getOption('user')) {
$mount->setBackendOption('user', $input->getOption('user'));
}
if ($input->getOption('password')) {
$mount->setBackendOption('password', $input->getOption('password'));
}
try {
$storage = $this->createStorage($mount);
} catch (\Exception $e) {
$output->writeln('<error>Error while trying to create storage</error>');
if ($noAuth) {
$output->writeln('<error>Username and/or password required</error>');
}
return 1;
}
if (!$storage instanceof INotifyStorage) {
$output->writeln('<error>Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications</error>');
return 1;
}
$verbose = $input->getOption('verbose');
$path = trim($input->getOption('path'), '/');
$storage->listen($path, function ($type, $path, $renameTarget) use ($mount, $verbose, $output) {
if ($verbose) {
$this->logUpdate($type, $path, $renameTarget, $output);
}
if ($type == INotifyStorage::NOTIFY_RENAMED) {
$this->markParentAsOutdated($mount->getId(), $renameTarget);
}
$this->markParentAsOutdated($mount->getId(), $path);
});
}
private function createStorage(StorageConfig $mount) {
$class = $mount->getBackend()->getStorageClass();
return new $class($mount->getBackendOptions());
}
private function markParentAsOutdated($mountId, $path) {
$parent = dirname($path);
if ($parent === '.') {
$parent = '';
}
$this->updateQuery->execute([$parent, $mountId]);
}
private function logUpdate($type, $path, $renameTarget, OutputInterface $output) {
switch ($type) {
case INotifyStorage::NOTIFY_ADDED:
$text = 'added';
break;
case INotifyStorage::NOTIFY_MODIFIED:
$text = 'modified';
break;
case INotifyStorage::NOTIFY_REMOVED:
$text = 'removed';
break;
case INotifyStorage::NOTIFY_RENAMED:
$text = 'renamed';
break;
default:
return;
}
$text .= ' ' . $path;
if ($type === INotifyStorage::NOTIFY_RENAMED) {
$text .= ' to ' . $renameTarget;
}
$output->writeln($text);
}
}

View File

@ -151,7 +151,8 @@ class ConfigAdapter implements IMountProvider {
'/' . $user->getUID() . '/files' . $storage->getMountPoint(),
null,
$loader,
$storage->getMountOptions()
$storage->getMountOptions(),
$storage->getId()
);
$mounts[$storage->getMountPoint()] = $mount;
}

View File

@ -34,15 +34,18 @@ use Icewind\SMB\Exception\ConnectException;
use Icewind\SMB\Exception\Exception;
use Icewind\SMB\Exception\ForbiddenException;
use Icewind\SMB\Exception\NotFoundException;
use Icewind\SMB\IShare;
use Icewind\SMB\NativeServer;
use Icewind\SMB\Server;
use Icewind\Streams\CallbackWrapper;
use Icewind\Streams\IteratorDirectory;
use OC\Cache\CappedMemoryCache;
use OC\Files\Filesystem;
use OC\Files\Storage\Common;
use OCP\Files\Storage\INotifyStorage;
use OCP\Files\StorageNotAvailableException;
class SMB extends \OC\Files\Storage\Common {
class SMB extends Common implements INotifyStorage {
/**
* @var \Icewind\SMB\Server
*/
@ -103,6 +106,16 @@ class SMB extends \OC\Files\Storage\Common {
return Filesystem::normalizePath($this->root . '/' . $path, true, false, true);
}
protected function relativePath($fullPath) {
if ($fullPath === $this->root) {
return '';
} else if (substr($fullPath, 0, strlen($this->root)) === $this->root) {
return substr($fullPath, strlen($this->root));
} else {
return null;
}
}
/**
* @param string $path
* @return \Icewind\SMB\IFileInfo
@ -413,4 +426,48 @@ class SMB extends \OC\Files\Storage\Common {
return false;
}
}
public function listen($path, callable $callback) {
$fullPath = $this->buildPath($path);
$oldRenamePath = null;
$this->share->notify($fullPath, function ($smbType, $fullPath) use (&$oldRenamePath, $callback) {
$path = $this->relativePath($fullPath);
if (is_null($path)) {
return true;
}
if ($smbType === IShare::NOTIFY_RENAMED_OLD) {
$oldRenamePath = $path;
return true;
}
$type = $this->mapNotifyType($smbType);
if (is_null($type)) {
return true;
}
if ($type === INotifyStorage::NOTIFY_RENAMED && !is_null($oldRenamePath)) {
$result = $callback($type, $path, $oldRenamePath);
$oldRenamePath = null;
} else {
$result = $callback($type, $path);
}
return $result;
});
}
private function mapNotifyType($smbType) {
switch ($smbType) {
case IShare::NOTIFY_ADDED:
return INotifyStorage::NOTIFY_ADDED;
case IShare::NOTIFY_REMOVED:
return INotifyStorage::NOTIFY_REMOVED;
case IShare::NOTIFY_MODIFIED:
case IShare::NOTIFY_ADDED_STREAM:
case IShare::NOTIFY_MODIFIED_STREAM:
case IShare::NOTIFY_REMOVED_STREAM:
return INotifyStorage::NOTIFY_MODIFIED;
case IShare::NOTIFY_RENAMED_NEW:
return INotifyStorage::NOTIFY_RENAMED;
default:
return null;
}
}
}

View File

@ -170,6 +170,11 @@
<length>4000</length>
</field>
<field>
<name>mount_id</name>
<type>integer</type>
</field>
<index>
<name>mounts_user_index</name>
<unique>false</unique>
@ -197,6 +202,15 @@
</field>
</index>
<index>
<name>mounts_mount_id_index</name>
<unique>false</unique>
<field>
<name>mount_id</name>
<sorting>ascending</sorting>
</field>
</index>
<index>
<name>mounts_user_root_index</name>
<unique>true</unique>

View File

@ -47,6 +47,11 @@ class CachedMountInfo implements ICachedMountInfo {
*/
protected $mountPoint;
/**
* @var int|null
*/
protected $mountId;
/**
* CachedMountInfo constructor.
*
@ -54,12 +59,14 @@ class CachedMountInfo implements ICachedMountInfo {
* @param int $storageId
* @param int $rootId
* @param string $mountPoint
* @param int|null $mountId
*/
public function __construct(IUser $user, $storageId, $rootId, $mountPoint) {
public function __construct(IUser $user, $storageId, $rootId, $mountPoint, $mountId = null) {
$this->user = $user;
$this->storageId = $storageId;
$this->rootId = $rootId;
$this->mountPoint = $mountPoint;
$this->mountId = $mountId;
}
/**
@ -104,4 +111,14 @@ class CachedMountInfo implements ICachedMountInfo {
public function getMountPoint() {
return $this->mountPoint;
}
/**
* Get the id of the configured mount
*
* @return int|null mount id or null if not applicable
* @since 9.1.0
*/
public function getMountId() {
return $this->mountId;
}
}

View File

@ -75,4 +75,8 @@ class LazyStorageMountInfo extends CachedMountInfo {
}
return parent::getMountPoint();
}
public function getMountId() {
return $this->mount->getMountId();
}
}

View File

@ -112,13 +112,7 @@ class UserMountCache implements IUserMountCache {
/** @var ICachedMountInfo[] $removedMounts */
$removedMounts = array_udiff($cachedMounts, $newMounts, $mountDiff);
$changedMounts = array_uintersect($newMounts, $cachedMounts, function (ICachedMountInfo $mount1, ICachedMountInfo $mount2) {
// filter mounts with the same root id and different mountpoints
if ($mount1->getRootId() !== $mount2->getRootId()) {
return -1;
}
return ($mount1->getMountPoint() !== $mount2->getMountPoint()) ? 0 : 1;
});
$changedMounts = $this->findChangedMounts($newMounts, $cachedMounts);
foreach ($addedMounts as $mount) {
$this->addToCache($mount);
@ -130,28 +124,50 @@ class UserMountCache implements IUserMountCache {
unset($this->mountsForUsers[$user->getUID()][$index]);
}
foreach ($changedMounts as $mount) {
$this->setMountPoint($mount);
$this->updateCachedMount($mount);
}
}
/**
* @param ICachedMountInfo[] $newMounts
* @param ICachedMountInfo[] $cachedMounts
* @return ICachedMountInfo[]
*/
private function findChangedMounts(array $newMounts, array $cachedMounts) {
$changed = [];
foreach ($newMounts as $newMount) {
foreach ($cachedMounts as $cachedMount) {
if (
$newMount->getRootId() === $cachedMount->getRootId() &&
($newMount->getMountPoint() !== $cachedMount->getMountPoint() || $newMount->getMountId() !== $cachedMount->getMountId())
) {
$changed[] = $newMount;
}
}
}
return $changed;
}
private function addToCache(ICachedMountInfo $mount) {
if ($mount->getStorageId() !== -1) {
$this->connection->insertIfNotExist('*PREFIX*mounts', [
'storage_id' => $mount->getStorageId(),
'root_id' => $mount->getRootId(),
'user_id' => $mount->getUser()->getUID(),
'mount_point' => $mount->getMountPoint()
'mount_point' => $mount->getMountPoint(),
'mount_id' => $mount->getMountId()
], ['root_id', 'user_id']);
} else {
$this->logger->error('Error getting storage info for mount at ' . $mount->getMountPoint());
}
}
private function setMountPoint(ICachedMountInfo $mount) {
private function updateCachedMount(ICachedMountInfo $mount) {
$builder = $this->connection->getQueryBuilder();
$query = $builder->update('mounts')
->set('mount_point', $builder->createNamedParameter($mount->getMountPoint()))
->set('mount_id', $builder->createNamedParameter($mount->getMountId(), IQueryBuilder::PARAM_INT))
->where($builder->expr()->eq('user_id', $builder->createNamedParameter($mount->getUser()->getUID())))
->andWhere($builder->expr()->eq('root_id', $builder->createNamedParameter($mount->getRootId(), IQueryBuilder::PARAM_INT)));
@ -169,7 +185,7 @@ class UserMountCache implements IUserMountCache {
private function dbRowToMountInfo(array $row) {
$user = $this->userManager->get($row['user_id']);
return new CachedMountInfo($user, (int)$row['storage_id'], (int)$row['root_id'], $row['mount_point']);
return new CachedMountInfo($user, (int)$row['storage_id'], (int)$row['root_id'], $row['mount_point'], $row['mount_id']);
}
/**
@ -179,7 +195,7 @@ class UserMountCache implements IUserMountCache {
public function getMountsForUser(IUser $user) {
if (!isset($this->mountsForUsers[$user->getUID()])) {
$builder = $this->connection->getQueryBuilder();
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point')
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id')
->from('mounts')
->where($builder->expr()->eq('user_id', $builder->createPositionalParameter($user->getUID())));
@ -196,7 +212,7 @@ class UserMountCache implements IUserMountCache {
*/
public function getMountsForStorageId($numericStorageId) {
$builder = $this->connection->getQueryBuilder();
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point')
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id')
->from('mounts')
->where($builder->expr()->eq('storage_id', $builder->createPositionalParameter($numericStorageId, IQueryBuilder::PARAM_INT)));
@ -211,7 +227,7 @@ class UserMountCache implements IUserMountCache {
*/
public function getMountsForRootId($rootFileId) {
$builder = $this->connection->getQueryBuilder();
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point')
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id')
->from('mounts')
->where($builder->expr()->eq('root_id', $builder->createPositionalParameter($rootFileId, IQueryBuilder::PARAM_INT)));

View File

@ -68,14 +68,19 @@ class MountPoint implements IMountPoint {
*/
private $invalidStorage = false;
/** @var int|null */
protected $mountId;
/**
* @param string|\OC\Files\Storage\Storage $storage
* @param string $mountpoint
* @param array $arguments (optional) configuration for the storage backend
* @param \OCP\Files\Storage\IStorageFactory $loader
* @param array $mountOptions mount specific options
* @param int|null $mountId
* @throws \Exception
*/
public function __construct($storage, $mountpoint, $arguments = null, $loader = null, $mountOptions = null) {
public function __construct($storage, $mountpoint, $arguments = null, $loader = null, $mountOptions = null, $mountId = null) {
if (is_null($arguments)) {
$arguments = array();
}
@ -102,6 +107,7 @@ class MountPoint implements IMountPoint {
$this->class = $storage;
$this->arguments = $arguments;
}
$this->mountId = $mountId;
}
/**
@ -249,4 +255,8 @@ class MountPoint implements IMountPoint {
public function getStorageRootId() {
return (int)$this->getStorage()->getCache()->getId('');
}
public function getMountId() {
return $this->mountId;
}
}

View File

@ -59,4 +59,12 @@ interface ICachedMountInfo {
* @since 9.0.0
*/
public function getMountPoint();
/**
* Get the id of the configured mount
*
* @return int|null mount id or null if not applicable
* @since 9.1.0
*/
public function getMountId();
}

View File

@ -102,4 +102,12 @@ interface IMountPoint {
* @since 9.1.0
*/
public function getStorageRootId();
/**
* Get the id of the configured mount
*
* @return int|null mount id or null if not applicable
* @since 9.1.0
*/
public function getMountId();
}

View File

@ -0,0 +1,51 @@
<?php
/**
* @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCP\Files\Storage;
/**
* Storage backend that support active notifications
*
* @since 9.1.0
*/
interface INotifyStorage {
const NOTIFY_ADDED = 1;
const NOTIFY_REMOVED = 2;
const NOTIFY_MODIFIED = 3;
const NOTIFY_RENAMED = 4;
/**
* Start listening for update notifications
*
* The provided callback will be called for every incoming notification with the following parameters
* - int $type the type of update, one of the INotifyStorage::NOTIFY_* constants
* - string $path the path of the update
* - string $renameTarget the target of the rename operation, only provided for rename updates
*
* Note that this call is blocking and will not exit on it's own, to stop listening for notifications return `false` from the callback
*
* @param string $path
* @param callable $callback
*
* @since 9.1.0
*/
public function listen($path, callable $callback);
}

View File

@ -163,12 +163,14 @@ class UserMountCacheTest extends TestCase {
$user = $this->userManager->get('u1');
$storage = $this->getStorage(10, 20);
$mount = new MountPoint($storage, '/foo/');
$mount = new MountPoint($storage, '/bar/');
$this->cache->registerMounts($user, [$mount]);
$this->clearCache();
$mount = new MountPoint($storage, '/foo/');
$this->cache->registerMounts($user, [$mount]);
$this->clearCache();
@ -180,6 +182,29 @@ class UserMountCacheTest extends TestCase {
$this->assertEquals('/foo/', $cachedMount->getMountPoint());
}
public function testChangeMountId() {
$user = $this->userManager->get('u1');
$storage = $this->getStorage(10, 20);
$mount = new MountPoint($storage, '/foo/', null, null, null, null);
$this->cache->registerMounts($user, [$mount]);
$this->clearCache();
$mount = new MountPoint($storage, '/foo/', null, null, null, 1);
$this->cache->registerMounts($user, [$mount]);
$this->clearCache();
$cachedMounts = $this->cache->getMountsForUser($user);
$this->assertCount(1, $cachedMounts);
$cachedMount = $cachedMounts[0];
$this->assertEquals(1, $cachedMount->getMountId());
}
public function testGetMountsForUser() {
$user1 = $this->userManager->get('u1');
$user2 = $this->userManager->get('u2');

View File

@ -25,7 +25,7 @@
// We only can count up. The 4. digit is only for the internal patchlevel to trigger DB upgrades
// between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel
// when updating major/minor version number.
$OC_Version = array(9, 1, 0, 11);
$OC_Version = array(9, 1, 0, 12);
// The human readable string
$OC_VersionString = '9.1.0 RC1';