mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #15143 from nextcloud/bugfix/noid/lookup-server-connector-background-jobs-problem 

Fix lookup server connector background jobs problem
This commit is contained in:
Roeland Jago Douma 2019-07-18 13:10:24 +02:00 committed by GitHub
parent a65c00472a 30051e786b
commit 940a3130b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 218 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/lookup_server_connector/appinfo/app.php
View File

@ -19,13 +19,5 @@
* *
*/ */
$dispatcher = \OC::$server->getEventDispatcher(); $app = new \OCA\LookupServerConnector\AppInfo\Application();
$app->register();
$dispatcher->addListener('OC\AccountManager::userUpdated', function(\Symfony\Component\EventDispatcher\GenericEvent $event) {
/** @var \OCP\IUser $user */
$user = $event->getSubject();
/** @var \OCA\LookupServerConnector\UpdateLookupServer $updateLookupServer */
$updateLookupServer = \OC::$server->query(\OCA\LookupServerConnector\UpdateLookupServer::class);
$updateLookupServer->userUpdated($user);
});

1
apps/lookup_server_connector/composer/composer/autoload_classmap.php
View File

@ -6,6 +6,7 @@ $vendorDir = dirname(dirname(__FILE__));
$baseDir = $vendorDir; $baseDir = $vendorDir;
return array( return array(
'OCA\\LookupServerConnector\\AppInfo\\Application' => $baseDir . '/../lib/AppInfo/Application.php',
'OCA\\LookupServerConnector\\BackgroundJobs\\RetryJob' => $baseDir . '/../lib/BackgroundJobs/RetryJob.php', 'OCA\\LookupServerConnector\\BackgroundJobs\\RetryJob' => $baseDir . '/../lib/BackgroundJobs/RetryJob.php',
'OCA\\LookupServerConnector\\UpdateLookupServer' => $baseDir . '/../lib/UpdateLookupServer.php', 'OCA\\LookupServerConnector\\UpdateLookupServer' => $baseDir . '/../lib/UpdateLookupServer.php',
); );

1
apps/lookup_server_connector/composer/composer/autoload_static.php
View File

@ -21,6 +21,7 @@ class ComposerStaticInitLookupServerConnector
); );
public static $classMap = array ( public static $classMap = array (
'OCA\\LookupServerConnector\\AppInfo\\Application' => __DIR__ . '/..' . '/../lib/AppInfo/Application.php',
'OCA\\LookupServerConnector\\BackgroundJobs\\RetryJob' => __DIR__ . '/..' . '/../lib/BackgroundJobs/RetryJob.php', 'OCA\\LookupServerConnector\\BackgroundJobs\\RetryJob' => __DIR__ . '/..' . '/../lib/BackgroundJobs/RetryJob.php',
'OCA\\LookupServerConnector\\UpdateLookupServer' => __DIR__ . '/..' . '/../lib/UpdateLookupServer.php', 'OCA\\LookupServerConnector\\UpdateLookupServer' => __DIR__ . '/..' . '/../lib/UpdateLookupServer.php',
); );

56
apps/lookup_server_connector/lib/AppInfo/Application.php Normal file
View File

@ -0,0 +1,56 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2019 Joas Schilling <coding@schilljs.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\LookupServerConnector\AppInfo;
use OCA\LookupServerConnector\UpdateLookupServer;
use OCP\AppFramework\App;
use OCP\IUser;
use Symfony\Component\EventDispatcher\GenericEvent;
class Application extends App {
public function __construct () {
parent::__construct('lookup_server_connector');
}
/**
* Register the different app parts
*/
public function register(): void {
$this->registerHooksAndEvents();
}
/**
* Register the hooks and events
*/
public function registerHooksAndEvents(): void {
$dispatcher = $this->getContainer()->getServer()->getEventDispatcher();
$dispatcher->addListener('OC\AccountManager::userUpdated', static function(GenericEvent $event) {
/** @var IUser $user */
$user = $event->getSubject();
/** @var UpdateLookupServer $updateLookupServer */
$updateLookupServer = \OC::$server->query(UpdateLookupServer::class);
$updateLookupServer->userUpdated($user);
});
}
}

189
apps/lookup_server_connector/lib/BackgroundJobs/RetryJob.php
View File

@ -1,6 +1,8 @@
<?php <?php
declare(strict_types=1);
/** /**
* @copyright Copyright (c) 2016 Bjoern Schiessle <bjoern@schiessle.org> * @copyright Copyright (c) 2016 Bjoern Schiessle <bjoern@schiessle.org>
* @copyright Copyright (c) 2019 Joas Schilling <coding@schilljs.com>
* *
* @license GNU AGPL version 3 or any later version * @license GNU AGPL version 3 or any later version
* *
@ -22,40 +24,55 @@
namespace OCA\LookupServerConnector\BackgroundJobs; namespace OCA\LookupServerConnector\BackgroundJobs;
use OC\BackgroundJob\Job; use OC\Security\IdentityProof\Signer;
use OC\BackgroundJob\JobList; use OCP\Accounts\IAccountManager;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\Job;
use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\IJobList;
use OCP\Http\Client\IClientService; use OCP\Http\Client\IClientService;
use OCP\IConfig; use OCP\IConfig;
use OCP\ILogger; use OCP\ILogger;
use OCP\IUser;
use OCP\IUserManager;
class RetryJob extends Job { class RetryJob extends Job {
/** @var IClientService */ /** @var IClientService */
private $clientService; private $clientService;
/** @var IJobList */
private $jobList;
/** @var string */ /** @var string */
private $lookupServer; private $lookupServer;
/** @var int how much time should be between two, will be increased for each retry */
private $interval = 100;
/** @var IConfig */ /** @var IConfig */
private $config; private $config;
/** @var IUserManager */
private $userManager;
/** @var IAccountManager */
private $accountManager;
/** @var Signer */
private $signer;
/** @var int */
protected $retries = 0;
/** @var bool */
protected $retainJob = false;
/** /**
* @param ITimeFactory $time
* @param IClientService $clientService * @param IClientService $clientService
* @param IJobList $jobList
* @param IConfig $config * @param IConfig $config
* @param IUserManager $userManager
* @param IAccountManager $accountManager
* @param Signer $signer
*/ */
public function __construct(IClientService $clientService, public function __construct(ITimeFactory $time,
IJobList $jobList, IClientService $clientService,
IConfig $config) { IConfig $config,
IUserManager $userManager,
IAccountManager $accountManager,
Signer $signer) {
parent::__construct($time);
$this->clientService = $clientService; $this->clientService = $clientService;
$this->jobList = $jobList;
$this->config = $config; $this->config = $config;
$this->userManager = $userManager;
if ($config->getSystemValue('has_internet_connection', true) === false) { $this->accountManager = $accountManager;
return; $this->signer = $signer;
}
$this->lookupServer = $config->getSystemValue('lookup_server', 'https://lookup.nextcloud.com'); $this->lookupServer = $config->getSystemValue('lookup_server', 'https://lookup.nextcloud.com');
if (!empty($this->lookupServer)) { if (!empty($this->lookupServer)) {
@ -67,71 +84,129 @@ class RetryJob extends Job {
/** /**
* run the job, then remove it from the jobList * run the job, then remove it from the jobList
* *
* @param JobList $jobList * @param IJobList $jobList
* @param ILogger|null $logger * @param ILogger|null $logger
*/ */
public function execute($jobList, ILogger $logger = null) { public function execute($jobList, ILogger $logger = null): void {
if ($this->shouldRun($this->argument)) { if (!isset($this->argument['userId'])) {
parent::execute($jobList, $logger); // Old background job without user id, just drop it.
$jobList->remove($this, $this->argument); $jobList->remove($this, $this->argument);
}
}
protected function run($argument) {
if ($this->killBackgroundJob((int)$argument['retryNo'])) {
return; return;
} }
$this->retries = (int) $this->config->getUserValue($this->argument['userId'], 'lookup_server_connector', 'update_retries', 0);
if ($this->shouldRemoveBackgroundJob()) {
$jobList->remove($this, $this->argument);
return;
}
if ($this->shouldRun()) {
parent::execute($jobList, $logger);
if (!$this->retainJob) {
$jobList->remove($this, $this->argument);
}
}
}
/**
* Check if we should kill the background job:
*
* - internet connection is disabled
* - no valid lookup server URL given
* - lookup server was disabled by the admin
* - max retries are reached (set to 5)
*
* @return bool
*/
protected function shouldRemoveBackgroundJob(): bool {
return $this->config->getSystemValueBool('has_internet_connection', true) === false ||
$this->config->getSystemValueString('lookup_server', 'https://lookup.nextcloud.com') === '' ||
$this->config->getAppValue('files_sharing', 'lookupServerUploadEnabled', 'yes') !== 'yes' ||
$this->retries >= 5;
}
protected function shouldRun(): bool {
$delay = 100 * 6 ** $this->retries;
return ($this->time->getTime() - $this->lastRun) > $delay;
}
protected function run($argument): void {
$user = $this->userManager->get($this->argument['userId']);
if (!$user instanceof IUser) {
// User does not exist anymore
return;
}
$data = $this->getUserAccountData($user);
$signedData = $this->signer->sign('lookupserver', $data, $user);
$client = $this->clientService->newClient(); $client = $this->clientService->newClient();
try { try {
$client->post($this->lookupServer, if (count($data) === 1) {
// No public data, just the federation Id
$client->delete($this->lookupServer,
[ [
'body' => json_encode($argument['dataArray']), 'body' => json_encode($signedData),
'timeout' => 10, 'timeout' => 10,
'connect_timeout' => 3, 'connect_timeout' => 3,
] ]
); );
} catch (\Exception $e) { } else {
$this->jobList->add(RetryJob::class, $client->post($this->lookupServer,
[ [
'dataArray' => $argument['dataArray'], 'body' => json_encode($signedData),
'retryNo' => $argument['retryNo'] + 1, 'timeout' => 10,
'lastRun' => time(), 'connect_timeout' => 3,
] ]
); );
}
// Reset retry counter
$this->config->deleteUserValue(
$user->getUID(),
'lookup_server_connector',
'update_retries'
);
} catch (\Exception $e) {
// An error occurred, retry later
$this->retainJob = true;
$this->config->setUserValue(
$user->getUID(),
'lookup_server_connector',
'update_retries',
$this->retries + 1
);
} }
} }
/** protected function getUserAccountData(IUser $user): array {
* test if it is time for the next run $account = $this->accountManager->getAccount($user);
*
* @param array $argument $publicData = [];
* @return bool foreach ($account->getProperties() as $property) {
*/ if ($property->getScope() === IAccountManager::VISIBILITY_PUBLIC) {
protected function shouldRun($argument) { $publicData[$property->getName()] = $property->getValue();
$retryNo = (int)$argument['retryNo']; }
$delay = $this->interval * 6 ** $retryNo;
return !isset($argument['lastRun']) || ((time() - $argument['lastRun']) > $delay);
} }
/** $data = ['federationId' => $user->getCloudId()];
* check if we should kill the background job if (!empty($publicData)) {
* $data['name'] = $publicData[IAccountManager::PROPERTY_DISPLAYNAME]['value'] ?? '';
* The lookup server should no longer be contacted if: $data['email'] = $publicData[IAccountManager::PROPERTY_EMAIL]['value'] ?? '';
* $data['address'] = $publicData[IAccountManager::PROPERTY_ADDRESS]['value'] ?? '';
* - max retries are reached (set to 5) $data['website'] = $publicData[IAccountManager::PROPERTY_WEBSITE]['value'] ?? '';
* - lookup server was disabled by the admin $data['twitter'] = $publicData[IAccountManager::PROPERTY_TWITTER]['value'] ?? '';
* - no valid lookup server URL given $data['phone'] = $publicData[IAccountManager::PROPERTY_PHONE]['value'] ?? '';
* $data['twitter_signature'] = $publicData[IAccountManager::PROPERTY_TWITTER]['signature'] ?? '';
* @param int $retryCount $data['website_signature'] = $publicData[IAccountManager::PROPERTY_WEBSITE]['signature'] ?? '';
* @return bool $data['verificationStatus'] = [
*/ IAccountManager::PROPERTY_WEBSITE => $publicData[IAccountManager::PROPERTY_WEBSITE]['verified'] ?? '',
protected function killBackgroundJob($retryCount) { IAccountManager::PROPERTY_TWITTER => $publicData[IAccountManager::PROPERTY_TWITTER]['verified'] ?? '',
$maxTriesReached = $retryCount >= 5; ];
$lookupServerDisabled = $this->config->getAppValue('files_sharing', 'lookupServerUploadEnabled', 'yes') !== 'yes'; }
return $maxTriesReached || $lookupServerDisabled || empty($this->lookupServer); return $data;
} }
} }

125
apps/lookup_server_connector/lib/UpdateLookupServer.php
View File

@ -1,4 +1,5 @@
<?php <?php
declare(strict_types=1);
/** /**
* @copyright Copyright (c) 2016 Bjoern Schiessle <bjoern@schiessle.org> * @copyright Copyright (c) 2016 Bjoern Schiessle <bjoern@schiessle.org>
* @copyright Copyright (c) 2016 Lukas Reschke <lukas@statuscode.ch> * @copyright Copyright (c) 2016 Lukas Reschke <lukas@statuscode.ch>
@ -22,11 +23,8 @@
namespace OCA\LookupServerConnector; namespace OCA\LookupServerConnector;
use OC\Accounts\AccountManager;
use OC\Security\IdentityProof\Signer;
use OCA\LookupServerConnector\BackgroundJobs\RetryJob; use OCA\LookupServerConnector\BackgroundJobs\RetryJob;
use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\IJobList;
use OCP\Http\Client\IClientService;
use OCP\IConfig; use OCP\IConfig;
use OCP\IUser; use OCP\IUser;
@ -36,136 +34,51 @@ use OCP\IUser;
* @package OCA\LookupServerConnector * @package OCA\LookupServerConnector
*/ */
class UpdateLookupServer { class UpdateLookupServer {
/** @var AccountManager */ /** @var IConfig */
private $accountManager; private $config;
/** @var IClientService */
private $clientService;
/** @var Signer */
private $signer;
/** @var IJobList */ /** @var IJobList */
private $jobList; private $jobList;
/** @var string URL point to lookup server */
private $lookupServer;
/** @var bool */
private $lookupServerEnabled;
/** /**
* @param AccountManager $accountManager
* @param IClientService $clientService
* @param Signer $signer
* @param IJobList $jobList * @param IJobList $jobList
* @param IConfig $config * @param IConfig $config
*/ */
public function __construct(AccountManager $accountManager, public function __construct(IJobList $jobList,
IClientService $clientService,
Signer $signer,
IJobList $jobList,
IConfig $config) { IConfig $config) {
$this->accountManager = $accountManager; $this->config = $config;
$this->clientService = $clientService;
$this->signer = $signer;
$this->jobList = $jobList; $this->jobList = $jobList;
if($config->getSystemValue('has_internet_connection', true) === false) {
return;
}
$this->lookupServerEnabled = $config->getAppValue('files_sharing', 'lookupServerUploadEnabled', 'yes') === 'yes';
$this->lookupServer = $config->getSystemValue('lookup_server', 'https://lookup.nextcloud.com');
if(!empty($this->lookupServer)) {
$this->lookupServer = rtrim($this->lookupServer, '/');
$this->lookupServer .= '/users';
}
} }
/** /**
* @param IUser $user * @param IUser $user
*/ */
public function userUpdated(IUser $user) { public function userUpdated(IUser $user): void {
if (!$this->shouldUpdateLookupServer()) { if (!$this->shouldUpdateLookupServer()) {
return; return;
} }
$userData = $this->accountManager->getUser($user); // Reset retry counter
$publicData = []; $this->config->deleteUserValue(
$user->getUID(),
foreach ($userData as $key => $data) { 'lookup_server_connector',
if ($data['scope'] === AccountManager::VISIBILITY_PUBLIC) { 'update_retries'
$publicData[$key] = $data;
}
}
$this->sendToLookupServer($user, $publicData);
}
/**
* send public user data to the lookup server
*
* @param IUser $user
* @param array $publicData
*/
protected function sendToLookupServer(IUser $user, array $publicData) {
$dataArray = ['federationId' => $user->getCloudId()];
if (!empty($publicData)) {
$dataArray['name'] = isset($publicData[AccountManager::PROPERTY_DISPLAYNAME]) ? $publicData[AccountManager::PROPERTY_DISPLAYNAME]['value'] : '';
$dataArray['email'] = isset($publicData[AccountManager::PROPERTY_EMAIL]) ? $publicData[AccountManager::PROPERTY_EMAIL]['value'] : '';
$dataArray['address'] = isset($publicData[AccountManager::PROPERTY_ADDRESS]) ? $publicData[AccountManager::PROPERTY_ADDRESS]['value'] : '';
$dataArray['website'] = isset($publicData[AccountManager::PROPERTY_WEBSITE]) ? $publicData[AccountManager::PROPERTY_WEBSITE]['value'] : '';
$dataArray['twitter'] = isset($publicData[AccountManager::PROPERTY_TWITTER]) ? $publicData[AccountManager::PROPERTY_TWITTER]['value'] : '';
$dataArray['phone'] = isset($publicData[AccountManager::PROPERTY_PHONE]) ? $publicData[AccountManager::PROPERTY_PHONE]['value'] : '';
$dataArray['twitter_signature'] = isset($publicData[AccountManager::PROPERTY_TWITTER]['signature']) ? $publicData[AccountManager::PROPERTY_TWITTER]['signature'] : '';
$dataArray['website_signature'] = isset($publicData[AccountManager::PROPERTY_WEBSITE]['signature']) ? $publicData[AccountManager::PROPERTY_WEBSITE]['signature'] : '';
$dataArray['verificationStatus'] =
[
AccountManager::PROPERTY_WEBSITE => isset($publicData[AccountManager::PROPERTY_WEBSITE]) ? $publicData[AccountManager::PROPERTY_WEBSITE]['verified'] : '',
AccountManager::PROPERTY_TWITTER => isset($publicData[AccountManager::PROPERTY_TWITTER]) ? $publicData[AccountManager::PROPERTY_TWITTER]['verified'] : '',
];
}
$dataArray = $this->signer->sign('lookupserver', $dataArray, $user);
$httpClient = $this->clientService->newClient();
try {
if (empty($publicData)) {
$httpClient->delete($this->lookupServer,
[
'body' => json_encode($dataArray),
'timeout' => 10,
'connect_timeout' => 3,
]
); );
} else { $this->jobList->add(RetryJob::class, ['userId' => $user->getUID()]);
$httpClient->post($this->lookupServer,
[
'body' => json_encode($dataArray),
'timeout' => 10,
'connect_timeout' => 3,
]
);
}
} catch (\Exception $e) {
$this->jobList->add(RetryJob::class,
[
'dataArray' => $dataArray,
'retryNo' => 0,
]
);
}
} }
/** /**
* check if we should update the lookup server, we only do it if * check if we should update the lookup server, we only do it if
* *
* * we have a valid URL * + we have an internet connection
* * the lookup server update was enabled by the admin * + the lookup server update was not disabled by the admin
* + we have a valid lookup server URL
* *
* @return bool * @return bool
*/ */
private function shouldUpdateLookupServer() { private function shouldUpdateLookupServer(): bool {
return $this->lookupServerEnabled && !empty($this->lookupServer); return $this->config->getSystemValueBool('has_internet_connection', true) === true &&
$this->config->getAppValue('files_sharing', 'lookupServerUploadEnabled', 'yes') === 'yes' &&
$this->config->getSystemValueString('lookup_server', 'https://lookup.nextcloud.com') !== '';
} }
} }