Rework background job system

This commit is contained in:
Robin Appelman 2013-04-20 23:27:46 +02:00
parent d0a5fe1f4a
commit 7948341a86
16 changed files with 530 additions and 354 deletions

View File

@ -49,7 +49,7 @@ $entry = array(
'name' => 'LDAP'
);
OCP\Backgroundjob::addRegularTask('OCA\user_ldap\lib\Jobs', 'updateGroups');
OCP\Backgroundjob::registerJob('OCA\user_ldap\lib\Jobs');
if(OCP\App::isEnabled('user_webdavauth')) {
OCP\Util::writeLog('user_ldap',
'user_ldap and user_webdavauth are incompatible. You may experience unexpected behaviour',

View File

@ -23,20 +23,22 @@
namespace OCA\user_ldap\lib;
class Jobs {
class Jobs extends \OC\BackgroundJob\TimedJob {
static private $groupsFromDB;
static private $groupBE;
static private $connector;
public function __construct(){
$this->interval = self::getRefreshInterval();
}
public function run($argument){
Jobs::updateGroups();
}
static public function updateGroups() {
\OCP\Util::writeLog('user_ldap', 'Run background job "updateGroups"', \OCP\Util::DEBUG);
$lastUpdate = \OCP\Config::getAppValue('user_ldap', 'bgjUpdateGroupsLastRun', 0);
if((time() - $lastUpdate) < self::getRefreshInterval()) {
\OCP\Util::writeLog('user_ldap', 'bgJ "updateGroups" last run too fresh, aborting.', \OCP\Util::DEBUG);
//komm runter Werner die Maurer geben ein aus
return;
}
$knownGroups = array_keys(self::getKnownGroups());
$actualGroups = self::getGroupBE()->getGroups();
@ -45,7 +47,6 @@ class Jobs {
\OCP\Util::writeLog('user_ldap',
'bgJ "updateGroups" groups do not seem to be configured properly, aborting.',
\OCP\Util::INFO);
\OCP\Config::setAppValue('user_ldap', 'bgjUpdateGroupsLastRun', time());
return;
}
@ -53,8 +54,6 @@ class Jobs {
self::handleCreatedGroups(array_diff($actualGroups, $knownGroups));
self::handleRemovedGroups(array_diff($knownGroups, $actualGroups));
\OCP\Config::setAppValue('user_ldap', 'bgjUpdateGroupsLastRun', time());
\OCP\Util::writeLog('user_ldap', 'bgJ "updateGroups" Finished.', \OCP\Util::DEBUG);
}

View File

@ -94,7 +94,11 @@ if (OC::$CLI) {
touch(TemporaryCronClass::$lockfile);
// Work
OC_BackgroundJob_Worker::doAllSteps();
$jobList = new \OC\BackgroundJob\JobList();
$jobs = $jobList->getAll();
foreach ($jobs as $job) {
$job->execute($jobList);
}
} else {
// We call cron.php from some website
if ($appmode == 'cron') {
@ -102,7 +106,10 @@ if (OC::$CLI) {
OC_JSON::error(array('data' => array('message' => 'Backgroundjobs are using system cron!')));
} else {
// Work and success :-)
OC_BackgroundJob_Worker::doNextStep();
$jobList = new \OC\BackgroundJob\JobList();
$job = $jobList->getNext();
$job->execute($jobList);
$jobList->setLastJob($job);
OC_JSON::success();
}
}

45
lib/backgroundjob/job.php Normal file
View File

@ -0,0 +1,45 @@
<?php
/**
* Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\BackgroundJob;
abstract class Job {
protected $id;
protected $lastRun;
protected $argument;
/**
* @param JobList $jobList
*/
public function execute($jobList) {
$jobList->setLastRun($this);
$this->run($this->argument);
}
abstract protected function run($argument);
public function setId($id) {
$this->id = $id;
}
public function setLastRun($lastRun) {
$this->lastRun = $lastRun;
}
public function setArgument($argument) {
$this->argument = $argument;
}
public function getId() {
return $this->id;
}
public function getLastRun() {
return $this->lastRun();
}
}

View File

@ -0,0 +1,158 @@
<?php
/**
* Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\BackgroundJob;
/**
* Class QueuedJob
*
* create a background job that is to be executed once
*
* @package OC\BackgroundJob
*/
class JobList {
/**
* @param Job|string $job
* @param mixed $argument
*/
public function add($job, $argument = null) {
if (!$this->has($job, $argument)) {
if ($job instanceof Job) {
$class = get_class($job);
} else {
$class = $job;
}
$argument = json_encode($argument);
$query = \OC_DB::prepare('INSERT INTO `*PREFIX*jobs`(`class`, `argument`, `last_run`) VALUES(?, ?, 0)');
$query->execute(array($class, $argument));
}
}
/**
* @param Job|string $job
* @param mixed $argument
*/
public function remove($job, $argument = null) {
if ($job instanceof Job) {
$class = get_class($job);
} else {
$class = $job;
}
if (!is_null($argument)) {
$argument = json_encode($argument);
$query = \OC_DB::prepare('DELETE FROM `*PREFIX*jobs` WHERE `class` = ? AND `argument` = ?');
$query->execute(array($class, $argument));
} else {
$query = \OC_DB::prepare('DELETE FROM `*PREFIX*jobs` WHERE `class` = ?');
$query->execute(array($class));
}
}
/**
* check if a job is in the list
*
* @param $job
* @param mixed $argument
* @return bool
*/
public function has($job, $argument) {
if ($job instanceof Job) {
$class = get_class($job);
} else {
$class = $job;
}
$argument = json_encode($argument);
$query = \OC_DB::prepare('SELECT `id` FROM `*PREFIX*jobs` WHERE `class` = ? AND `argument` = ?');
$result = $query->execute(array($class, $argument));
return (bool)$result->fetchRow();
}
/**
* get all jobs in the list
*
* @return Job[]
*/
public function getAll() {
$query = \OC_DB::prepare('SELECT `id`, `class`, `last_run`, `argument` FROM `*PREFIX*jobs`');
$result = $query->execute();
$jobs = array();
while ($row = $result->fetchRow()) {
$jobs[] = $this->buildJob($row);
}
return $jobs;
}
/**
* get the next job in the list
*
* @return Job
*/
public function getNext() {
$lastId = $this->getLastJob();
$query = \OC_DB::prepare('SELECT `id`, `class`, `last_run`, `argument` FROM `*PREFIX*jobs` WHERE `id` > ? ORDER BY `id` ASC', 1);
$result = $query->execute(array($lastId));
if ($row = $result->fetchRow()) {
return $this->buildJob($row);
} else {
//begin at the start of the queue
$query = \OC_DB::prepare('SELECT `id`, `class`, `last_run`, `argument` FROM `*PREFIX*jobs` ORDER BY `id` ASC', 1);
$result = $query->execute();
if ($row = $result->fetchRow()) {
return $this->buildJob($row);
} else {
return null; //empty job list
}
}
}
/**
* get the job object from a row in the db
*
* @param array $row
* @return Job
*/
private function buildJob($row) {
$class = $row['class'];
/**
* @var Job $job
*/
$job = new $class();
$job->setId($row['id']);
$job->setLastRun($row['last_run']);
$job->setArgument(json_decode($row['argument']));
return $job;
}
/**
* set the job that was last ran
*
* @param Job $job
*/
public function setLastJob($job) {
\OC_Appconfig::setValue('backgroundjob', 'lastjob', $job->getId());
}
/**
* get the id of the last ran job
*
* @return int
*/
public function getLastJob() {
return \OC_Appconfig::getValue('backgroundjob', 'lastjob', 0);
}
/**
* set the lastRun of $job to now
*
* @param Job $job
*/
public function setLastRun($job) {
$query = \OC_DB::prepare('UPDATE `*PREFIX*jobs` SET `last_run` = ? WHERE `id` = ?');
$query->execute(array(time(), $job->getId()));
}
}

View File

@ -0,0 +1,28 @@
<?php
/**
* Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\BackgroundJob;
/**
* Class QueuedJob
*
* create a background job that is to be executed once
*
* @package OC\BackgroundJob
*/
abstract class QueuedJob extends Job {
/**
* run the job, then remove it from the joblist
*
* @param JobList $jobList
*/
public function execute($jobList) {
$jobList->remove($this);
$this->run($this->argument);
}
}

View File

@ -1,105 +0,0 @@
<?php
/**
* ownCloud
*
* @author Jakob Sack
* @copyright 2012 Jakob Sack owncloud@jakobsack.de
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
* License as published by the Free Software Foundation; either
* version 3 of the License, or any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU Affero General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
*
*/
/**
* This class manages our queued tasks.
*/
class OC_BackgroundJob_QueuedTask{
/**
* @brief Gets one queued task
* @param $id ID of the task
* @return associative array
*/
public static function find( $id ) {
$stmt = OC_DB::prepare( 'SELECT * FROM `*PREFIX*queuedtasks` WHERE `id` = ?' );
$result = $stmt->execute(array($id));
return $result->fetchRow();
}
/**
* @brief Gets all queued tasks
* @return array with associative arrays
*/
public static function all() {
// Array for objects
$return = array();
// Get Data
$stmt = OC_DB::prepare( 'SELECT * FROM `*PREFIX*queuedtasks`' );
$result = $stmt->execute(array());
while( $row = $result->fetchRow()) {
$return[] = $row;
}
return $return;
}
/**
* @brief Gets all queued tasks of a specific app
* @param $app app name
* @return array with associative arrays
*/
public static function whereAppIs( $app ) {
// Array for objects
$return = array();
// Get Data
$stmt = OC_DB::prepare( 'SELECT * FROM `*PREFIX*queuedtasks` WHERE `app` = ?' );
$result = $stmt->execute(array($app));
while( $row = $result->fetchRow()) {
$return[] = $row;
}
// Und weg damit
return $return;
}
/**
* @brief queues a task
* @param $app app name
* @param $klass class name
* @param $method method name
* @param $parameters all useful data as text
* @return id of task
*/
public static function add( $app, $klass, $method, $parameters ) {
$stmt = OC_DB::prepare( 'INSERT INTO `*PREFIX*queuedtasks` (`app`, `klass`, `method`, `parameters`)'
.' VALUES(?,?,?,?)' );
$result = $stmt->execute(array($app, $klass, $method, $parameters ));
return OC_DB::insertid();
}
/**
* @brief deletes a queued task
* @param $id id of task
* @return true/false
*
* Deletes a report
*/
public static function delete( $id ) {
$stmt = OC_DB::prepare( 'DELETE FROM `*PREFIX*queuedtasks` WHERE `id` = ?' );
$result = $stmt->execute(array($id));
return true;
}
}

View File

@ -1,52 +0,0 @@
<?php
/**
* ownCloud
*
* @author Jakob Sack
* @copyright 2012 Jakob Sack owncloud@jakobsack.de
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
* License as published by the Free Software Foundation; either
* version 3 of the License, or any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU Affero General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
*
*/
/**
* This class manages the regular tasks.
*/
class OC_BackgroundJob_RegularTask{
static private $registered = array();
/**
* @brief creates a regular task
* @param $klass class name
* @param $method method name
* @return true
*/
static public function register( $klass, $method ) {
// Create the data structure
self::$registered["$klass-$method"] = array( $klass, $method );
// No chance for failure ;-)
return true;
}
/**
* @brief gets all regular tasks
* @return associative array
*
* key is string "$klass-$method", value is array( $klass, $method )
*/
static public function all() {
return self::$registered;
}
}

View File

@ -0,0 +1,41 @@
<?php
/**
* Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\BackgroundJob;
/**
* Class QueuedJob
*
* create a background job that is to be executed at an interval
*
* @package OC\BackgroundJob
*/
abstract class TimedJob extends Job {
protected $interval = 0;
/**
* set the interval for the job
*
* @param int $interval
*/
public function setInterval($interval) {
$this->interval = $interval;
}
/**
* run the job if
*
* @param JobList $jobList
*/
public function execute($jobList) {
if ((time() - $this->lastRun) > $this->interval) {
$jobList->setLastRun($this);
$this->run($this->argument);
}
}
}

View File

@ -1,118 +0,0 @@
<?php
/**
* ownCloud
*
* @author Jakob Sack
* @copyright 2012 Jakob Sack owncloud@jakobsack.de
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
* License as published by the Free Software Foundation; either
* version 3 of the License, or any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
*
* You should have received a copy of the GNU Affero General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
*
*/
/**
* This class does the dirty work.
*
* TODO: locking in doAllSteps
*/
class OC_BackgroundJob_Worker{
/**
* @brief executes all tasks
* @return boolean
*
* This method executes all regular tasks and then all queued tasks.
* This method should be called by cli scripts that do not let the user
* wait.
*/
public static function doAllSteps() {
// Do our regular work
$lasttask = OC_Appconfig::getValue( 'core', 'backgroundjobs_task', '' );
$regular_tasks = OC_BackgroundJob_RegularTask::all();
ksort( $regular_tasks );
foreach( $regular_tasks as $key => $value ) {
if( strcmp( $key, $lasttask ) > 0 ) {
// Set "restart here" config value
OC_Appconfig::setValue( 'core', 'backgroundjobs_task', $key );
call_user_func( $value );
}
}
// Reset "start here" config value
OC_Appconfig::setValue( 'core', 'backgroundjobs_task', '' );
// Do our queued tasks
$queued_tasks = OC_BackgroundJob_QueuedTask::all();
foreach( $queued_tasks as $task ) {
OC_BackgroundJob_QueuedTask::delete( $task['id'] );
call_user_func( array( $task['klass'], $task['method'] ), $task['parameters'] );
}
return true;
}
/**
* @brief does a single task
* @return boolean
*
* This method executes one task. It saves the last state and continues
* with the next step. This method should be used by webcron and ajax
* services.
*/
public static function doNextStep() {
$laststep = OC_Appconfig::getValue( 'core', 'backgroundjobs_step', 'regular_tasks' );
if( $laststep == 'regular_tasks' ) {
// get last app
$lasttask = OC_Appconfig::getValue( 'core', 'backgroundjobs_task', '' );
// What's the next step?
$regular_tasks = OC_BackgroundJob_RegularTask::all();
ksort( $regular_tasks );
$done = false;
// search for next background job
foreach( $regular_tasks as $key => $value ) {
if( strcmp( $key, $lasttask ) > 0 ) {
OC_Appconfig::setValue( 'core', 'backgroundjobs_task', $key );
$done = true;
call_user_func( $value );
break;
}
}
if( $done == false ) {
// Next time load queued tasks
OC_Appconfig::setValue( 'core', 'backgroundjobs_step', 'queued_tasks' );
}
}
else{
$tasks = OC_BackgroundJob_QueuedTask::all();
if( count( $tasks )) {
$task = $tasks[0];
// delete job before we execute it. This prevents endless loops
// of failing jobs.
OC_BackgroundJob_QueuedTask::delete($task['id']);
// execute job
call_user_func( array( $task['klass'], $task['method'] ), $task['parameters'] );
}
else{
// Next time load queued tasks
OC_Appconfig::setValue( 'core', 'backgroundjobs_step', 'regular_tasks' );
OC_Appconfig::setValue( 'core', 'backgroundjobs_task', '' );
}
}
return true;
}
}

View File

@ -561,7 +561,7 @@ class OC {
*/
public static function registerCacheHooks() {
// register cache cleanup jobs
OC_BackgroundJob_RegularTask::register('OC_Cache_FileGlobal', 'gc');
\OCP\BackgroundJob::registerJob('OC_Cache_FileGlobalGC');
OC_Hook::connect('OC_User', 'post_login', 'OC_Cache_File', 'loginListener');
}

8
lib/cache/fileglobalgc.php vendored Normal file
View File

@ -0,0 +1,8 @@
<?php
class OC_Cache_FileGlobalGC extends \OC\BackgroundJob\Job{
public function run($argument){
OC_Cache_FileGlobal::gc();
}
}

View File

@ -59,7 +59,7 @@ class BackgroundJob {
/**
* @brief sets the background jobs execution type
* @param $type execution type
* @param string $type execution type
* @return boolean
*
* This method sets the execution type of the background jobs. Possible types
@ -70,71 +70,12 @@ class BackgroundJob {
}
/**
* @brief creates a regular task
* @param $klass class name
* @param $method method name
* @return true
* @param \OC\BackgroundJob\Job|string $job
* @param mixed $argument
*/
public static function addRegularTask( $klass, $method ) {
return \OC_BackgroundJob_RegularTask::register( $klass, $method );
public static function registerJob($job, $argument = null){
$jobList = new \OC\BackgroundJob\JobList();
$jobList->add($job, $argument);
}
/**
* @brief gets all regular tasks
* @return associative array
*
* key is string "$klass-$method", value is array( $klass, $method )
*/
static public function allRegularTasks() {
return \OC_BackgroundJob_RegularTask::all();
}
/**
* @brief Gets one queued task
* @param $id ID of the task
* @return associative array
*/
public static function findQueuedTask( $id ) {
return \OC_BackgroundJob_QueuedTask::find( $id );
}
/**
* @brief Gets all queued tasks
* @return array with associative arrays
*/
public static function allQueuedTasks() {
return \OC_BackgroundJob_QueuedTask::all();
}
/**
* @brief Gets all queued tasks of a specific app
* @param $app app name
* @return array with associative arrays
*/
public static function queuedTaskWhereAppIs( $app ) {
return \OC_BackgroundJob_QueuedTask::whereAppIs( $app );
}
/**
* @brief queues a task
* @param $app app name
* @param $klass class name
* @param $method method name
* @param $parameters all useful data as text
* @return id of task
*/
public static function addQueuedTask( $app, $klass, $method, $parameters ) {
return \OC_BackgroundJob_QueuedTask::add( $app, $klass, $method, $parameters );
}
/**
* @brief deletes a queued task
* @param $id id of task
* @return true/false
*
* Deletes a report
*/
public static function deleteQueuedTask( $id ) {
return \OC_BackgroundJob_QueuedTask::delete( $id );
}
}

View File

@ -0,0 +1,114 @@
<?php
/**
* Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test\BackgroundJob;
class JobRun extends \Exception {
}
/**
* Class DummyJobList
*
* in memory job list for testing purposes
*/
class DummyJobList extends \OC\BackgroundJob\JobList {
/**
* @var \OC\BackgroundJob\Job[]
*/
private $jobs = array();
private $last = 0;
/**
* @param \OC\BackgroundJob\Job|string $job
*/
public function add($job) {
if (!$this->has($job)) {
$this->jobs[] = $job;
}
}
/**
* @param \OC\BackgroundJob\Job|string $job
*/
public function remove($job) {
$index = array_search($job, $this->jobs);
if ($index !== false) {
unset($this->jobs[$index]);
}
}
/**
* check if a job is in the list
*
* @param $job
* @return bool
*/
public function has($job) {
return array_search($job, $this->jobs) !== false;
}
/**
* get all jobs in the list
*
* @return \OC\BackgroundJob\Job[]
*/
public function getAll() {
return $this->jobs;
}
/**
* get the next job in the list
*
* @return \OC\BackgroundJob\Job
*/
public function getNext() {
if (count($this->jobs) > 0) {
if ($this->last < (count($this->jobs) - 1)) {
$i = $this->last + 1;
} else {
$i = 0;
}
return $this->jobs[$i];
} else {
return null;
}
}
/**
* set the job that was last ran
*
* @param \OC\BackgroundJob\Job $job
*/
public function setLastJob($job) {
$i = array_search($job, $this->jobs);
if ($i !== false) {
$this->last = $i;
} else {
$this->last = 0;
}
}
/**
* get the id of the last ran job
*
* @return int
*/
public function getLastJob() {
return $this->last;
}
/**
* set the lastRun of $job to now
*
* @param \OC\BackgroundJob\Job $job
*/
public function setLastRun($job) {
$job->setLastRun(time());
}
}

View File

@ -0,0 +1,42 @@
<?php
/**
* Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test\BackgroundJob;
class TestQueuedJob extends \OC\BackgroundJob\QueuedJob {
public function run() {
throw new JobRun(); //throw an exception so we can detect if this function is called
}
}
class QueuedJob extends \PHPUnit_Framework_TestCase {
/**
* @var DummyJobList $jobList
*/
private $jobList;
/**
* @var \OC\BackgroundJob\TimedJob $job
*/
private $job;
public function setup() {
$this->jobList = new DummyJobList();
$this->job = new TestQueuedJob();
$this->jobList->add($this->job);
}
public function testJobShouldBeRemoved() {
try {
$this->assertTrue($this->jobList->has($this->job));
$this->job->execute($this->jobList);
$this->fail("job should have been run");
} catch (JobRun $e) {
$this->assertFalse($this->jobList->has($this->job));
}
}
}

View File

@ -0,0 +1,68 @@
<?php
/**
* Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test\BackgroundJob;
class TestTimedJob extends \OC\BackgroundJob\TimedJob {
public function __construct() {
$this->setInterval(10);
}
public function run() {
throw new JobRun(); //throw an exception so we can detect if this function is called
}
}
class TimedJob extends \PHPUnit_Framework_TestCase {
/**
* @var DummyJobList $jobList
*/
private $jobList;
/**
* @var \OC\BackgroundJob\TimedJob $job
*/
private $job;
public function setup() {
$this->jobList = new DummyJobList();
$this->job = new TestTimedJob();
$this->jobList->add($this->job);
}
public function testShouldRunAfterInterval() {
$this->job->setLastRun(time() - 12);
try {
$this->job->execute($this->jobList);
$this->fail("job should have run");
} catch (JobRun $e) {
}
}
public function testShouldNotRunWithinInterval() {
$this->job->setLastRun(time() - 5);
try {
$this->job->execute($this->jobList);
} catch (JobRun $e) {
$this->fail("job should not have run");
}
}
public function testShouldNotTwice() {
$this->job->setLastRun(time() - 15);
try {
$this->job->execute($this->jobList);
$this->fail("job should have run the first time");
} catch (JobRun $e) {
try {
$this->job->execute($this->jobList);
} catch (JobRun $e) {
$this->fail("job should not have run the second time");
}
}
}
}