Merge pull request #4066 from nextcloud/always-fix-the-values-live

Directly fix invalid values of DTEND and DTSTART
This commit is contained in:
Roeland Jago Douma 2017-03-29 10:13:10 +02:00 committed by GitHub
commit 00839a5ac5
7 changed files with 371 additions and 452 deletions

View File

@ -23,7 +23,7 @@
<repair-steps> <repair-steps>
<post-migration> <post-migration>
<step>OCA\DAV\Migration\FixBirthdayCalendarComponent</step> <step>OCA\DAV\Migration\FixBirthdayCalendarComponent</step>
<step>OCA\DAV\Migration\ValueFixInsert</step> <step>OCA\DAV\Migration\CalDAVRemoveEmptyValue</step>
</post-migration> </post-migration>
</repair-steps> </repair-steps>
<commands> <commands>

View File

@ -0,0 +1,119 @@
<?php
/**
* @copyright 2017 Joas Schilling <coding@schilljs.com>
*
* @author Joas Schilling <coding@schilljs.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Migration;
use OCA\DAV\CalDAV\CalDavBackend;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use OCP\ILogger;
use OCP\Migration\IOutput;
use OCP\Migration\IRepairStep;
use Sabre\VObject\InvalidDataException;
class CalDAVRemoveEmptyValue implements IRepairStep {
/** @var IDBConnection */
private $db;
/** @var CalDavBackend */
private $calDavBackend;
/** @var ILogger */
private $logger;
/**
* @param IDBConnection $db
* @param CalDavBackend $calDavBackend
* @param ILogger $logger
*/
public function __construct(IDBConnection $db, CalDavBackend $calDavBackend, ILogger $logger) {
$this->db = $db;
$this->calDavBackend = $calDavBackend;
$this->logger = $logger;
}
public function getName() {
return 'Fix broken values of calendar objects';
}
public function run(IOutput $output) {
$pattern = ';VALUE=:';
$count = $warnings = 0;
$objects = $this->getInvalidObjects($pattern);
$output->startProgress(count($objects));
foreach ($objects as $row) {
$calObject = $this->calDavBackend->getCalendarObject((int)$row['calendarid'], $row['uri']);
$data = preg_replace('/' . $pattern . '/', ':', $calObject['calendardata']);
if ($data !== $calObject['calendardata']) {
$output->advance();
try {
$this->calDavBackend->getDenormalizedData($data);
} catch (InvalidDataException $e) {
$this->logger->info('Calendar object for calendar {cal} with uri {uri} still invalid', [
'app' => 'dav',
'cal' => (int)$row['calendarid'],
'uri' => $row['uri'],
]);
$warnings++;
continue;
}
$this->calDavBackend->updateCalendarObject((int)$row['calendarid'], $row['uri'], $data);
$count++;
}
}
$output->finishProgress();
if ($warnings > 0) {
$output->warning(sprintf('%d events could not be updated, see log file for more information', $warnings));
}
if ($count > 0) {
$output->info(sprintf('Updated %d events', $count));
}
}
protected function getInvalidObjects($pattern) {
$query = $this->db->getQueryBuilder();
$query->select(['calendarid', 'uri'])
->from('calendarobjects')
->where($query->expr()->like(
'calendardata',
$query->createNamedParameter(
'%' . $this->db->escapeLikeParameter($pattern) . '%',
IQueryBuilder::PARAM_STR
),
IQueryBuilder::PARAM_STR
));
$result = $query->execute();
$rows = $result->fetchAll();
$result->closeCursor();
return $rows;
}
}

View File

@ -1,71 +0,0 @@
<?php
/**
* @copyright 2016, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Migration;
use OC\BackgroundJob\QueuedJob;
use OCA\DAV\CalDAV\CalDavBackend;
use OCP\ILogger;
use Sabre\VObject\InvalidDataException;
class ValueFix extends QueuedJob {
/** @var CalDavBackend */
private $calDavBackend;
/** @var ILogger */
private $logger;
public function __construct(CalDavBackend $calDavBackend, ILogger $logger) {
$this->calDavBackend = $calDavBackend;
$this->logger = $logger;
}
public function run($argument) {
$user = $argument['user'];
$pattern = '/;VALUE=:/';
$principal = 'principals/users/' . $user;
$calendars = $this->calDavBackend->getCalendarsForUser($principal);
foreach ($calendars as $calendar) {
$objects = $this->calDavBackend->getCalendarObjects($calendar['id']);
foreach ($objects as $object) {
$calObject = $this->calDavBackend->getCalendarObject($calendar['id'], $object['uri']);
$data = preg_replace($pattern, ':', $calObject['calendardata']);
if ($data !== $calObject['calendardata']) {
try {
$this->calDavBackend->getDenormalizedData($data);
} catch (InvalidDataException $e) {
$this->logger->info('Calendar object for calendar {cal} with uri {uri} still invalid', [
'app'=> 'dav',
'cal' => $calendar['id'],
'uri' => $object['uri'],
]);
continue;
}
$this->calDavBackend->updateCalendarObject($calendar['id'], $object['uri'], $data);
}
}
}
}
}

View File

@ -1,63 +0,0 @@
<?php
/**
* @copyright 2016, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Migration;
use OCP\BackgroundJob\IJobList;
use OCP\IConfig;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Migration\IOutput;
use OCP\Migration\IRepairStep;
class ValueFixInsert implements IRepairStep {
/** @var IUserManager */
private $userManager;
/** @var IJobList */
private $jobList;
/** @var IConfig */
private $config;
public function __construct(IUserManager $userManager,
IJobList $jobList,
IConfig $config) {
$this->userManager = $userManager;
$this->jobList = $jobList;
$this->config = $config;
}
public function getName() {
return 'Insert ValueFix background job for each user';
}
public function run(IOutput $output) {
if ($this->config->getAppValue('dav', self::class . '_ran', 'false') !== 'true') {
$this->userManager->callForSeenUsers(function (IUser $user) {
$this->jobList->add(ValueFix::class, ['user' => $user->getUID()]);
});
$this->config->setAppValue('dav', self::class . '_ran', 'true');
}
}
}

View File

@ -0,0 +1,251 @@
<?php
/**
* @copyright 2016, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Tests\Unit\DAV\Migration;
use OCA\DAV\CalDAV\CalDavBackend;
use OCA\DAV\Migration\CalDAVRemoveEmptyValue;
use OCP\ILogger;
use OCP\Migration\IOutput;
use Sabre\VObject\InvalidDataException;
use Test\TestCase;
/**
* Class CalDAVRemoveEmptyValueTest
*
* @package OCA\DAV\Tests\Unit\DAV\Migration
* @group DB
*/
class CalDAVRemoveEmptyValueTest extends TestCase {
/** @var ILogger|\PHPUnit_Framework_MockObject_MockObject */
private $logger;
/** @var CalDavBackend|\PHPUnit_Framework_MockObject_MockObject */
private $backend;
/** @var IOutput|\PHPUnit_Framework_MockObject_MockObject */
private $output;
/** @var string */
private $invalid = 'BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//Mac OS X 10.11.2//EN
CALSCALE:GREGORIAN
BEGIN:VEVENT
TRANSP:OPAQUE
DTEND;VALUE=:20151223T223000Z
LAST-MODIFIED:20151214T091032Z
ORGANIZER;CN="User 1":mailto:user1@example.com
UID:1234567890@example.com
DTSTAMP:20151214T091032Z
STATUS:CONFIRMED
SEQUENCE:0
SUMMARY:Ein Geburtstag
DTSTART;VALUE=:20151223T173000Z
X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC
CREATED;VALUE=:20151214T091032Z
END:VEVENT
END:VCALENDAR';
/** @var string */
private $valid = 'BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//Mac OS X 10.11.2//EN
CALSCALE:GREGORIAN
BEGIN:VEVENT
TRANSP:OPAQUE
DTEND:20151223T223000Z
LAST-MODIFIED:20151214T091032Z
ORGANIZER;CN="User 1":mailto:user1@example.com
UID:1234567890@example.com
DTSTAMP:20151214T091032Z
STATUS:CONFIRMED
SEQUENCE:0
SUMMARY:Ein Geburtstag
DTSTART:20151223T173000Z
X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC
CREATED:20151214T091032Z
END:VEVENT
END:VCALENDAR';
public function setUp() {
parent::setUp();
$this->logger = $this->createMock(ILogger::class);
$this->backend = $this->createMock(CalDavBackend::class);
$this->output = $this->createMock(IOutput::class);
}
public function testRunAllValid() {
/** @var CalDAVRemoveEmptyValue|\PHPUnit_Framework_MockObject_MockObject $step */
$step = $this->getMockBuilder(CalDAVRemoveEmptyValue::class)
->setConstructorArgs([
\OC::$server->getDatabaseConnection(),
$this->backend,
$this->logger
])
->setMethods(['getInvalidObjects'])
->getMock();
$step->expects($this->once())
->method('getInvalidObjects')
->willReturn([]);
$this->output->expects($this->once())
->method('startProgress')
->with(0);
$this->output->expects($this->once())
->method('finishProgress');
$step->run($this->output);
}
public function testRunInvalid() {
/** @var CalDAVRemoveEmptyValue|\PHPUnit_Framework_MockObject_MockObject $step */
$step = $this->getMockBuilder(CalDAVRemoveEmptyValue::class)
->setConstructorArgs([
\OC::$server->getDatabaseConnection(),
$this->backend,
$this->logger
])
->setMethods(['getInvalidObjects'])
->getMock();
$step->expects($this->once())
->method('getInvalidObjects')
->willReturn([
['calendarid' => '42', 'uri' => 'myuri'],
]);
$this->output->expects($this->once())
->method('startProgress')
->with(1);
$this->output->expects($this->once())
->method('finishProgress');
$this->backend->expects($this->exactly(1))
->method('getCalendarObject')
->with(42, 'myuri')
->willReturn([
'calendardata' => $this->invalid
]);
$this->output->expects($this->exactly(1))
->method('advance');
$this->backend->expects($this->exactly(1))
->method('getDenormalizedData')
->with($this->valid);
$this->backend->expects($this->exactly(1))
->method('updateCalendarObject')
->with(42, 'myuri', $this->valid);
$step->run($this->output);
}
public function testRunValid() {
/** @var CalDAVRemoveEmptyValue|\PHPUnit_Framework_MockObject_MockObject $step */
$step = $this->getMockBuilder(CalDAVRemoveEmptyValue::class)
->setConstructorArgs([
\OC::$server->getDatabaseConnection(),
$this->backend,
$this->logger
])
->setMethods(['getInvalidObjects'])
->getMock();
$step->expects($this->once())
->method('getInvalidObjects')
->willReturn([
['calendarid' => '42', 'uri' => 'myuri'],
]);
$this->output->expects($this->once())
->method('startProgress')
->with(1);
$this->output->expects($this->once())
->method('finishProgress');
$this->backend->expects($this->exactly(1))
->method('getCalendarObject')
->with(42, 'myuri')
->willReturn([
'calendardata' => $this->valid
]);
$this->output->expects($this->never())
->method('advance');
$this->backend->expects($this->never())
->method('getDenormalizedData');
$this->backend->expects($this->never())
->method('updateCalendarObject');
$step->run($this->output);
}
public function testRunStillInvalid() {
/** @var CalDAVRemoveEmptyValue|\PHPUnit_Framework_MockObject_MockObject $step */
$step = $this->getMockBuilder(CalDAVRemoveEmptyValue::class)
->setConstructorArgs([
\OC::$server->getDatabaseConnection(),
$this->backend,
$this->logger
])
->setMethods(['getInvalidObjects'])
->getMock();
$step->expects($this->once())
->method('getInvalidObjects')
->willReturn([
['calendarid' => '42', 'uri' => 'myuri'],
]);
$this->output->expects($this->once())
->method('startProgress')
->with(1);
$this->output->expects($this->once())
->method('finishProgress');
$this->backend->expects($this->exactly(1))
->method('getCalendarObject')
->with(42, 'myuri')
->willReturn([
'calendardata' => $this->invalid
]);
$this->output->expects($this->exactly(1))
->method('advance');
$this->backend->expects($this->exactly(1))
->method('getDenormalizedData')
->with($this->valid)
->willThrowException(new InvalidDataException());
$this->backend->expects($this->never())
->method('updateCalendarObject');
$step->run($this->output);
}
}

View File

@ -1,117 +0,0 @@
<?php
/**
* @copyright 2016, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Tests\Unit\DAV\Migration;
use OCA\DAV\Migration\ValueFix;
use OCA\DAV\Migration\ValueFixInsert;
use OCP\BackgroundJob\IJobList;
use OCP\IConfig;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Migration\IOutput;
use Test\TestCase;
class ValueFixInsertTest extends TestCase {
/** @var IUserManager|\PHPUnit_Framework_MockObject_MockObject */
private $userManager;
/** @var IJobList|\PHPUnit_Framework_MockObject_MockObject */
private $jobList;
/** @var IConfig|\PHPUnit_Framework_MockObject_MockObject */
private $config;
/** @var ValueFixInsert */
private $job;
public function setUp() {
parent::setUp();
$this->userManager = $this->createMock(IUserManager::class);
$this->jobList = $this->createMock(IJobList::class);
$this->config = $this->createMock(IConfig::class);
$this->job = new ValueFixInsert(
$this->userManager,
$this->jobList,
$this->config
);
}
public function testGetName() {
$this->assertSame('Insert ValueFix background job for each user', $this->job->getName());
}
public function testRun() {
$user1 = $this->createMock(IUser::class);
$user1->method('getUID')->willReturn('user1');
$user2 = $this->createMock(IUser::class);
$user2->method('getUID')->willReturn('user2');
$this->config->method('getAppValue')
->with(
$this->equalTo('dav'),
$this->equalTo(ValueFixInsert::class.'_ran'),
$this->anything()
)->will($this->returnCallback(function($app, $key, $value) {
return $value;
}));
$this->userManager->method('callForSeenUsers')
->will($this->returnCallback(function(\Closure $function) use ($user1, $user2) {
$function($user1);
$function($user2);
}));
$this->jobList->expects($this->at(0))
->method('add')
->with(
$this->equalTo(ValueFix::class),
$this->equalTo(['user' => 'user1'])
);
$this->jobList->expects($this->at(1))
->method('add')
->with(
$this->equalTo(ValueFix::class),
$this->equalTo(['user' => 'user2'])
);
$this->job->run($this->createMock(IOutput::class));
}
public function testRunOnlyOnce() {
$this->config->method('getAppValue')
->with(
$this->equalTo('dav'),
$this->equalTo(ValueFixInsert::class.'_ran'),
$this->anything()
)->willReturn('true');
$this->userManager->expects($this->never())
->method($this->anything());;
$this->jobList->expects($this->never())
->method($this->anything());
$this->job->run($this->createMock(IOutput::class));
}
}

View File

@ -1,200 +0,0 @@
<?php
/**
* @copyright 2016, Roeland Jago Douma <roeland@famdouma.nl>
*
* @author Roeland Jago Douma <roeland@famdouma.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\DAV\Tests\Unit\DAV\Migration;
use OCA\DAV\CalDAV\CalDavBackend;
use OCA\DAV\Migration\ValueFix;
use OCP\ILogger;
use Sabre\VObject\InvalidDataException;
use Test\TestCase;
class ValueFixTest extends TestCase {
/** @var ILogger|\PHPUnit_Framework_MockObject_MockObject */
private $logger;
/** @var CalDavBackend|\PHPUnit_Framework_MockObject_MockObject */
private $backend;
/** @var string */
private $invalid = 'BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//Mac OS X 10.11.2//EN
CALSCALE:GREGORIAN
BEGIN:VEVENT
TRANSP:OPAQUE
DTEND;VALUE=:20151223T223000Z
LAST-MODIFIED:20151214T091032Z
ORGANIZER;CN="User 1":mailto:user1@example.com
UID:1234567890@example.com
DTSTAMP:20151214T091032Z
STATUS:CONFIRMED
SEQUENCE:0
SUMMARY:Ein Geburtstag
DTSTART;VALUE=:20151223T173000Z
X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC
CREATED;VALUE=:20151214T091032Z
END:VEVENT
END:VCALENDAR';
/** @var string */
private $valid = 'BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//Apple Inc.//Mac OS X 10.11.2//EN
CALSCALE:GREGORIAN
BEGIN:VEVENT
TRANSP:OPAQUE
DTEND:20151223T223000Z
LAST-MODIFIED:20151214T091032Z
ORGANIZER;CN="User 1":mailto:user1@example.com
UID:1234567890@example.com
DTSTAMP:20151214T091032Z
STATUS:CONFIRMED
SEQUENCE:0
SUMMARY:Ein Geburtstag
DTSTART:20151223T173000Z
X-APPLE-TRAVEL-ADVISORY-BEHAVIOR:AUTOMATIC
CREATED:20151214T091032Z
END:VEVENT
END:VCALENDAR';
/** @var ValueFix */
private $job;
public function setUp() {
parent::setUp();
$this->logger = $this->createMock(ILogger::class);
$this->backend = $this->createMock(CalDavBackend::class);
$this->job = new ValueFix(
$this->backend,
$this->logger
);
}
public function testRunInvalid() {
$calendars = [['id' => 42]];
$objects = [['uri' => 'myuri']];
$this->backend->method('getCalendarsForUser')
->with($this->equalTo('principals/users/u1'))
->willReturn($calendars);
$this->backend->method('getCalendarObjects')
->with($this->equalTo(42))
->willReturn($objects);
$this->backend->method('getCalendarObject')
->with(
$this->equalTo(42),
$this->equalTo('myuri')
)->willReturn([
'calendardata' => $this->invalid
]);
$this->backend->expects($this->once())
->method('getDenormalizedData')
->with($this->valid);
$this->backend->expects($this->once())
->method('updateCalendarObject')
->with(
$this->equalTo(42),
$this->equalTo('myuri'),
$this->equalTo($this->valid)
);
$this->job->run(['user' => 'u1']);
}
public function testRunValid() {
$calendars = [['id' => 42]];
$objects = [['uri' => 'myuri']];
$this->backend->method('getCalendarsForUser')
->with($this->equalTo('principals/users/u1'))
->willReturn($calendars);
$this->backend->method('getCalendarObjects')
->with($this->equalTo(42))
->willReturn($objects);
$this->backend->method('getCalendarObject')
->with(
$this->equalTo(42),
$this->equalTo('myuri')
)->willReturn([
'calendardata' => $this->valid
]);
$this->backend->expects($this->never())
->method('getDenormalizedData');
$this->backend->expects($this->never())
->method('updateCalendarObject');
$this->job->run(['user' => 'u1']);
}
public function testRunStillInvalid() {
$calendars = [['id' => 42]];
$objects = [['uri' => 'myuri']];
$this->backend->method('getCalendarsForUser')
->with($this->equalTo('principals/users/u1'))
->willReturn($calendars);
$this->backend->method('getCalendarObjects')
->with($this->equalTo(42))
->willReturn($objects);
$this->backend->method('getCalendarObject')
->with(
$this->equalTo(42),
$this->equalTo('myuri')
)->willReturn([
'calendardata' => $this->invalid
]);
$this->backend->expects($this->once())
->method('getDenormalizedData')
->with($this->valid)
->willThrowException(new InvalidDataException());
$this->logger->expects($this->once())
->method('info')
->with(
$this->equalTo('Calendar object for calendar {cal} with uri {uri} still invalid'),
$this->equalTo([
'app'=> 'dav',
'cal' => 42,
'uri' => 'myuri',
])
);
$this->backend->expects($this->never())
->method('updateCalendarObject');
$this->job->run(['user' => 'u1']);
}
}