Start migrations

Fixme:
- Install and update of apps
- No revert on live systems (debug only)
- Service adjustment to our interface
- Loading via autoloader

Signed-off-by: Joas Schilling <coding@schilljs.com>
This commit is contained in:
Joas Schilling 2017-06-01 16:56:34 +02:00 committed by Morris Jobke
parent efa52ec111
commit 15eec7b83c
18 changed files with 1260 additions and 17 deletions

View File

@ -0,0 +1,67 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Core\Command\Db\Migrations;
use OC\DB\MigrationService;
use OC\Migration\ConsoleOutput;
use OCP\IDBConnection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class ExecuteCommand extends Command {
/** @var IDBConnection */
private $connection;
/**
* ExecuteCommand constructor.
*
* @param IDBConnection $connection
*/
public function __construct(IDBConnection $connection) {
$this->connection = $connection;
parent::__construct();
}
protected function configure() {
$this
->setName('migrations:execute')
->setDescription('Execute a single migration version manually.')
->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on')
->addArgument('version', InputArgument::REQUIRED, 'The version to execute.', null);
parent::configure();
}
public function execute(InputInterface $input, OutputInterface $output) {
$appName = $input->getArgument('app');
$ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output));
$version = $input->getArgument('version');
$ms->executeStep($version);
}
}

View File

@ -0,0 +1,166 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Core\Command\Db\Migrations;
use OC\DB\MigrationService;
use OC\Migration\ConsoleOutput;
use OCP\IConfig;
use OCP\IDBConnection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Exception\RuntimeException;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class GenerateCommand extends Command {
private static $_templateSimple =
'<?php
namespace <namespace>;
use OCP\Migration\ISimpleMigration;
use OCP\Migration\IOutput;
/**
* Auto-generated migration step: Please modify to your needs!
*/
class Version<version> implements ISimpleMigration {
/**
* @param IOutput $out
*/
public function run(IOutput $out) {
// auto-generated - please modify it to your needs
}
}
';
private static $_templateSchema =
'<?php
namespace <namespace>;
use Doctrine\DBAL\Schema\Schema;
use OCP\Migration\ISchemaMigration;
/**
* Auto-generated migration step: Please modify to your needs!
*/
class Version<version> implements ISchemaMigration {
public function changeSchema(Schema $schema, array $options) {
// auto-generated - please modify it to your needs
}
}
';
private static $_templateSql =
'<?php
namespace <namespace>;
use OCP\IDBConnection;
use OCP\Migration\ISqlMigration;
/**
* Auto-generated migration step: Please modify to your needs!
*/
class Version<version> implements ISqlMigration {
public function sql(IDBConnection $connection) {
// auto-generated - please modify it to your needs
}
}
';
/** @var IDBConnection */
private $connection;
/**
* @param IDBConnection $connection
*/
public function __construct(IDBConnection $connection) {
$this->connection = $connection;
parent::__construct();
}
protected function configure() {
$this
->setName('migrations:generate')
->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on')
->addArgument('kind', InputArgument::REQUIRED, 'simple, schema or sql - defines the kind of migration to be generated');
parent::configure();
}
public function execute(InputInterface $input, OutputInterface $output) {
$appName = $input->getArgument('app');
$ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output));
$kind = $input->getArgument('kind');
$version = date('YmdHis');
$path = $this->generateMigration($ms, $version, $kind);
$output->writeln("New migration class has been generated to <info>$path</info>");
}
/**
* @param MigrationService $ms
* @param string $version
* @param string $kind
* @return string
*/
private function generateMigration(MigrationService $ms, $version, $kind) {
$placeHolders = [
'<namespace>',
'<version>',
];
$replacements = [
$ms->getMigrationsNamespace(),
$version,
];
$code = str_replace($placeHolders, $replacements, $this->getTemplate($kind));
$dir = $ms->getMigrationsDirectory();
$path = $dir . '/Version' . $version . '.php';
if (file_put_contents($path, $code) === false) {
throw new RuntimeException('Failed to generate new migration step.');
}
return $path;
}
private function getTemplate($kind) {
if ($kind === 'simple') {
return self::$_templateSimple;
}
if ($kind === 'schema') {
return self::$_templateSchema;
}
if ($kind === 'sql') {
return self::$_templateSql;
}
throw new \InvalidArgumentException('Kind can only be one of the following: simple, schema or sql');
}
}

View File

@ -0,0 +1,64 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Core\Command\Db\Migrations;
use OC\DB\MigrationService;
use OC\Migration\ConsoleOutput;
use OCP\IDBConnection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class MigrateCommand extends Command {
/** @var IDBConnection */
private $connection;
/**
* @param IDBConnection $connection
*/
public function __construct(IDBConnection $connection) {
$this->connection = $connection;
parent::__construct();
}
protected function configure() {
$this
->setName('migrations:migrate')
->setDescription('Execute a migration to a specified version or the latest available version.')
->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on')
->addArgument('version', InputArgument::OPTIONAL, 'The version number (YYYYMMDDHHMMSS) or alias (first, prev, next, latest) to migrate to.', 'latest');
parent::configure();
}
public function execute(InputInterface $input, OutputInterface $output) {
$appName = $input->getArgument('app');
$ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output));
$version = $input->getArgument('version');
$ms->migrate($version);
}
}

View File

@ -0,0 +1,115 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Core\Command\Db\Migrations;
use OC\DB\MigrationService;
use OC\Migration\ConsoleOutput;
use OCP\IDBConnection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class StatusCommand extends Command {
/** @var IDBConnection */
private $connection;
/**
* @param IDBConnection $connection
*/
public function __construct(IDBConnection $connection) {
$this->connection = $connection;
parent::__construct();
}
protected function configure() {
$this
->setName('migrations:status')
->setDescription('View the status of a set of migrations.')
->addArgument('app', InputArgument::REQUIRED, 'Name of the app this migration command shall work on');
}
public function execute(InputInterface $input, OutputInterface $output) {
$appName = $input->getArgument('app');
$ms = new MigrationService($appName, $this->connection, new ConsoleOutput($output));
$infos = $this->getMigrationsInfos($ms);
foreach ($infos as $key => $value) {
$output->writeln(" <comment>>></comment> $key: " . str_repeat(' ', 50 - strlen($key)) . $value);
}
}
/**
* @param MigrationService $ms
* @return array associative array of human readable info name as key and the actual information as value
*/
public function getMigrationsInfos(MigrationService $ms) {
$executedMigrations = $ms->getMigratedVersions();
$availableMigrations = $ms->getAvailableVersions();
$executedUnavailableMigrations = array_diff($executedMigrations, array_keys($availableMigrations));
$numExecutedUnavailableMigrations = count($executedUnavailableMigrations);
$numNewMigrations = count(array_diff(array_keys($availableMigrations), $executedMigrations));
$infos = [
'App' => $ms->getApp(),
'Version Table Name' => $ms->getMigrationsTableName(),
'Migrations Namespace' => $ms->getMigrationsNamespace(),
'Migrations Directory' => $ms->getMigrationsDirectory(),
'Previous Version' => $this->getFormattedVersionAlias($ms, 'prev'),
'Current Version' => $this->getFormattedVersionAlias($ms, 'current'),
'Next Version' => $this->getFormattedVersionAlias($ms, 'next'),
'Latest Version' => $this->getFormattedVersionAlias($ms, 'latest'),
'Executed Migrations' => count($executedMigrations),
'Executed Unavailable Migrations' => $numExecutedUnavailableMigrations,
'Available Migrations' => count($availableMigrations),
'New Migrations' => $numNewMigrations,
];
return $infos;
}
/**
* @param MigrationService $migrationService
* @param string $alias
* @return mixed|null|string
*/
private function getFormattedVersionAlias(MigrationService $migrationService, $alias) {
$migration = $migrationService->getMigration($alias);
//No version found
if ($migration === null) {
if ($alias === 'next') {
return 'Already at latest migration step';
}
if ($alias === 'prev') {
return 'Already at first migration step';
}
}
return $migration;
}
}

View File

@ -85,6 +85,10 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
$application->add(new OC\Core\Command\Db\GenerateChangeScript());
$application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig())));
$application->add(new OC\Core\Command\Db\ConvertMysqlToMB4(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection(), \OC::$server->getURLGenerator(), \OC::$server->getLogger()));
$application->add(new OC\Core\Command\Db\Migrations\StatusCommand(\OC::$server->getDatabaseConnection()));
$application->add(new OC\Core\Command\Db\Migrations\MigrateCommand(\OC::$server->getDatabaseConnection()));
$application->add(new OC\Core\Command\Db\Migrations\GenerateCommand(\OC::$server->getDatabaseConnection()));
$application->add(new OC\Core\Command\Db\Migrations\ExecuteCommand(\OC::$server->getDatabaseConnection()));
$application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig()));
$application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager()));

View File

@ -35,6 +35,7 @@ use Doctrine\DBAL\Cache\QueryCacheProfile;
use Doctrine\Common\EventManager;
use Doctrine\DBAL\Platforms\MySqlPlatform;
use Doctrine\DBAL\Exception\ConstraintViolationException;
use Doctrine\DBAL\Schema\Schema;
use OC\DB\QueryBuilder\QueryBuilder;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
@ -418,4 +419,27 @@ class Connection extends \Doctrine\DBAL\Connection implements IDBConnection {
}
return $this->getParams()['charset'] === 'utf8mb4';
}
/**
* Create the schema of the connected database
*
* @return Schema
*/
public function createSchema() {
$schemaManager = new MDB2SchemaManager($this);
$migrator = $schemaManager->getMigrator();
return $migrator->createSchema();
}
/**
* Migrate the database to the given schema
*
* @param Schema $toSchema
*/
public function migrateToSchema(Schema $toSchema) {
$schemaManager = new MDB2SchemaManager($this);
$migrator = $schemaManager->getMigrator();
$migrator->migrate($toSchema);
}
}

View File

@ -0,0 +1,401 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
* @author Vincent Petry <pvince81@owncloud.com>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\DB;
use OC\IntegrityCheck\Helpers\AppLocator;
use OC\Migration\SimpleOutput;
use OCP\AppFramework\QueryException;
use OCP\IDBConnection;
use OCP\Migration\IOutput;
use OCP\Migration\ISchemaMigration;
use OCP\Migration\ISimpleMigration;
use OCP\Migration\ISqlMigration;
use Doctrine\DBAL\Schema\Column;
use Doctrine\DBAL\Schema\Table;
use Doctrine\DBAL\Types\Type;
class MigrationService {
/** @var boolean */
private $migrationTableCreated;
/** @var array */
private $migrations;
/** @var IOutput */
private $output;
/** @var Connection */
private $connection;
/** @var string */
private $appName;
/**
* MigrationService constructor.
*
* @param $appName
* @param IDBConnection $connection
* @param AppLocator $appLocator
* @param IOutput|null $output
* @throws \Exception
*/
function __construct($appName, IDBConnection $connection, IOutput $output = null, AppLocator $appLocator = null) {
$this->appName = $appName;
$this->connection = $connection;
$this->output = $output;
if (is_null($this->output)) {
$this->output = new SimpleOutput(\OC::$server->getLogger(), $appName);
}
if ($appName === 'core') {
$this->migrationsPath = \OC::$SERVERROOT . '/core/Migrations';
$this->migrationsNamespace = 'OC\\Migrations';
} else {
if (is_null($appLocator)) {
$appLocator = new AppLocator();
}
$appPath = $appLocator->getAppPath($appName);
$this->migrationsPath = "$appPath/appinfo/Migrations";
$this->migrationsNamespace = "OCA\\$appName\\Migrations";
}
if (!is_dir($this->migrationsPath)) {
if (!mkdir($this->migrationsPath)) {
throw new \Exception("Could not create migration folder \"{$this->migrationsPath}\"");
};
}
}
private static function requireOnce($file) {
require_once $file;
}
/**
* Returns the name of the app for which this migration is executed
*
* @return string
*/
public function getApp() {
return $this->appName;
}
/**
* @return bool
* @codeCoverageIgnore - this will implicitly tested on installation
*/
private function createMigrationTable() {
if ($this->migrationTableCreated) {
return false;
}
if ($this->connection->tableExists('migrations')) {
$this->migrationTableCreated = true;
return false;
}
$tableName = $this->connection->getPrefix() . 'migrations';
$tableName = $this->connection->getDatabasePlatform()->quoteIdentifier($tableName);
$columns = [
'app' => new Column($this->connection->getDatabasePlatform()->quoteIdentifier('app'), Type::getType('string'), ['length' => 255]),
'version' => new Column($this->connection->getDatabasePlatform()->quoteIdentifier('version'), Type::getType('string'), ['length' => 255]),
];
$table = new Table($tableName, $columns);
$table->setPrimaryKey([
$this->connection->getDatabasePlatform()->quoteIdentifier('app'),
$this->connection->getDatabasePlatform()->quoteIdentifier('version')]);
$this->connection->getSchemaManager()->createTable($table);
$this->migrationTableCreated = true;
return true;
}
/**
* Returns all versions which have already been applied
*
* @return string[]
* @codeCoverageIgnore - no need to test this
*/
public function getMigratedVersions() {
$this->createMigrationTable();
$qb = $this->connection->getQueryBuilder();
$qb->select('version')
->from('migrations')
->where($qb->expr()->eq('app', $qb->createNamedParameter($this->getApp())))
->orderBy('version');
$result = $qb->execute();
$rows = $result->fetchAll(\PDO::FETCH_COLUMN);
$result->closeCursor();
return $rows;
}
/**
* Returns all versions which are available in the migration folder
*
* @return array
*/
public function getAvailableVersions() {
$this->ensureMigrationsAreLoaded();
return array_keys($this->migrations);
}
protected function findMigrations() {
$directory = realpath($this->migrationsPath);
$iterator = new \RegexIterator(
new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::LEAVES_ONLY
),
'#^.+\\/Version[^\\/]{1,255}\\.php$#i',
\RegexIterator::GET_MATCH);
$files = array_keys(iterator_to_array($iterator));
uasort($files, function ($a, $b) {
return (basename($a) < basename($b)) ? -1 : 1;
});
$migrations = [];
foreach ($files as $file) {
static::requireOnce($file);
$className = basename($file, '.php');
$version = (string) substr($className, 7);
if ($version === '0') {
throw new \InvalidArgumentException(
"Cannot load a migrations with the name '$version' because it is a reserved number"
);
}
$migrations[$version] = sprintf('%s\\%s', $this->migrationsNamespace, $className);
}
return $migrations;
}
/**
* @param string $to
*/
private function getMigrationsToExecute($to) {
$knownMigrations = $this->getMigratedVersions();
$availableMigrations = $this->getAvailableVersions();
$toBeExecuted = [];
foreach ($availableMigrations as $v) {
if ($to !== 'latest' && $v > $to) {
continue;
}
if ($this->shallBeExecuted($v, $knownMigrations)) {
$toBeExecuted[] = $v;
}
}
return $toBeExecuted;
}
/**
* @param string[] $knownMigrations
*/
private function shallBeExecuted($m, $knownMigrations) {
if (in_array($m, $knownMigrations)) {
return false;
}
return true;
}
/**
* @param string $version
*/
private function markAsExecuted($version) {
$this->connection->insertIfNotExist('*PREFIX*migrations', [
'app' => $this->appName,
'version' => $version
]);
}
/**
* Returns the name of the table which holds the already applied versions
*
* @return string
*/
public function getMigrationsTableName() {
return $this->connection->getPrefix() . 'migrations';
}
/**
* Returns the namespace of the version classes
*
* @return string
*/
public function getMigrationsNamespace() {
return $this->migrationsNamespace;
}
/**
* Returns the directory which holds the versions
*
* @return string
*/
public function getMigrationsDirectory() {
return $this->migrationsPath;
}
/**
* Return the explicit version for the aliases; current, next, prev, latest
*
* @param string $alias
* @return mixed|null|string
*/
public function getMigration($alias) {
switch($alias) {
case 'current':
return $this->getCurrentVersion();
case 'next':
return $this->getRelativeVersion($this->getCurrentVersion(), 1);
case 'prev':
return $this->getRelativeVersion($this->getCurrentVersion(), -1);
case 'latest':
$this->ensureMigrationsAreLoaded();
return @end($this->getAvailableVersions());
}
return '0';
}
/**
* @param string $version
* @param int $delta
* @return null|string
*/
private function getRelativeVersion($version, $delta) {
$this->ensureMigrationsAreLoaded();
$versions = $this->getAvailableVersions();
array_unshift($versions, 0);
$offset = array_search($version, $versions);
if ($offset === false || !isset($versions[$offset + $delta])) {
// Unknown version or delta out of bounds.
return null;
}
return (string) $versions[$offset + $delta];
}
/**
* @return string
*/
private function getCurrentVersion() {
$m = $this->getMigratedVersions();
if (count($m) === 0) {
return '0';
}
return @end(array_values($m));
}
/**
* @return string
*/
private function getClass($version) {
$this->ensureMigrationsAreLoaded();
if (isset($this->migrations[$version])) {
return $this->migrations[$version];
}
throw new \InvalidArgumentException("Version $version is unknown.");
}
/**
* Allows to set an IOutput implementation which is used for logging progress and messages
*
* @param IOutput $output
*/
public function setOutput(IOutput $output) {
$this->output = $output;
}
/**
* Applies all not yet applied versions up to $to
*
* @param string $to
*/
public function migrate($to = 'latest') {
// read known migrations
$toBeExecuted = $this->getMigrationsToExecute($to);
foreach ($toBeExecuted as $version) {
$this->executeStep($version);
}
}
/**
* @param string $version
* @return mixed
* @throws \Exception
*/
protected function createInstance($version) {
$class = $this->getClass($version);
try {
$s = \OC::$server->query($class);
} catch (QueryException $e) {
if (class_exists($class)) {
$s = new $class();
} else {
throw new \Exception("Migration step '$class' is unknown");
}
}
return $s;
}
/**
* Executes one explicit version
*
* @param string $version
*/
public function executeStep($version) {
// FIXME our interface
$instance = $this->createInstance($version);
if ($instance instanceof ISimpleMigration) {
$instance->run($this->output);
}
if ($instance instanceof ISqlMigration) {
$sqls = $instance->sql($this->connection);
foreach ($sqls as $s) {
$this->connection->executeQuery($s);
}
}
if ($instance instanceof ISchemaMigration) {
$toSchema = $this->connection->createSchema();
$instance->changeSchema($toSchema, ['tablePrefix' => $this->connection->getPrefix()]);
$this->connection->migrateToSchema($toSchema);
}
$this->markAsExecuted($version);
}
private function ensureMigrationsAreLoaded() {
if (empty($this->migrations)) {
$this->migrations = $this->findMigrations();
}
}
}

View File

@ -43,14 +43,10 @@ use Symfony\Component\EventDispatcher\GenericEvent;
class Migrator {
/**
* @var \Doctrine\DBAL\Connection $connection
*/
/** @var \Doctrine\DBAL\Connection */
protected $connection;
/**
* @var ISecureRandom
*/
/** @var ISecureRandom */
private $random;
/** @var IConfig */
@ -197,6 +193,12 @@ class Migrator {
return new Table($newName, $table->getColumns(), $newIndexes, array(), 0, $table->getOptions());
}
public function createSchema() {
$filterExpression = $this->getFilterExpression();
$this->connection->getConfiguration()->setFilterSchemaAssetsExpression($filterExpression);
return $this->connection->getSchemaManager()->createSchema();
}
/**
* @param Schema $targetSchema
* @param \Doctrine\DBAL\Connection $connection
@ -217,8 +219,7 @@ class Migrator {
}
$filterExpression = $this->getFilterExpression();
$this->connection->getConfiguration()->
setFilterSchemaAssetsExpression($filterExpression);
$this->connection->getConfiguration()->setFilterSchemaAssetsExpression($filterExpression);
$sourceSchema = $connection->getSchemaManager()->createSchema();
// remove tables we don't know about

View File

@ -30,9 +30,14 @@ class OracleConnection extends Connection {
* Quote the keys of the array
*/
private function quoteKeys(array $data) {
$return = array();
$return = [];
$c = $this->getDatabasePlatform()->getIdentifierQuoteCharacter();
foreach($data as $key => $value) {
$return[$this->quoteIdentifier($key)] = $value;
if ($key[0] !== $c) {
$return[$this->quoteIdentifier($key)] = $value;
} else {
$return[$key] = $value;
}
}
return $return;
}
@ -41,7 +46,9 @@ class OracleConnection extends Connection {
* {@inheritDoc}
*/
public function insert($tableName, array $data, array $types = array()) {
$tableName = $this->quoteIdentifier($tableName);
if ($tableName[0] !== $this->getDatabasePlatform()->getIdentifierQuoteCharacter()) {
$tableName = $this->quoteIdentifier($tableName);
}
$data = $this->quoteKeys($data);
return parent::insert($tableName, $data, $types);
}
@ -50,7 +57,9 @@ class OracleConnection extends Connection {
* {@inheritDoc}
*/
public function update($tableName, array $data, array $identifier, array $types = array()) {
$tableName = $this->quoteIdentifier($tableName);
if ($tableName[0] !== $this->getDatabasePlatform()->getIdentifierQuoteCharacter()) {
$tableName = $this->quoteIdentifier($tableName);
}
$data = $this->quoteKeys($data);
$identifier = $this->quoteKeys($identifier);
return parent::update($tableName, $data, $identifier, $types);
@ -60,9 +69,11 @@ class OracleConnection extends Connection {
* {@inheritDoc}
*/
public function delete($tableExpression, array $identifier, array $types = array()) {
$tableName = $this->quoteIdentifier($tableExpression);
if ($tableExpression[0] !== $this->getDatabasePlatform()->getIdentifierQuoteCharacter()) {
$tableExpression = $this->quoteIdentifier($tableExpression);
}
$identifier = $this->quoteKeys($identifier);
return parent::delete($tableName, $identifier);
return parent::delete($tableExpression, $identifier);
}
/**

View File

@ -24,19 +24,75 @@
namespace OC\DB;
use Doctrine\DBAL\DBALException;
use Doctrine\DBAL\Schema\Column;
use Doctrine\DBAL\Schema\ColumnDiff;
use Doctrine\DBAL\Schema\Index;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Schema\Table;
class OracleMigrator extends NoCheckMigrator {
/**
* @param Schema $targetSchema
* @param \Doctrine\DBAL\Connection $connection
* @return \Doctrine\DBAL\Schema\SchemaDiff
* @throws DBALException
*/
protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) {
$schemaDiff = parent::getDiff($targetSchema, $connection);
// oracle forces us to quote the identifiers
$schemaDiff->newTables = array_map(function(Table $table) {
return new Table(
$this->connection->quoteIdentifier($table->getName()),
array_map(function(Column $column) {
$newColumn = new Column(
$this->connection->quoteIdentifier($column->getName()),
$column->getType()
);
$newColumn->setAutoincrement($column->getAutoincrement());
$newColumn->setColumnDefinition($column->getColumnDefinition());
$newColumn->setComment($column->getComment());
$newColumn->setDefault($column->getDefault());
$newColumn->setFixed($column->getFixed());
$newColumn->setLength($column->getLength());
$newColumn->setNotnull($column->getNotnull());
$newColumn->setPrecision($column->getPrecision());
$newColumn->setScale($column->getScale());
$newColumn->setUnsigned($column->getUnsigned());
$newColumn->setPlatformOptions($column->getPlatformOptions());
$newColumn->setCustomSchemaOptions($column->getPlatformOptions());
return $newColumn;
}, $table->getColumns()),
array_map(function(Index $index) {
return new Index(
$this->connection->quoteIdentifier($index->getName()),
array_map(function($columnName) {
return $this->connection->quoteIdentifier($columnName);
}, $index->getColumns()),
$index->isUnique(),
$index->isPrimary(),
$index->getFlags(),
$index->getOptions()
);
}, $table->getIndexes()),
$table->getForeignKeys(),
0,
$table->getOptions()
);
}, $schemaDiff->newTables);
$schemaDiff->removedTables = array_map(function(Table $table) {
return new Table(
$this->connection->quoteIdentifier($table->getName()),
$table->getColumns(),
$table->getIndexes(),
$table->getForeignKeys(),
0,
$table->getOptions()
);
}, $schemaDiff->removedTables);
foreach ($schemaDiff->changedTables as $tableDiff) {
$tableDiff->name = $this->connection->quoteIdentifier($tableDiff->name);
foreach ($tableDiff->changedColumns as $column) {

View File

@ -0,0 +1,84 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Migration;
use OCP\ILogger;
use OCP\Migration\IOutput;
/**
* Class SimpleOutput
*
* Just a simple IOutput implementation with writes messages to the log file.
* Alternative implementations will write to the console or to the web ui (web update case)
*
* @package OC\Migration
*/
class SimpleOutput implements IOutput {
/** @var ILogger */
private $logger;
private $appName;
public function __construct(ILogger $logger, $appName) {
$this->logger = $logger;
$this->appName = $appName;
}
/**
* @param string $message
* @since 9.1.0
*/
public function info($message) {
$this->logger->info($message, ['app' => $this->appName]);
}
/**
* @param string $message
* @since 9.1.0
*/
public function warning($message) {
$this->logger->warning($message, ['app' => $this->appName]);
}
/**
* @param int $max
* @since 9.1.0
*/
public function startProgress($max = 0) {
}
/**
* @param int $step
* @param string $description
* @since 9.1.0
*/
public function advance($step = 1, $description = '') {
}
/**
* @since 9.1.0
*/
public function finishProgress() {
}
}

View File

@ -332,6 +332,8 @@ class Setup {
try {
$dbSetup->initialize($options);
$dbSetup->setupDatabase($username);
// apply necessary migrations
$dbSetup->runMigrations();
} catch (\OC\DatabaseSetupException $e) {
$error[] = array(
'error' => $e->getMessage(),

View File

@ -27,6 +27,7 @@
namespace OC\Setup;
use OC\DB\ConnectionFactory;
use OC\DB\MigrationService;
use OC\SystemConfig;
use OCP\IL10N;
use OCP\ILogger;
@ -143,4 +144,12 @@ abstract class AbstractDatabase {
* @param string $userName
*/
abstract public function setupDatabase($userName);
public function runMigrations() {
if (!is_dir(\OC::$SERVERROOT."/core/Migrations")) {
return;
}
$ms = new MigrationService('core', \OC::$server->getDatabaseConnection());
$ms->migrate();
}
}

View File

@ -32,6 +32,7 @@
namespace OC;
use OC\DB\MigrationService;
use OC\Hooks\BasicEmitter;
use OC\IntegrityCheck\Checker;
use OC_App;
@ -300,8 +301,11 @@ class Updater extends BasicEmitter {
protected function doCoreUpgrade() {
$this->emit('\OC\Updater', 'dbUpgradeBefore');
// do the real upgrade
\OC_DB::updateDbFromStructure(\OC::$SERVERROOT . '/db_structure.xml');
// execute core migrations
if (is_dir(\OC::$SERVERROOT . '/core/Migrations')) {
$ms = new MigrationService('core', \OC::$server->getDatabaseConnection());
$ms->migrate();
}
$this->emit('\OC\Updater', 'dbUpgrade');
}

View File

@ -50,6 +50,7 @@
use OC\App\DependencyAnalyzer;
use OC\App\InfoParser;
use OC\App\Platform;
use OC\DB\MigrationService;
use OC\Installer;
use OC\Repair;
use OCP\App\ManagerEvent;
@ -1043,12 +1044,18 @@ class OC_App {
}
$appData = self::getAppInfo($appId);
self::executeRepairSteps($appId, $appData['repair-steps']['pre-migration']);
if (file_exists($appPath . '/appinfo/database.xml')) {
if (isset($appData['use-migrations']) && $appData['use-migrations'] === 'true') {
$ms = new MigrationService($appId, \OC::$server->getDatabaseConnection());
$ms->migrate();
} else if (file_exists($appPath . '/appinfo/database.xml')) {
OC_DB::updateDbFromStructure($appPath . '/appinfo/database.xml');
}
self::executeRepairSteps($appId, $appData['repair-steps']['post-migration']);
self::setupLiveMigrations($appId, $appData['repair-steps']['live-migration']);
unset(self::$appVersion[$appId]);
// run upgrade code
if (file_exists($appPath . '/appinfo/update.php')) {
self::loadApp($appId);

View File

@ -34,6 +34,7 @@
// use OCP namespace for all classes that are considered public.
// This means that they should be used by apps instead of the internal ownCloud classes
namespace OCP;
use Doctrine\DBAL\Schema\Schema;
use OCP\DB\QueryBuilder\IQueryBuilder;
/**
@ -259,4 +260,20 @@ interface IDBConnection {
* @since 11.0.0
*/
public function supports4ByteText();
/**
* Create the schema of the connected database
*
* @return Schema
* @since 13.0.0
*/
public function createSchema();
/**
* Migrate the database to the given schema
*
* @param Schema $toSchema
* @since 13.0.0
*/
public function migrateToSchema(Schema $toSchema);
}

View File

@ -0,0 +1,49 @@
<?php
/**
* @copyright Copyright (c) 2017 Joas Schilling <coding@schilljs.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCP\Migration;
use Doctrine\DBAL\Schema\Schema;
/**
* @since 13.0.0
*/
interface IMigrationStep {
/**
* @param IOutput $output
* @since 13.0.0
*/
public function preSchemaChange(IOutput $output);
/**
* @param Schema $schema
* @param array $options
* @since 13.0.0
*/
public function changeSchema(Schema $schema, array $options);
/**
* @param IOutput $output
* @since 13.0.0
*/
public function postSchemaChange(IOutput $output);
}

View File

@ -0,0 +1,162 @@
<?php
/**
* Copyright (c) 2016 Thomas Müller <thomas.mueller@tmit.eu>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace Test\DB;
use Doctrine\DBAL\Schema\Schema;
use OC\DB\Connection;
use OC\DB\MigrationService;
use OCP\IDBConnection;
use OCP\Migration\ISchemaMigration;
use OCP\Migration\ISqlMigration;
/**
* Class MigrationsTest
*
* @package Test\DB
*/
class MigrationsTest extends \Test\TestCase {
/** @var MigrationService | \PHPUnit_Framework_MockObject_MockObject */
private $migrationService;
/** @var \PHPUnit_Framework_MockObject_MockObject | IDBConnection $db */
private $db;
public function setUp() {
parent::setUp();
$this->db = $this->createMock(Connection::class);
$this->db->expects($this->any())->method('getPrefix')->willReturn('test_oc_');
$this->migrationService = new MigrationService('testing', $this->db);
}
public function testGetters() {
$this->assertEquals('testing', $this->migrationService->getApp());
$this->assertEquals(\OC::$SERVERROOT . '/apps/testing/appinfo/Migrations', $this->migrationService->getMigrationsDirectory());
$this->assertEquals('OCA\testing\Migrations', $this->migrationService->getMigrationsNamespace());
$this->assertEquals('test_oc_migrations', $this->migrationService->getMigrationsTableName());
}
public function testCore() {
$this->migrationService = new MigrationService('core', $this->db);
$this->assertEquals('core', $this->migrationService->getApp());
$this->assertEquals(\OC::$SERVERROOT . '/core/Migrations', $this->migrationService->getMigrationsDirectory());
$this->assertEquals('OC\Migrations', $this->migrationService->getMigrationsNamespace());
$this->assertEquals('test_oc_migrations', $this->migrationService->getMigrationsTableName());
}
/**
* @expectedException \InvalidArgumentException
* @expectedExceptionMessage Version 20170130180000 is unknown.
*/
public function testExecuteUnknownStep() {
$this->migrationService->executeStep('20170130180000');
}
/**
* @expectedException \Exception
* @expectedExceptionMessage App not found
*/
public function testUnknownApp() {
$migrationService = new MigrationService('unknown-bloody-app', $this->db);
}
/**
* @expectedException \Exception
* @expectedExceptionMessage Migration step 'X' is unknown
*/
public function testExecuteStepWithUnknownClass() {
$this->migrationService = $this->getMockBuilder(MigrationService::class)
->setMethods(['findMigrations'])
->setConstructorArgs(['testing', $this->db])
->getMock();
$this->migrationService->expects($this->any())->method('findMigrations')->willReturn(
['20170130180000' => 'X', '20170130180001' => 'Y', '20170130180002' => 'Z', '20170130180003' => 'A']
);
$this->migrationService->executeStep('20170130180000');
}
public function testExecuteStepWithSchemaMigrationStep() {
$schema = $this->createMock(Schema::class);
$this->db->expects($this->any())->method('createSchema')->willReturn($schema);
$step = $this->createMock(ISchemaMigration::class);
$step->expects($this->once())->method('changeSchema');
$this->migrationService = $this->getMockBuilder(MigrationService::class)
->setMethods(['createInstance'])
->setConstructorArgs(['testing', $this->db])
->getMock();
$this->migrationService->expects($this->any())->method('createInstance')->with('20170130180000')->willReturn($step);
$this->migrationService->executeStep('20170130180000');
}
public function testExecuteStepWithSqlMigrationStep() {
$this->db->expects($this->exactly(3))->method('executeQuery')->withConsecutive(['1'], ['2'], ['3']);
$step = $this->createMock(ISqlMigration::class);
$step->expects($this->once())->method('sql')->willReturn(['1', '2', '3']);
$this->migrationService = $this->getMockBuilder(MigrationService::class)
->setMethods(['createInstance'])
->setConstructorArgs(['testing', $this->db])
->getMock();
$this->migrationService->expects($this->any())->method('createInstance')->with('20170130180000')->willReturn($step);
$this->migrationService->executeStep('20170130180000');
}
public function testGetMigration() {
$this->migrationService = $this->getMockBuilder(MigrationService::class)
->setMethods(['getMigratedVersions', 'findMigrations'])
->setConstructorArgs(['testing', $this->db])
->getMock();
$this->migrationService->expects($this->any())->method('getMigratedVersions')->willReturn(
['20170130180000', '20170130180001']
);
$this->migrationService->expects($this->any())->method('findMigrations')->willReturn(
['20170130180000' => 'X', '20170130180001' => 'Y', '20170130180002' => 'Z', '20170130180003' => 'A']
);
$this->assertEquals(
['20170130180000', '20170130180001', '20170130180002', '20170130180003'],
$this->migrationService->getAvailableVersions());
$migration = $this->migrationService->getMigration('current');
$this->assertEquals('20170130180001', $migration);
$migration = $this->migrationService->getMigration('prev');
$this->assertEquals('20170130180000', $migration);
$migration = $this->migrationService->getMigration('next');
$this->assertEquals('20170130180002', $migration);
$migration = $this->migrationService->getMigration('latest');
$this->assertEquals('20170130180003', $migration);
}
public function testMigrate() {
$this->migrationService = $this->getMockBuilder(MigrationService::class)
->setMethods(['getMigratedVersions', 'findMigrations', 'executeStep'])
->setConstructorArgs(['testing', $this->db])
->getMock();
$this->migrationService->expects($this->any())->method('getMigratedVersions')->willReturn(
['20170130180000', '20170130180001']
);
$this->migrationService->expects($this->any())->method('findMigrations')->willReturn(
['20170130180000' => 'X', '20170130180001' => 'Y', '20170130180002' => 'Z', '20170130180003' => 'A']
);
$this->assertEquals(
['20170130180000', '20170130180001', '20170130180002', '20170130180003'],
$this->migrationService->getAvailableVersions());
$this->migrationService->expects($this->exactly(2))->method('executeStep')
->withConsecutive(['20170130180002'], ['20170130180003']);
$this->migrationService->migrate();
}
}