mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #6457 from owncloud/db-convert-tool 

Command line tool to convert current database to others, except sqlite
This commit is contained in:
Frank Karlitschek 2014-05-19 16:41:31 +01:00
parent 95741f3936 8314e5f4d1
commit 090d127050
5 changed files with 494 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

295
core/command/db/converttype.php Normal file
View File

@ -0,0 +1,295 @@
<?php
/**
* Copyright (c) 2013 Bart Visscher <bartv@thisnet.nl>
* Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*
*/
namespace OC\Core\Command\Db;
use OC\Config;
use OC\DB\Connection;
use OC\DB\ConnectionFactory;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ConvertType extends Command {
/**
* @var \OC\Config
*/
protected $config;
/**
* @var \OC\DB\ConnectionFactory
*/
protected $connectionFactory;
/**
* @param \OC\Config $config
* @param \OC\DB\ConnectionFactory $connectionFactory
*/
public function __construct(Config $config, ConnectionFactory $connectionFactory) {
$this->config = $config;
$this->connectionFactory = $connectionFactory;
parent::__construct();
}
protected function configure() {
$this
->setName('db:convert-type')
->setDescription('Convert the ownCloud database to the newly configured one')
->addArgument(
'type',
InputArgument::REQUIRED,
'the type of the database to convert to'
)
->addArgument(
'username',
InputArgument::REQUIRED,
'the username of the database to convert to'
)
->addArgument(
'hostname',
InputArgument::REQUIRED,
'the hostname of the database to convert to'
)
->addArgument(
'database',
InputArgument::REQUIRED,
'the name of the database to convert to'
)
->addOption(
'port',
null,
InputOption::VALUE_REQUIRED,
'the port of the database to convert to'
)
->addOption(
'password',
null,
InputOption::VALUE_REQUIRED,
'the password of the database to convert to. Will be asked when not specified. Can also be passed via stdin.'
)
->addOption(
'clear-schema',
null,
InputOption::VALUE_NONE,
'remove all tables from the destination database'
)
->addOption(
'all-apps',
null,
InputOption::VALUE_NONE,
'whether to create schema for all apps instead of only installed apps'
)
;
}
protected function validateInput(InputInterface $input, OutputInterface $output) {
$type = $this->connectionFactory->normalizeType($input->getArgument('type'));
if ($type === 'sqlite3') {
throw new \InvalidArgumentException(
'Converting to SQLite (sqlite3) is currently not supported.'
);
}
if ($type === 'mssql') {
throw new \InvalidArgumentException(
'Converting to Microsoft SQL Server (mssql) is currently not supported.'
);
}
if ($type === $this->config->getValue('dbtype', '')) {
throw new \InvalidArgumentException(sprintf(
'Can not convert from %1$s to %1$s.',
$type
));
}
if ($type === 'oci' && $input->getOption('clear-schema')) {
// Doctrine unconditionally tries (at least in version 2.3)
// to drop sequence triggers when dropping a table, even though
// such triggers may not exist. This results in errors like
// "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist".
throw new \InvalidArgumentException(
'The --clear-schema option is not supported when converting to Oracle (oci).'
);
}
}
protected function readPassword(InputInterface $input, OutputInterface $output) {
// Explicitly specified password
if ($input->getOption('password')) {
return;
}
// Read from stdin. stream_set_blocking is used to prevent blocking
// when nothing is passed via stdin.
stream_set_blocking(STDIN, 0);
$password = file_get_contents('php://stdin');
stream_set_blocking(STDIN, 1);
if (trim($password) !== '') {
$input->setOption('password', $password);
return;
}
// Read password by interacting
if ($input->isInteractive()) {
/** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */
$dialog = $this->getHelperSet()->get('dialog');
$password = $dialog->askHiddenResponse(
$output,
'<question>What is the database password?</question>',
false
);
$input->setOption('password', $password);
return;
}
}
protected function execute(InputInterface $input, OutputInterface $output) {
$this->validateInput($input, $output);
$this->readPassword($input, $output);
$fromDB = \OC_DB::getConnection();
$toDB = $this->getToDBConnection($input, $output);
if ($input->getOption('clear-schema')) {
$this->clearSchema($toDB, $input, $output);
}
$this->createSchema($toDB, $input, $output);
$toTables = $this->getTables($toDB);
$fromTables = $this->getTables($fromDB);
// warn/fail if there are more tables in 'from' database
$extraFromTables = array_diff($fromTables, $toTables);
if (!empty($extraFromTables)) {
$output->writeln('<comment>The following tables will not be converted:</comment>');
$output->writeln($extraFromTables);
if (!$input->getOption('all-apps')) {
$output->writeln('<comment>Please note that tables belonging to available but currently not installed apps</comment>');
$output->writeln('<comment>can be included by specifying the --all-apps option.</comment>');
}
/** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */
$dialog = $this->getHelperSet()->get('dialog');
if (!$dialog->askConfirmation(
$output,
'<question>Continue with the conversion?</question>',
false
)) {
return;
}
}
$intersectingTables = array_intersect($toTables, $fromTables);
$this->convertDB($fromDB, $toDB, $intersectingTables, $input, $output);
}
protected function createSchema(Connection $toDB, InputInterface $input, OutputInterface $output) {
$output->writeln('<info>Creating schema in new database</info>');
$schemaManager = new \OC\DB\MDB2SchemaManager($toDB);
$schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml');
$apps = $input->getOption('all-apps') ? \OC_App::getAllApps() : \OC_App::getEnabledApps();
foreach($apps as $app) {
if (file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) {
$schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml');
}
}
}
protected function getToDBConnection(InputInterface $input, OutputInterface $output) {
$type = $input->getArgument('type');
$connectionParams = array(
'host' => $input->getArgument('hostname'),
'user' => $input->getArgument('username'),
'password' => $input->getOption('password'),
'dbname' => $input->getArgument('database'),
'tablePrefix' => $this->config->getValue('dbtableprefix', 'oc_'),
);
if ($input->getOption('port')) {
$connectionParams['port'] = $input->getOption('port');
}
return $this->connectionFactory->getConnection($type, $connectionParams);
}
protected function clearSchema(Connection $db, InputInterface $input, OutputInterface $output) {
$toTables = $this->getTables($db);
if (!empty($toTables)) {
$output->writeln('<info>Clearing schema in new database</info>');
}
foreach($toTables as $table) {
$db->getSchemaManager()->dropTable($table);
}
}
protected function getTables(Connection $db) {
return $db->getSchemaManager()->listTableNames();
}
protected function copyTable(Connection $fromDB, Connection $toDB, $table, InputInterface $input, OutputInterface $output) {
/** @var $progress \Symfony\Component\Console\Helper\ProgressHelper */
$progress = $this->getHelperSet()->get('progress');
$query = 'SELECT COUNT(*) FROM '.$table;
$count = $fromDB->fetchColumn($query);
$query = 'SELECT * FROM '.$table;
$statement = $fromDB->executeQuery($query);
$progress->start($output, $count);
$progress->setRedrawFrequency($count > 100 ? 5 : 1);
while($row = $statement->fetch()) {
$progress->advance();
if ($input->getArgument('type') === 'oci') {
$data = $row;
} else {
$data = array();
foreach ($row as $columnName => $value) {
$data[$toDB->quoteIdentifier($columnName)] = $value;
}
}
$toDB->insert($table, $data);
}
$progress->finish();
}
protected function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) {
$this->config->setValue('maintenance', true);
try {
// copy table rows
foreach($tables as $table) {
$output->writeln($table);
$this->copyTable($fromDB, $toDB, $table, $input, $output);
}
if ($input->getArgument('type') === 'pgsql') {
$tools = new \OC\DB\PgSqlTools;
$tools->resynchronizeDatabaseSequences($toDB);
}
// save new database config
$this->saveDBInfo($input);
} catch(\Exception $e) {
$this->config->setValue('maintenance', false);
throw $e;
}
$this->config->setValue('maintenance', false);
}
protected function saveDBInfo(InputInterface $input) {
$type = $input->getArgument('type');
$username = $input->getArgument('username');
$dbhost = $input->getArgument('hostname');
$dbname = $input->getArgument('database');
$password = $input->getOption('password');
if ($input->getOption('port')) {
$dbhost .= ':'.$input->getOption('port');
}
$this->config->setValue('dbtype', $type);
$this->config->setValue('dbname', $dbname);
$this->config->setValue('dbhost', $dbhost);
$this->config->setValue('dbuser', $username);
$this->config->setValue('dbpassword', $password);
}
}

1
core/register_command.php
View File

@ -9,6 +9,7 @@
/** @var $application Symfony\Component\Console\Application */
$application->add(new OC\Core\Command\Status);
$application->add(new OC\Core\Command\Db\GenerateChangeScript());
$application->add(new OC\Core\Command\Db\ConvertType(OC_Config::getObject(), new \OC\DB\ConnectionFactory()));
$application->add(new OC\Core\Command\Upgrade());
$application->add(new OC\Core\Command\Maintenance\SingleUser());
$application->add(new OC\Core\Command\App\Disable());

137
lib/private/db.php
View File

@ -72,102 +72,45 @@ class OC_DB {
$port=false;
}
// do nothing if the connection already has been established
if (!self::$connection) {
$config = new \Doctrine\DBAL\Configuration();
$eventManager = new \Doctrine\Common\EventManager();
switch($type) {
case 'sqlite':
case 'sqlite3':
$datadir=OC_Config::getValue( "datadirectory", OC::$SERVERROOT.'/data' );
$connectionParams = array(
'user' => $user,
'password' => $pass,
'path' => $datadir.'/'.$name.'.db',
'driver' => 'pdo_sqlite',
);
$connectionParams['adapter'] = '\OC\DB\AdapterSqlite';
$connectionParams['wrapperClass'] = 'OC\DB\Connection';
break;
case 'mysql':
$connectionParams = array(
'user' => $user,
'password' => $pass,
'host' => $host,
'port' => $port,
'dbname' => $name,
'charset' => 'UTF8',
'driver' => 'pdo_mysql',
);
$connectionParams['adapter'] = '\OC\DB\Adapter';
$connectionParams['wrapperClass'] = 'OC\DB\Connection';
// Send "SET NAMES utf8". Only required on PHP 5.3 below 5.3.6.
// See http://stackoverflow.com/questions/4361459/php-pdo-charset-set-names#4361485
$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\MysqlSessionInit);
break;
case 'pgsql':
$connectionParams = array(
'user' => $user,
'password' => $pass,
'host' => $host,
'port' => $port,
'dbname' => $name,
'driver' => 'pdo_pgsql',
);
$connectionParams['adapter'] = '\OC\DB\AdapterPgSql';
$connectionParams['wrapperClass'] = 'OC\DB\Connection';
break;
case 'oci':
$connectionParams = array(
'user' => $user,
'password' => $pass,
'host' => $host,
'dbname' => $name,
'charset' => 'AL32UTF8',
'driver' => 'oci8',
);
if (!empty($port)) {
$connectionParams['port'] = $port;
}
$connectionParams['adapter'] = '\OC\DB\AdapterOCI8';
$connectionParams['wrapperClass'] = 'OC\DB\OracleConnection';
$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\OracleSessionInit);
break;
case 'mssql':
$connectionParams = array(
'user' => $user,
'password' => $pass,
'host' => $host,
'port' => $port,
'dbname' => $name,
'charset' => 'UTF8',
'driver' => 'pdo_sqlsrv',
);
$connectionParams['adapter'] = '\OC\DB\AdapterSQLSrv';
$connectionParams['wrapperClass'] = 'OC\DB\Connection';
break;
default:
return false;
}
$connectionParams['tablePrefix'] = OC_Config::getValue('dbtableprefix', 'oc_' );
try {
self::$connection = \Doctrine\DBAL\DriverManager::getConnection($connectionParams, $config, $eventManager);
if ($type === 'sqlite' || $type === 'sqlite3') {
// Sqlite doesn't handle query caching and schema changes
// TODO: find a better way to handle this
self::$connection->disableQueryStatementCaching();
}
} catch(\Doctrine\DBAL\DBALException $e) {
OC_Log::write('core', $e->getMessage(), OC_Log::FATAL);
OC_User::setUserId(null);
$factory = new \OC\DB\ConnectionFactory();
if (!$factory->isValidType($type)) {
return false;
}
// send http status 503
header('HTTP/1.1 503 Service Temporarily Unavailable');
header('Status: 503 Service Temporarily Unavailable');
OC_Template::printErrorPage('Failed to connect to database');
die();
if ($factory->normalizeType($type) === 'sqlite3') {
$datadir = OC_Config::getValue("datadirectory", OC::$SERVERROOT.'/data');
$connectionParams = array(
'user' => $user,
'password' => $pass,
'path' => $datadir.'/'.$name.'.db',
);
} else {
$connectionParams = array(
'user' => $user,
'password' => $pass,
'host' => $host,
'dbname' => $name,
);
if (!empty($port)) {
$connectionParams['port'] = $port;
}
}
$connectionParams['tablePrefix'] = OC_Config::getValue('dbtableprefix', 'oc_');
try {
self::$connection = $factory->getConnection($type, $connectionParams);
} catch(\Doctrine\DBAL\DBALException $e) {
OC_Log::write('core', $e->getMessage(), OC_Log::FATAL);
OC_User::setUserId(null);
// send http status 503
header('HTTP/1.1 503 Service Temporarily Unavailable');
header('Status: 503 Service Temporarily Unavailable');
OC_Template::printErrorPage('Failed to connect to database');
die();
}
return true;
}
@ -202,12 +145,12 @@ class OC_DB {
*/
static public function prepare( $query , $limit = null, $offset = null, $isManipulation = null) {
self::connect();
if ($isManipulation === null) {
//try to guess, so we return the number of rows on manipulations
$isManipulation = self::isManipulation($query);
}
// return the result
try {
$result = self::$connection->prepare($query, $limit, $offset);
@ -222,7 +165,7 @@ class OC_DB {
/**
* tries to guess the type of statement based on the first 10 characters
* the current check allows some whitespace but does not work with IF EXISTS or other more complex statements
*
*
* @param string $sql
* @return bool
*/
@ -245,7 +188,7 @@ class OC_DB {
}
return false;
}
/**
* @brief execute a prepared statement, on error write log and throw exception
* @param mixed $stmt OC_DB_StatementWrapper,

118
lib/private/db/connectionfactory.php Normal file
View File

@ -0,0 +1,118 @@
<?php
/**
* Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\DB;
/**
* Takes care of creating and configuring Doctrine connections.
*/
class ConnectionFactory {
/**
* @var array
*
* Array mapping DBMS type to default connection parameters passed to
* \Doctrine\DBAL\DriverManager::getConnection().
*/
protected $defaultConnectionParams = array(
'mssql' => array(
'adapter' => '\OC\DB\AdapterSQLSrv',
'charset' => 'UTF8',
'driver' => 'pdo_sqlsrv',
'wrapperClass' => 'OC\DB\Connection',
),
'mysql' => array(
'adapter' => '\OC\DB\Adapter',
'charset' => 'UTF8',
'driver' => 'pdo_mysql',
'wrapperClass' => 'OC\DB\Connection',
),
'oci' => array(
'adapter' => '\OC\DB\AdapterOCI8',
'charset' => 'AL32UTF8',
'driver' => 'oci8',
'wrapperClass' => 'OC\DB\OracleConnection',
),
'pgsql' => array(
'adapter' => '\OC\DB\AdapterPgSql',
'driver' => 'pdo_pgsql',
'wrapperClass' => 'OC\DB\Connection',
),
'sqlite3' => array(
'adapter' => '\OC\DB\AdapterSqlite',
'driver' => 'pdo_sqlite',
'wrapperClass' => 'OC\DB\Connection',
),
);
/**
* @brief Get default connection parameters for a given DBMS.
* @param string $type DBMS type
* @throws \InvalidArgumentException If $type is invalid
* @return array Default connection parameters.
*/
public function getDefaultConnectionParams($type) {
$normalizedType = $this->normalizeType($type);
if (!isset($this->defaultConnectionParams[$normalizedType])) {
throw new \InvalidArgumentException("Unsupported type: $type");
}
return $this->defaultConnectionParams[$normalizedType];
}
/**
* @brief Get default connection parameters for a given DBMS.
* @param string $type DBMS type
* @param array $additionalConnectionParams Additional connection parameters
* @return \OC\DB\Connection
*/
public function getConnection($type, $additionalConnectionParams) {
$normalizedType = $this->normalizeType($type);
$eventManager = new \Doctrine\Common\EventManager();
switch ($normalizedType) {
case 'mysql':
// Send "SET NAMES utf8". Only required on PHP 5.3 below 5.3.6.
// See http://stackoverflow.com/questions/4361459/php-pdo-charset-set-names#4361485
$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\MysqlSessionInit);
break;
case 'oci':
$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\OracleSessionInit);
break;
}
$connection = \Doctrine\DBAL\DriverManager::getConnection(
array_merge($this->getDefaultConnectionParams($type), $additionalConnectionParams),
new \Doctrine\DBAL\Configuration(),
$eventManager
);
switch ($normalizedType) {
case 'sqlite3':
// Sqlite doesn't handle query caching and schema changes
// TODO: find a better way to handle this
/** @var $connection \OC\DB\Connection */
$connection->disableQueryStatementCaching();
break;
}
return $connection;
}
/**
* @brief Normalize DBMS type
* @param string $type DBMS type
* @return string Normalized DBMS type
*/
public function normalizeType($type) {
return $type === 'sqlite' ? 'sqlite3' : $type;
}
/**
* @brief Checks whether the specified DBMS type is valid.
* @return bool
*/
public function isValidType($type) {
$normalizedType = $this->normalizeType($type);
return isset($this->defaultConnectionParams[$normalizedType]);
}
}

40
lib/private/db/pgsqltools.php Normal file
View File

@ -0,0 +1,40 @@
<?php
/**
* Copyright (c) 2013 Bart Visscher <bartv@thisnet.nl>
* Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
* This file is licensed under the Affero General Public License version 3 or
* later.
* See the COPYING-README file.
*/
namespace OC\DB;
/**
* Various PostgreSQL specific helper functions.
*/
class PgSqlTools {
/**
* @brief Resynchronizes all sequences of a database after using INSERTs
* without leaving out the auto-incremented column.
* @param \OC\DB\Connection $conn
* @return null
*/
public function resynchronizeDatabaseSequences(Connection $conn) {
$databaseName = $conn->getDatabase();
foreach ($conn->getSchemaManager()->listSequences() as $sequence) {
$sequenceName = $sequence->getName();
$sqlInfo = 'SELECT table_schema, table_name, column_name
FROM information_schema.columns
WHERE column_default = ? AND table_catalog = ?';
$sequenceInfo = $conn->fetchAssoc($sqlInfo, array(
"nextval('$sequenceName'::regclass)",
$databaseName
));
$tableName = $sequenceInfo['table_name'];
$columnName = $sequenceInfo['column_name'];
$sqlMaxId = "SELECT MAX($columnName) FROM $tableName";
$sqlSetval = "SELECT setval('$sequenceName', ($sqlMaxId))";
$conn->executeQuery($sqlSetval);
}
}
}