Merge pull request #6457 from owncloud/db-convert-tool

Command line tool to convert current database to others, except sqlite

core/command/db/converttype.php

@@ -0,0 +1,295 @@
+<?php
+/**
+ * Copyright (c) 2013 Bart Visscher <bartv@thisnet.nl>
+ * Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ *
+ */
+
+namespace OC\Core\Command\Db;
+
+use OC\Config;
+use OC\DB\Connection;
+use OC\DB\ConnectionFactory;
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class ConvertType extends Command {
+	/**
+	 * @var \OC\Config
+	 */
+	protected $config;
+
+	/**
+	 * @var \OC\DB\ConnectionFactory
+	 */
+	protected $connectionFactory;
+
+	/**
+	 * @param \OC\Config $config
+	 * @param \OC\DB\ConnectionFactory $connectionFactory
+	 */
+	public function __construct(Config $config, ConnectionFactory $connectionFactory) {
+		$this->config = $config;
+		$this->connectionFactory = $connectionFactory;
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this
+			->setName('db:convert-type')
+			->setDescription('Convert the ownCloud database to the newly configured one')
+			->addArgument(
+				'type',
+				InputArgument::REQUIRED,
+				'the type of the database to convert to'
+			)
+			->addArgument(
+				'username',
+				InputArgument::REQUIRED,
+				'the username of the database to convert to'
+			)
+			->addArgument(
+				'hostname',
+				InputArgument::REQUIRED,
+				'the hostname of the database to convert to'
+			)
+			->addArgument(
+				'database',
+				InputArgument::REQUIRED,
+				'the name of the database to convert to'
+			)
+			->addOption(
+				'port',
+				null,
+				InputOption::VALUE_REQUIRED,
+				'the port of the database to convert to'
+			)
+			->addOption(
+				'password',
+				null,
+				InputOption::VALUE_REQUIRED,
+				'the password of the database to convert to. Will be asked when not specified. Can also be passed via stdin.'
+			)
+			->addOption(
+				'clear-schema',
+				null,
+				InputOption::VALUE_NONE,
+				'remove all tables from the destination database'
+			)
+			->addOption(
+				'all-apps',
+				null,
+				InputOption::VALUE_NONE,
+				'whether to create schema for all apps instead of only installed apps'
+			)
+		;
+	}
+
+	protected function validateInput(InputInterface $input, OutputInterface $output) {
+		$type = $this->connectionFactory->normalizeType($input->getArgument('type'));
+		if ($type === 'sqlite3') {
+			throw new \InvalidArgumentException(
+				'Converting to SQLite (sqlite3) is currently not supported.'
+			);
+		}
+		if ($type === 'mssql') {
+			throw new \InvalidArgumentException(
+				'Converting to Microsoft SQL Server (mssql) is currently not supported.'
+			);
+		}
+		if ($type === $this->config->getValue('dbtype', '')) {
+			throw new \InvalidArgumentException(sprintf(
+				'Can not convert from %1$s to %1$s.',
+				$type
+			));
+		}
+		if ($type === 'oci' && $input->getOption('clear-schema')) {
+			// Doctrine unconditionally tries (at least in version 2.3)
+			// to drop sequence triggers when dropping a table, even though
+			// such triggers may not exist. This results in errors like
+			// "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist".
+			throw new \InvalidArgumentException(
+				'The --clear-schema option is not supported when converting to Oracle (oci).'
+			);
+		}
+	}
+
+	protected function readPassword(InputInterface $input, OutputInterface $output) {
+		// Explicitly specified password
+		if ($input->getOption('password')) {
+			return;
+		}
+
+		// Read from stdin. stream_set_blocking is used to prevent blocking
+		// when nothing is passed via stdin.
+		stream_set_blocking(STDIN, 0);
+		$password = file_get_contents('php://stdin');
+		stream_set_blocking(STDIN, 1);
+		if (trim($password) !== '') {
+			$input->setOption('password', $password);
+			return;
+		}
+
+		// Read password by interacting
+		if ($input->isInteractive()) {
+			/** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */
+			$dialog = $this->getHelperSet()->get('dialog');
+			$password = $dialog->askHiddenResponse(
+				$output,
+				'<question>What is the database password?</question>',
+				false
+			);
+			$input->setOption('password', $password);
+			return;
+		}
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		$this->validateInput($input, $output);
+		$this->readPassword($input, $output);
+
+		$fromDB = \OC_DB::getConnection();
+		$toDB = $this->getToDBConnection($input, $output);
+
+		if ($input->getOption('clear-schema')) {
+			$this->clearSchema($toDB, $input, $output);
+		}
+
+		$this->createSchema($toDB, $input, $output);
+
+		$toTables = $this->getTables($toDB);
+		$fromTables = $this->getTables($fromDB);
+
+		// warn/fail if there are more tables in 'from' database
+		$extraFromTables = array_diff($fromTables, $toTables);
+		if (!empty($extraFromTables)) {
+			$output->writeln('<comment>The following tables will not be converted:</comment>');
+			$output->writeln($extraFromTables);
+			if (!$input->getOption('all-apps')) {
+				$output->writeln('<comment>Please note that tables belonging to available but currently not installed apps</comment>');
+				$output->writeln('<comment>can be included by specifying the --all-apps option.</comment>');
+			}
+			/** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */
+			$dialog = $this->getHelperSet()->get('dialog');
+			if (!$dialog->askConfirmation(
+				$output,
+				'<question>Continue with the conversion?</question>',
+				false
+			)) {
+				return;
+			}
+		}
+		$intersectingTables = array_intersect($toTables, $fromTables);
+		$this->convertDB($fromDB, $toDB, $intersectingTables, $input, $output);
+	}
+
+	protected function createSchema(Connection $toDB, InputInterface $input, OutputInterface $output) {
+		$output->writeln('<info>Creating schema in new database</info>');
+		$schemaManager = new \OC\DB\MDB2SchemaManager($toDB);
+		$schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml');
+		$apps = $input->getOption('all-apps') ? \OC_App::getAllApps() : \OC_App::getEnabledApps();
+		foreach($apps as $app) {
+			if (file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) {
+				$schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml');
+			}
+		}
+	}
+
+	protected function getToDBConnection(InputInterface $input, OutputInterface $output) {
+		$type = $input->getArgument('type');
+		$connectionParams = array(
+			'host' => $input->getArgument('hostname'),
+			'user' => $input->getArgument('username'),
+			'password' => $input->getOption('password'),
+			'dbname' => $input->getArgument('database'),
+			'tablePrefix' => $this->config->getValue('dbtableprefix', 'oc_'),
+		);
+		if ($input->getOption('port')) {
+			$connectionParams['port'] = $input->getOption('port');
+		}
+		return $this->connectionFactory->getConnection($type, $connectionParams);
+	}
+
+	protected function clearSchema(Connection $db, InputInterface $input, OutputInterface $output) {
+		$toTables = $this->getTables($db);
+		if (!empty($toTables)) {
+			$output->writeln('<info>Clearing schema in new database</info>');
+		}
+		foreach($toTables as $table) {
+			$db->getSchemaManager()->dropTable($table);
+		}
+	}
+
+	protected function getTables(Connection $db) {
+		return $db->getSchemaManager()->listTableNames();
+	}
+
+	protected function copyTable(Connection $fromDB, Connection $toDB, $table, InputInterface $input, OutputInterface $output) {
+		/** @var $progress \Symfony\Component\Console\Helper\ProgressHelper */
+		$progress = $this->getHelperSet()->get('progress');
+		$query = 'SELECT COUNT(*) FROM '.$table;
+		$count = $fromDB->fetchColumn($query);
+		$query = 'SELECT * FROM '.$table;
+		$statement = $fromDB->executeQuery($query);
+		$progress->start($output, $count);
+		$progress->setRedrawFrequency($count > 100 ? 5 : 1);
+		while($row = $statement->fetch()) {
+			$progress->advance();
+			if ($input->getArgument('type') === 'oci') {
+				$data = $row;
+			} else {
+				$data = array();
+				foreach ($row as $columnName => $value) {
+					$data[$toDB->quoteIdentifier($columnName)] = $value;
+				}
+			}
+			$toDB->insert($table, $data);
+		}
+		$progress->finish();
+	}
+
+	protected function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) {
+		$this->config->setValue('maintenance', true);
+		try {
+			// copy table rows
+			foreach($tables as $table) {
+				$output->writeln($table);
+				$this->copyTable($fromDB, $toDB, $table, $input, $output);
+			}
+			if ($input->getArgument('type') === 'pgsql') {
+				$tools = new \OC\DB\PgSqlTools;
+				$tools->resynchronizeDatabaseSequences($toDB);
+			}
+			// save new database config
+			$this->saveDBInfo($input);
+		} catch(\Exception $e) {
+			$this->config->setValue('maintenance', false);
+			throw $e;
+		}
+		$this->config->setValue('maintenance', false);
+	}
+
+	protected function saveDBInfo(InputInterface $input) {
+		$type = $input->getArgument('type');
+		$username = $input->getArgument('username');
+		$dbhost = $input->getArgument('hostname');
+		$dbname = $input->getArgument('database');
+		$password = $input->getOption('password');
+		if ($input->getOption('port')) {
+			$dbhost .= ':'.$input->getOption('port');
+		}
+
+		$this->config->setValue('dbtype', $type);
+		$this->config->setValue('dbname', $dbname);
+		$this->config->setValue('dbhost', $dbhost);
+		$this->config->setValue('dbuser', $username);
+		$this->config->setValue('dbpassword', $password);
+	}
+}

core/register_command.php

@@ -9,6 +9,7 @@
 /** @var $application Symfony\Component\Console\Application */
 $application->add(new OC\Core\Command\Status);
 $application->add(new OC\Core\Command\Db\GenerateChangeScript());
+$application->add(new OC\Core\Command\Db\ConvertType(OC_Config::getObject(), new \OC\DB\ConnectionFactory()));
 $application->add(new OC\Core\Command\Upgrade());
 $application->add(new OC\Core\Command\Maintenance\SingleUser());
 $application->add(new OC\Core\Command\App\Disable());

lib/private/db.php

@@ -72,102 +72,45 @@ class OC_DB {
 			$port=false;
 		}
 
-		// do nothing if the connection already has been established
+		$factory = new \OC\DB\ConnectionFactory();
-		if (!self::$connection) {
+		if (!$factory->isValidType($type)) {
-			$config = new \Doctrine\DBAL\Configuration();
+			return false;
-			$eventManager = new \Doctrine\Common\EventManager();
+		}
-			switch($type) {
-				case 'sqlite':
-				case 'sqlite3':
-					$datadir=OC_Config::getValue( "datadirectory", OC::$SERVERROOT.'/data' );
-					$connectionParams = array(
-							'user' => $user,
-							'password' => $pass,
-							'path' => $datadir.'/'.$name.'.db',
-							'driver' => 'pdo_sqlite',
-					);
-					$connectionParams['adapter'] = '\OC\DB\AdapterSqlite';
-					$connectionParams['wrapperClass'] = 'OC\DB\Connection';
-					break;
-				case 'mysql':
-					$connectionParams = array(
-							'user' => $user,
-							'password' => $pass,
-							'host' => $host,
-							'port' => $port,
-							'dbname' => $name,
-							'charset' => 'UTF8',
-							'driver' => 'pdo_mysql',
-					);
-					$connectionParams['adapter'] = '\OC\DB\Adapter';
-					$connectionParams['wrapperClass'] = 'OC\DB\Connection';
-					// Send "SET NAMES utf8". Only required on PHP 5.3 below 5.3.6.
-					// See http://stackoverflow.com/questions/4361459/php-pdo-charset-set-names#4361485
-					$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\MysqlSessionInit);
-					break;
-				case 'pgsql':
-					$connectionParams = array(
-							'user' => $user,
-							'password' => $pass,
-							'host' => $host,
-							'port' => $port,
-							'dbname' => $name,
-							'driver' => 'pdo_pgsql',
-					);
-					$connectionParams['adapter'] = '\OC\DB\AdapterPgSql';
-					$connectionParams['wrapperClass'] = 'OC\DB\Connection';
-					break;
-				case 'oci':
-					$connectionParams = array(
-							'user' => $user,
-							'password' => $pass,
-							'host' => $host,
-							'dbname' => $name,
-							'charset' => 'AL32UTF8',
-							'driver' => 'oci8',
-					);
-					if (!empty($port)) {
-						$connectionParams['port'] = $port;
-					}
-					$connectionParams['adapter'] = '\OC\DB\AdapterOCI8';
-					$connectionParams['wrapperClass'] = 'OC\DB\OracleConnection';
-					$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\OracleSessionInit);
-					break;
-				case 'mssql':
-					$connectionParams = array(
-							'user' => $user,
-							'password' => $pass,
-							'host' => $host,
-							'port' => $port,
-							'dbname' => $name,
-							'charset' => 'UTF8',
-							'driver' => 'pdo_sqlsrv',
-					);
-					$connectionParams['adapter'] = '\OC\DB\AdapterSQLSrv';
-					$connectionParams['wrapperClass'] = 'OC\DB\Connection';
-					break;
-				default:
-					return false;
-			}
-			$connectionParams['tablePrefix'] = OC_Config::getValue('dbtableprefix', 'oc_' );
-			try {
-				self::$connection = \Doctrine\DBAL\DriverManager::getConnection($connectionParams, $config, $eventManager);
-				if ($type === 'sqlite' || $type === 'sqlite3') {
-					// Sqlite doesn't handle query caching and schema changes
-					// TODO: find a better way to handle this
-					self::$connection->disableQueryStatementCaching();
-				}
-			} catch(\Doctrine\DBAL\DBALException $e) {
-				OC_Log::write('core', $e->getMessage(), OC_Log::FATAL);
-				OC_User::setUserId(null);
 
-				// send http status 503
+		if ($factory->normalizeType($type) === 'sqlite3') {
-				header('HTTP/1.1 503 Service Temporarily Unavailable');
+			$datadir = OC_Config::getValue("datadirectory", OC::$SERVERROOT.'/data');
-				header('Status: 503 Service Temporarily Unavailable');
+			$connectionParams = array(
-				OC_Template::printErrorPage('Failed to connect to database');
+				'user' => $user,
-				die();
+				'password' => $pass,
+				'path' => $datadir.'/'.$name.'.db',
+			);
+		} else {
+			$connectionParams = array(
+				'user' => $user,
+				'password' => $pass,
+				'host' => $host,
+				'dbname' => $name,
+			);
+			if (!empty($port)) {
+				$connectionParams['port'] = $port;
 			}
 		}
+
+		$connectionParams['tablePrefix'] = OC_Config::getValue('dbtableprefix', 'oc_');
+
+		try {
+			self::$connection = $factory->getConnection($type, $connectionParams);
+		} catch(\Doctrine\DBAL\DBALException $e) {
+			OC_Log::write('core', $e->getMessage(), OC_Log::FATAL);
+			OC_User::setUserId(null);
+
+			// send http status 503
+			header('HTTP/1.1 503 Service Temporarily Unavailable');
+			header('Status: 503 Service Temporarily Unavailable');
+			OC_Template::printErrorPage('Failed to connect to database');
+			die();
+		}
+
 		return true;
 	}
 
@@ -202,12 +145,12 @@ class OC_DB {
 	 */
 	static public function prepare( $query , $limit = null, $offset = null, $isManipulation = null) {
 		self::connect();
-		
+
 		if ($isManipulation === null) {
 			//try to guess, so we return the number of rows on manipulations
 			$isManipulation = self::isManipulation($query);
 		}
-		
+
 		// return the result
 		try {
 			$result = self::$connection->prepare($query, $limit, $offset);
@@ -222,7 +165,7 @@ class OC_DB {
 	/**
 	 * tries to guess the type of statement based on the first 10 characters
 	 * the current check allows some whitespace but does not work with IF EXISTS or other more complex statements
-	 * 
+	 *
 	 * @param string $sql
 	 * @return bool
 	 */
@@ -245,7 +188,7 @@ class OC_DB {
 		}
 		return false;
 	}
-	
+
 	/**
 	 * @brief execute a prepared statement, on error write log and throw exception
 	 * @param mixed $stmt OC_DB_StatementWrapper,

lib/private/db/connectionfactory.php

@@ -0,0 +1,118 @@
+<?php
+/**
+ * Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+/**
+* Takes care of creating and configuring Doctrine connections.
+*/
+class ConnectionFactory {
+	/**
+	* @var array
+	*
+	* Array mapping DBMS type to default connection parameters passed to
+	* \Doctrine\DBAL\DriverManager::getConnection().
+	*/
+	protected $defaultConnectionParams = array(
+		'mssql' => array(
+			'adapter' => '\OC\DB\AdapterSQLSrv',
+			'charset' => 'UTF8',
+			'driver' => 'pdo_sqlsrv',
+			'wrapperClass' => 'OC\DB\Connection',
+		),
+		'mysql' => array(
+			'adapter' => '\OC\DB\Adapter',
+			'charset' => 'UTF8',
+			'driver' => 'pdo_mysql',
+			'wrapperClass' => 'OC\DB\Connection',
+		),
+		'oci' => array(
+			'adapter' => '\OC\DB\AdapterOCI8',
+			'charset' => 'AL32UTF8',
+			'driver' => 'oci8',
+			'wrapperClass' => 'OC\DB\OracleConnection',
+		),
+		'pgsql' => array(
+			'adapter' => '\OC\DB\AdapterPgSql',
+			'driver' => 'pdo_pgsql',
+			'wrapperClass' => 'OC\DB\Connection',
+		),
+		'sqlite3' => array(
+			'adapter' => '\OC\DB\AdapterSqlite',
+			'driver' => 'pdo_sqlite',
+			'wrapperClass' => 'OC\DB\Connection',
+		),
+	);
+
+	/**
+	* @brief Get default connection parameters for a given DBMS.
+	* @param string $type DBMS type
+	* @throws \InvalidArgumentException If $type is invalid
+	* @return array Default connection parameters.
+	*/
+	public function getDefaultConnectionParams($type) {
+		$normalizedType = $this->normalizeType($type);
+		if (!isset($this->defaultConnectionParams[$normalizedType])) {
+			throw new \InvalidArgumentException("Unsupported type: $type");
+		}
+		return $this->defaultConnectionParams[$normalizedType];
+	}
+
+	/**
+	* @brief Get default connection parameters for a given DBMS.
+	* @param string $type DBMS type
+	* @param array $additionalConnectionParams Additional connection parameters
+	* @return \OC\DB\Connection
+	*/
+	public function getConnection($type, $additionalConnectionParams) {
+		$normalizedType = $this->normalizeType($type);
+		$eventManager = new \Doctrine\Common\EventManager();
+		switch ($normalizedType) {
+			case 'mysql':
+				// Send "SET NAMES utf8". Only required on PHP 5.3 below 5.3.6.
+				// See http://stackoverflow.com/questions/4361459/php-pdo-charset-set-names#4361485
+				$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\MysqlSessionInit);
+				break;
+			case 'oci':
+				$eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\OracleSessionInit);
+				break;
+		}
+		$connection = \Doctrine\DBAL\DriverManager::getConnection(
+			array_merge($this->getDefaultConnectionParams($type), $additionalConnectionParams),
+			new \Doctrine\DBAL\Configuration(),
+			$eventManager
+		);
+		switch ($normalizedType) {
+			case 'sqlite3':
+				// Sqlite doesn't handle query caching and schema changes
+				// TODO: find a better way to handle this
+				/** @var $connection \OC\DB\Connection */
+				$connection->disableQueryStatementCaching();
+				break;
+		}
+		return $connection;
+	}
+
+	/**
+	* @brief Normalize DBMS type
+	* @param string $type DBMS type
+	* @return string Normalized DBMS type
+	*/
+	public function normalizeType($type) {
+		return $type === 'sqlite' ? 'sqlite3' : $type;
+	}
+
+	/**
+	* @brief Checks whether the specified DBMS type is valid.
+	* @return bool
+	*/
+	public function isValidType($type) {
+		$normalizedType = $this->normalizeType($type);
+		return isset($this->defaultConnectionParams[$normalizedType]);
+	}
+}

lib/private/db/pgsqltools.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Copyright (c) 2013 Bart Visscher <bartv@thisnet.nl>
+ * Copyright (c) 2014 Andreas Fischer <bantu@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+/**
+* Various PostgreSQL specific helper functions.
+*/
+class PgSqlTools {
+	/**
+	* @brief Resynchronizes all sequences of a database after using INSERTs
+	*        without leaving out the auto-incremented column.
+	* @param \OC\DB\Connection $conn
+	* @return null
+	*/
+	public function resynchronizeDatabaseSequences(Connection $conn) {
+		$databaseName = $conn->getDatabase();
+		foreach ($conn->getSchemaManager()->listSequences() as $sequence) {
+			$sequenceName = $sequence->getName();
+			$sqlInfo = 'SELECT table_schema, table_name, column_name
+				FROM information_schema.columns
+				WHERE column_default = ? AND table_catalog = ?';
+			$sequenceInfo = $conn->fetchAssoc($sqlInfo, array(
+				"nextval('$sequenceName'::regclass)",
+				$databaseName
+			));
+			$tableName = $sequenceInfo['table_name'];
+			$columnName = $sequenceInfo['column_name'];
+			$sqlMaxId = "SELECT MAX($columnName) FROM $tableName";
+			$sqlSetval = "SELECT setval('$sequenceName', ($sqlMaxId))";
+			$conn->executeQuery($sqlSetval);
+		}
+	}
+}