mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #3928 from nextcloud/downstream-17978 

Adjust 4 byte MySQL code to upstream
This commit is contained in:
Morris Jobke 2017-03-23 23:45:43 -06:00 committed by GitHub
parent d197f609a8 8a4466f9ae
commit 1397b84777
13 changed files with 256 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
core/Command/Db/ConvertMysqlToMB4.php Normal file
View File

@ -0,0 +1,92 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Core\Command\Db;
use Doctrine\DBAL\Platforms\MySqlPlatform;
use OC\Migration\ConsoleOutput;
use OC\Repair\Collation;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\ILogger;
use OCP\IURLGenerator;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class ConvertMysqlToMB4 extends Command {
/** @var IConfig */
private $config;
/** @var IDBConnection */
private $connection;
/** @var IURLGenerator */
private $urlGenerator;
/** @var ILogger */
private $logger;
/**
* @param IConfig $config
* @param IDBConnection $connection
* @param IURLGenerator $urlGenerator
* @param ILogger $logger
*/
public function __construct(IConfig $config, IDBConnection $connection, IURLGenerator $urlGenerator, ILogger $logger) {
$this->config = $config;
$this->connection = $connection;
$this->urlGenerator = $urlGenerator;
$this->logger = $logger;
parent::__construct();
}
protected function configure() {
$this
->setName('db:convert-mysql-charset')
->setDescription('Convert charset of MySQL/MariaDB to use utf8mb4');
}
protected function execute(InputInterface $input, OutputInterface $output) {
if (!$this->connection->getDatabasePlatform() instanceof MySqlPlatform) {
$output->writeln("This command is only valid for MySQL/MariaDB databases.");
return 1;
}
$oldValue = $this->config->getSystemValue('mysql.utf8mb4', false);
// enable charset
$this->config->setSystemValue('mysql.utf8mb4', true);
if (!$this->connection->supports4ByteText()) {
$url = $this->urlGenerator->linkToDocs('admin-mysql-utf8mb4');
$output->writeln("The database is not properly setup to use the charset utf8mb4.");
$output->writeln("For more information please read the documentation at $url");
$this->config->setSystemValue('mysql.utf8mb4', $oldValue);
return 1;
}
// run conversion
$coll = new Collation($this->config, $this->logger, $this->connection, false);
$coll->run(new ConsoleOutput($output));
return 0;
}
}

1
core/register_command.php
View File

@ -84,6 +84,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
$application->add(new OC\Core\Command\Db\GenerateChangeScript());
$application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig())));
$application->add(new OC\Core\Command\Db\ConvertMysqlToMB4(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection(), \OC::$server->getURLGenerator(), \OC::$server->getLogger()));
$application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig()));
$application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager()));

2
lib/composer/composer/autoload_classmap.php
View File

@ -389,6 +389,7 @@ return array(
'OC\\Core\\Command\\Config\\System\\DeleteConfig' => $baseDir . '/core/Command/Config/System/DeleteConfig.php',
'OC\\Core\\Command\\Config\\System\\GetConfig' => $baseDir . '/core/Command/Config/System/GetConfig.php',
'OC\\Core\\Command\\Config\\System\\SetConfig' => $baseDir . '/core/Command/Config/System/SetConfig.php',
'OC\\Core\\Command\\Db\\ConvertMysqlToMB4' => $baseDir . '/core/Command/Db/ConvertMysqlToMB4.php',
'OC\\Core\\Command\\Db\\ConvertType' => $baseDir . '/core/Command/Db/ConvertType.php',
'OC\\Core\\Command\\Db\\GenerateChangeScript' => $baseDir . '/core/Command/Db/GenerateChangeScript.php',
'OC\\Core\\Command\\Encryption\\ChangeKeyStorageRoot' => $baseDir . '/core/Command/Encryption/ChangeKeyStorageRoot.php',
@ -642,6 +643,7 @@ return array(
'OC\\Memcache\\Redis' => $baseDir . '/lib/private/Memcache/Redis.php',
'OC\\Memcache\\XCache' => $baseDir . '/lib/private/Memcache/XCache.php',
'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php',
'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php',
'OC\\NaturalSort' => $baseDir . '/lib/private/NaturalSort.php',
'OC\\NaturalSort_DefaultCollator' => $baseDir . '/lib/private/NaturalSort_DefaultCollator.php',
'OC\\NavigationManager' => $baseDir . '/lib/private/NavigationManager.php',

2
lib/composer/composer/autoload_static.php
View File

@ -419,6 +419,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Core\\Command\\Config\\System\\DeleteConfig' => __DIR__ . '/../../..' . '/core/Command/Config/System/DeleteConfig.php',
'OC\\Core\\Command\\Config\\System\\GetConfig' => __DIR__ . '/../../..' . '/core/Command/Config/System/GetConfig.php',
'OC\\Core\\Command\\Config\\System\\SetConfig' => __DIR__ . '/../../..' . '/core/Command/Config/System/SetConfig.php',
'OC\\Core\\Command\\Db\\ConvertMysqlToMB4' => __DIR__ . '/../../..' . '/core/Command/Db/ConvertMysqlToMB4.php',
'OC\\Core\\Command\\Db\\ConvertType' => __DIR__ . '/../../..' . '/core/Command/Db/ConvertType.php',
'OC\\Core\\Command\\Db\\GenerateChangeScript' => __DIR__ . '/../../..' . '/core/Command/Db/GenerateChangeScript.php',
'OC\\Core\\Command\\Encryption\\ChangeKeyStorageRoot' => __DIR__ . '/../../..' . '/core/Command/Encryption/ChangeKeyStorageRoot.php',
@ -672,6 +673,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Memcache\\Redis' => __DIR__ . '/../../..' . '/lib/private/Memcache/Redis.php',
'OC\\Memcache\\XCache' => __DIR__ . '/../../..' . '/lib/private/Memcache/XCache.php',
'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php',
'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php',
'OC\\NaturalSort' => __DIR__ . '/../../..' . '/lib/private/NaturalSort.php',
'OC\\NaturalSort_DefaultCollator' => __DIR__ . '/../../..' . '/lib/private/NaturalSort_DefaultCollator.php',
'OC\\NavigationManager' => __DIR__ . '/../../..' . '/lib/private/NavigationManager.php',

5
lib/private/DB/Connection.php
View File

@ -416,6 +416,9 @@ class Connection extends \Doctrine\DBAL\Connection implements IDBConnection {
* @since 11.0.0
*/
public function supports4ByteText() {
return ! ($this->getDatabasePlatform() instanceof MySqlPlatform && $this->getParams()['charset'] !== 'utf8mb4');
if (!$this->getDatabasePlatform() instanceof MySqlPlatform) {
return true;
}
return $this->getParams()['charset'] === 'utf8mb4';
}
}

15
lib/private/DB/ConnectionFactory.php
View File

@ -201,6 +201,21 @@ class ConnectionFactory {
$connectionParams['driverOptions'] = $driverOptions;
}
// set default table creation options
$connectionParams['defaultTableOptions'] = [
'collate' => 'utf8_bin',
'tablePrefix' => $connectionParams['tablePrefix']
];
if($this->config->getValue('mysql.utf8mb4', false)) {
$connectionParams['defaultTableOptions'] = [
'collate' => 'utf8mb4_bin',
'charset' => 'utf8mb4',
'row_format' => 'compressed',
'tablePrefix' => $connectionParams['tablePrefix']
];
}
return $connectionParams;
}
}

10
lib/private/DB/MDB2SchemaManager.php
View File

@ -33,6 +33,7 @@ use Doctrine\DBAL\Platforms\MySqlPlatform;
use Doctrine\DBAL\Platforms\OraclePlatform;
use Doctrine\DBAL\Platforms\PostgreSqlPlatform;
use Doctrine\DBAL\Platforms\SqlitePlatform;
use Doctrine\DBAL\Schema\Schema;
use OCP\IDBConnection;
class MDB2SchemaManager {
@ -66,7 +67,8 @@ class MDB2SchemaManager {
*/
public function createDbFromStructure($file) {
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform());
$toSchema = $schemaReader->loadSchemaFromFile($file);
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
$toSchema = $schemaReader->loadSchemaFromFile($file, $toSchema);
return $this->executeSchemaChange($toSchema);
}
@ -100,7 +102,8 @@ class MDB2SchemaManager {
private function readSchemaFromFile($file) {
$platform = $this->conn->getDatabasePlatform();
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $platform);
return $schemaReader->loadSchemaFromFile($file);
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
return $schemaReader->loadSchemaFromFile($file, $toSchema);
}
/**
@ -137,7 +140,8 @@ class MDB2SchemaManager {
*/
public function removeDBStructure($file) {
$schemaReader = new MDB2SchemaReader(\OC::$server->getConfig(), $this->conn->getDatabasePlatform());
$fromSchema = $schemaReader->loadSchemaFromFile($file);
$toSchema = new Schema([], [], $this->conn->getSchemaManager()->createSchemaConfig());
$fromSchema = $schemaReader->loadSchemaFromFile($file, $toSchema);
$toSchema = clone $fromSchema;
/** @var $table \Doctrine\DBAL\Schema\Table */
foreach ($toSchema->getTables() as $table) {

27
lib/private/DB/MDB2SchemaReader.php
View File

@ -38,10 +38,6 @@ use Doctrine\DBAL\Schema\Schema;
use OCP\IConfig;
class MDB2SchemaReader {
/**
* @var string $DBNAME
*/
protected $DBNAME;
/**
* @var string $DBTABLEPREFIX
@ -53,9 +49,6 @@ class MDB2SchemaReader {
*/
protected $platform;
/** @var \Doctrine\DBAL\Schema\SchemaConfig $schemaConfig */
protected $schemaConfig;
/** @var IConfig */
protected $config;
@ -66,23 +59,16 @@ class MDB2SchemaReader {
public function __construct(IConfig $config, AbstractPlatform $platform) {
$this->platform = $platform;
$this->config = $config;
$this->DBNAME = $config->getSystemValue('dbname', 'owncloud');
$this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_');
// Oracle does not support longer index names then 30 characters.
// We use this limit for all DBs to make sure it does not cause a
// problem.
$this->schemaConfig = new SchemaConfig();
$this->schemaConfig->setMaxIdentifierLength(30);
}
/**
* @param string $file
* @param Schema $schema
* @return Schema
* @throws \DomainException
*/
public function loadSchemaFromFile($file) {
$schema = new \Doctrine\DBAL\Schema\Schema();
public function loadSchemaFromFile($file, Schema $schema) {
$loadEntities = libxml_disable_entity_loader(false);
$xml = simplexml_load_file($file);
libxml_disable_entity_loader($loadEntities);
@ -124,15 +110,6 @@ class MDB2SchemaReader {
$name = str_replace('*dbprefix*', $this->DBTABLEPREFIX, $name);
$name = $this->platform->quoteIdentifier($name);
$table = $schema->createTable($name);
$table->setSchemaConfig($this->schemaConfig);
if($this->platform instanceof MySqlPlatform && $this->config->getSystemValue('mysql.utf8mb4', false)) {
$table->addOption('charset', 'utf8mb4');
$table->addOption('collate', 'utf8mb4_bin');
$table->addOption('row_format', 'compressed');
} else {
$table->addOption('collate', 'utf8_bin');
}
break;
case 'create':
case 'overwrite':

93
lib/private/Migration/ConsoleOutput.php Normal file
View File

@ -0,0 +1,93 @@
<?php
/**
* @author Thomas Müller <thomas.mueller@tmit.eu>
*
* @copyright Copyright (c) 2017, ownCloud GmbH
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Migration;
use OCP\Migration\IOutput;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Class SimpleOutput
*
* Just a simple IOutput implementation with writes messages to the log file.
* Alternative implementations will write to the console or to the web ui (web update case)
*
* @package OC\Migration
*/
class ConsoleOutput implements IOutput {
/** @var OutputInterface */
private $output;
/** @var ProgressBar */
private $progressBar;
public function __construct(OutputInterface $output) {
$this->output = $output;
}
/**
* @param string $message
*/
public function info($message) {
$this->output->writeln("<info>$message</info>");
}
/**
* @param string $message
*/
public function warning($message) {
$this->output->writeln("<comment>$message</comment>");
}
/**
* @param int $max
*/
public function startProgress($max = 0) {
if (!is_null($this->progressBar)) {
$this->progressBar->finish();
}
$this->progressBar = new ProgressBar($this->output);
$this->progressBar->start($max);
}
/**
* @param int $step
* @param string $description
*/
public function advance($step = 1, $description = '') {
if (!is_null($this->progressBar)) {
$this->progressBar = new ProgressBar($this->output);
$this->progressBar->start();
}
$this->progressBar->advance($step);
}
public function finishProgress() {
if (is_null($this->progressBar)) {
return;
}
$this->progressBar->finish();
}
}

38
lib/private/Repair/Collation.php
View File

@ -88,6 +88,11 @@ class Collation implements IRepairStep {
}
$output->info("Change collation for $table ...");
if ($characterSet === 'utf8mb4') {
// need to set row compression first
$query = $this->connection->prepare('ALTER TABLE `' . $table . '` ROW_FORMAT=COMPRESSED;');
$query->execute();
}
$query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;');
try {
$query->execute();
@ -99,16 +104,21 @@ class Collation implements IRepairStep {
}
}
}
if (empty($tables)) {
$output->info('All tables already have the correct collation -> nothing to do');
}
}
/**
* @param \Doctrine\DBAL\Connection $connection
* @param IDBConnection $connection
* @return string[]
*/
protected function getAllNonUTF8BinTables($connection) {
protected function getAllNonUTF8BinTables(IDBConnection $connection) {
$dbName = $this->config->getSystemValue("dbname");
$characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
$rows = $connection->fetchAll(
// fetch tables by columns
$statement = $connection->executeQuery(
"SELECT DISTINCT(TABLE_NAME) AS `table`" .
" FROM INFORMATION_SCHEMA . COLUMNS" .
" WHERE TABLE_SCHEMA = ?" .
@ -116,11 +126,27 @@ class Collation implements IRepairStep {
" AND TABLE_NAME LIKE \"*PREFIX*%\"",
array($dbName)
);
$result = array();
$rows = $statement->fetchAll();
$result = [];
foreach ($rows as $row) {
$result[] = $row['table'];
$result[$row['table']] = true;
}
return $result;
// fetch tables by collation
$statement = $connection->executeQuery(
"SELECT DISTINCT(TABLE_NAME) AS `table`" .
" FROM INFORMATION_SCHEMA . TABLES" .
" WHERE TABLE_SCHEMA = ?" .
" AND TABLE_COLLATION <> '" . $characterSet . "_bin'" .
" AND TABLE_NAME LIKE \"*PREFIX*%\"",
[$dbName]
);
$rows = $statement->fetchAll();
foreach ($rows as $row) {
$result[$row['table']] = true;
}
return array_keys($result);
}
}

2
tests/lib/DB/MDB2SchemaReaderTest.php
View File

@ -47,7 +47,7 @@ class MDB2SchemaReaderTest extends TestCase {
public function testRead() {
$reader = new MDB2SchemaReader($this->getConfig(), new MySqlPlatform());
$schema = $reader->loadSchemaFromFile(__DIR__ . '/testschema.xml');
$schema = $reader->loadSchemaFromFile(__DIR__ . '/testschema.xml', new Schema());
$this->assertCount(1, $schema->getTables());
$table = $schema->getTable('test_table');

4
tests/lib/DB/SchemaDiffTest.php
View File

@ -21,6 +21,7 @@
namespace Test\DB;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Schema\SchemaDiff;
use OC\DB\MDB2SchemaManager;
use OC\DB\MDB2SchemaReader;
@ -79,7 +80,8 @@ class SchemaDiffTest extends TestCase {
$this->manager->createDbFromStructure($schemaFile);
$schemaReader = new MDB2SchemaReader($this->config, $this->connection->getDatabasePlatform());
$endSchema = $schemaReader->loadSchemaFromFile($schemaFile);
$toSchema = new Schema([], [], $this->connection->getSchemaManager()->createSchemaConfig());
$endSchema = $schemaReader->loadSchemaFromFile($schemaFile, $toSchema);
// get the diff
/** @var SchemaDiff $diff */

4
tests/lib/Repair/RepairCollationTest.php
View File

@ -18,10 +18,10 @@ use Test\TestCase;
class TestCollationRepair extends Collation {
/**
* @param \Doctrine\DBAL\Connection $connection
* @param IDBConnection $connection
* @return string[]
*/
public function getAllNonUTF8BinTables($connection) {
public function getAllNonUTF8BinTables(IDBConnection $connection) {
return parent::getAllNonUTF8BinTables($connection);
}
}