From cc28f82b369c2e8ebf2d0b4390379b9cda4af40b Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 30 Jul 2015 13:57:04 +0200 Subject: [PATCH] Add config option to update charset of mysql to utf8mb4 * fully optional * requires additional options set in the database --- config/config.sample.php | 29 ++++++++++++++++++++++++++++ core/register_command.php | 2 +- lib/private/DB/AdapterMySQL.php | 3 ++- lib/private/DB/ConnectionFactory.php | 7 +++++++ lib/private/DB/MDB2SchemaReader.php | 14 +++++++++++++- lib/private/DB/MDB2SchemaWriter.php | 6 +++++- lib/private/Repair/Collation.php | 7 +++++-- lib/private/Server.php | 2 +- lib/private/Setup/MySQL.php | 5 +++-- 9 files changed, 66 insertions(+), 9 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index 3aa0f353c5..df1e2d16fc 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -129,6 +129,7 @@ $CONFIG = array( */ 'dbtableprefix' => '', + /** * Indicates whether the Nextcloud instance was installed successfully; ``true`` * indicates a successful installation, and ``false`` indicates an unsuccessful @@ -1079,6 +1080,34 @@ $CONFIG = array( */ 'sqlite.journal_mode' => 'DELETE', +/** + * If this setting is set to true MySQL can handle 4 byte characters instead of + * 3 byte characters + * + * MySQL requires a special setup for longer indexes (> 767 bytes) which are + * needed: + * + * [mysqld] + * innodb_large_prefix=true + * innodb_file_format=barracuda + * innodb_file_per_table=true + * + * Tables will be created with + * * character set: utf8mb4 + * * collation: utf8mb4_bin + * * row_format: compressed + * + * See: + * https://dev.mysql.com/doc/refman/5.7/en/charset-unicode-utf8mb4.html + * https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_large_prefix + * https://mariadb.com/kb/en/mariadb/xtradbinnodb-server-system-variables/#innodb_large_prefix + * http://www.tocker.ca/2013/10/31/benchmarking-innodb-page-compression-performance.html + * http://mechanics.flite.com/blog/2014/07/29/using-innodb-large-prefix-to-avoid-error-1071/ + * + * WARNING: EXPERIMENTAL + */ +'mysql.utf8mb4' => false, + /** * Database types that are supported for installation. * diff --git a/core/register_command.php b/core/register_command.php index a6da3cbd89..fa85aea895 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -83,7 +83,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) { $application->add(new OC\Core\Command\Config\System\SetConfig(\OC::$server->getSystemConfig())); $application->add(new OC\Core\Command\Db\GenerateChangeScript()); - $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory())); + $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig()))); $application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager())); diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php index 3e2fceda8d..0c0c6b3102 100644 --- a/lib/private/DB/AdapterMySQL.php +++ b/lib/private/DB/AdapterMySQL.php @@ -39,7 +39,8 @@ class AdapterMySQL extends Adapter { } public function fixupStatement($statement) { - $statement = str_replace(' ILIKE ', ' COLLATE utf8_general_ci LIKE ', $statement); + $characterSet = \OC::$server->getConfig()->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; + $statement = str_replace(' ILIKE ', ' COLLATE ' . $characterSet . '_general_ci LIKE ', $statement); return $statement; } } diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php index b2c356edef..a7aae32f67 100644 --- a/lib/private/DB/ConnectionFactory.php +++ b/lib/private/DB/ConnectionFactory.php @@ -28,6 +28,7 @@ namespace OC\DB; use Doctrine\DBAL\Event\Listeners\OracleSessionInit; use Doctrine\DBAL\Event\Listeners\SQLSessionInit; use Doctrine\DBAL\Event\Listeners\MysqlSessionInit; +use OC\SystemConfig; /** * Takes care of creating and configuring Doctrine connections. @@ -64,6 +65,12 @@ class ConnectionFactory { ), ); + public function __construct(SystemConfig $systemConfig) { + if($systemConfig->getValue('mysql.utf8mb4', false)) { + $defaultConnectionParams['mysql']['charset'] = 'utf8mb4'; + } + } + /** * @brief Get default connection parameters for a given DBMS. * @param string $type DBMS type diff --git a/lib/private/DB/MDB2SchemaReader.php b/lib/private/DB/MDB2SchemaReader.php index 3f183c9723..c198bb31e0 100644 --- a/lib/private/DB/MDB2SchemaReader.php +++ b/lib/private/DB/MDB2SchemaReader.php @@ -33,6 +33,7 @@ namespace OC\DB; use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\SchemaConfig; +use Doctrine\DBAL\Platforms\MySqlPlatform; use OCP\IConfig; class MDB2SchemaReader { @@ -54,12 +55,16 @@ class MDB2SchemaReader { /** @var \Doctrine\DBAL\Schema\SchemaConfig $schemaConfig */ protected $schemaConfig; + /** @var IConfig */ + protected $config; + /** * @param \OCP\IConfig $config * @param \Doctrine\DBAL\Platforms\AbstractPlatform $platform */ public function __construct(IConfig $config, AbstractPlatform $platform) { $this->platform = $platform; + $this->config = $config; $this->DBNAME = $config->getSystemValue('dbname', 'owncloud'); $this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_'); @@ -118,8 +123,15 @@ class MDB2SchemaReader { $name = str_replace('*dbprefix*', $this->DBTABLEPREFIX, $name); $name = $this->platform->quoteIdentifier($name); $table = $schema->createTable($name); - $table->addOption('collate', 'utf8_bin'); $table->setSchemaConfig($this->schemaConfig); + + if($this->platform instanceof MySqlPlatform && $this->config->getSystemValue('mysql.utf8mb4', false)) { + $table->addOption('charset', 'utf8mb4'); + $table->addOption('collate', 'utf8mb4_bin'); + $table->addOption('row_format', 'compressed'); + } else { + $table->addOption('collate', 'utf8_bin'); + } break; case 'create': case 'overwrite': diff --git a/lib/private/DB/MDB2SchemaWriter.php b/lib/private/DB/MDB2SchemaWriter.php index 26e32b036f..7664b4359a 100644 --- a/lib/private/DB/MDB2SchemaWriter.php +++ b/lib/private/DB/MDB2SchemaWriter.php @@ -43,7 +43,11 @@ class MDB2SchemaWriter { $xml->addChild('name', $config->getSystemValue('dbname', 'owncloud')); $xml->addChild('create', 'true'); $xml->addChild('overwrite', 'false'); - $xml->addChild('charset', 'utf8'); + if($config->getSystemValue('dbtype', 'sqlite') === 'mysql' && $config->getSystemValue('mysql.utf8mb4', false)) { + $xml->addChild('charset', 'utf8mb4'); + } else { + $xml->addChild('charset', 'utf8'); + } // FIX ME: bloody work around if ($config->getSystemValue('dbtype', 'sqlite') === 'oci') { diff --git a/lib/private/Repair/Collation.php b/lib/private/Repair/Collation.php index 74c4ca2e6a..c19b8eea5e 100644 --- a/lib/private/Repair/Collation.php +++ b/lib/private/Repair/Collation.php @@ -61,10 +61,12 @@ class Collation implements IRepairStep { return; } + $characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; + $tables = $this->getAllNonUTF8BinTables($this->connection); foreach ($tables as $table) { $output->info("Change collation for $table ..."); - $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin;'); + $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;'); $query->execute(); } } @@ -75,11 +77,12 @@ class Collation implements IRepairStep { */ protected function getAllNonUTF8BinTables($connection) { $dbName = $this->config->getSystemValue("dbname"); + $characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; $rows = $connection->fetchAll( "SELECT DISTINCT(TABLE_NAME) AS `table`" . " FROM INFORMATION_SCHEMA . COLUMNS" . " WHERE TABLE_SCHEMA = ?" . - " AND (COLLATION_NAME <> 'utf8_bin' OR CHARACTER_SET_NAME <> 'utf8')" . + " AND (COLLATION_NAME <> '" . $characterSet . "_bin' OR CHARACTER_SET_NAME <> '" . $characterSet . "')" . " AND TABLE_NAME LIKE \"*PREFIX*%\"", array($dbName) ); diff --git a/lib/private/Server.php b/lib/private/Server.php index 291714b23d..063ff4a3d3 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -407,8 +407,8 @@ class Server extends ServerContainer implements IServerContainer { return new CredentialsManager($c->getCrypto(), $c->getDatabaseConnection()); }); $this->registerService('DatabaseConnection', function (Server $c) { - $factory = new \OC\DB\ConnectionFactory(); $systemConfig = $c->getSystemConfig(); + $factory = new \OC\DB\ConnectionFactory($systemConfig); $type = $systemConfig->getValue('dbtype', 'sqlite'); if (!$factory->isValidType($type)) { throw new \OC\DatabaseException('Invalid database type'); diff --git a/lib/private/Setup/MySQL.php b/lib/private/Setup/MySQL.php index 4ad6926c2d..c022616d8b 100644 --- a/lib/private/Setup/MySQL.php +++ b/lib/private/Setup/MySQL.php @@ -58,8 +58,9 @@ class MySQL extends AbstractDatabase { try{ $name = $this->dbName; $user = $this->dbUser; - //we can't use OC_BD functions here because we need to connect as the administrative user. - $query = "CREATE DATABASE IF NOT EXISTS `$name` CHARACTER SET utf8 COLLATE utf8_bin;"; + //we can't use OC_DB functions here because we need to connect as the administrative user. + $characterSet = \OC::$server->getSystemConfig()->getValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; + $query = "CREATE DATABASE IF NOT EXISTS `$name` CHARACTER SET $characterSet COLLATE ${characterSet}_bin;"; $connection->executeUpdate($query); } catch (\Exception $ex) { $this->logger->error('Database creation failed: {error}', [