From aa22f930181cf66a0610a11ebbe4de7377623444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Fri, 3 Feb 2017 15:38:48 +0100 Subject: [PATCH 1/7] During setup of a mysql database we try to detect if charset 'utf8mb4' can be used --- lib/private/Setup/MySQL.php | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/private/Setup/MySQL.php b/lib/private/Setup/MySQL.php index 9998ca401d..d72a106053 100644 --- a/lib/private/Setup/MySQL.php +++ b/lib/private/Setup/MySQL.php @@ -27,6 +27,7 @@ */ namespace OC\Setup; +use OC\DB\ConnectionFactory; use OCP\IDBConnection; class MySQL extends AbstractDatabase { @@ -36,6 +37,12 @@ class MySQL extends AbstractDatabase { //check if the database user has admin right $connection = $this->connect(['dbname' => null]); + // detect mb4 + if (is_null($this->config->getSValue('mysql.utf8mb4', null)) && $this->supports4ByteCharset($connection)) { + $this->config->setValue('mysql.utf8mb4', true); + $connection = $this->connect(); + } + $this->createSpecificUser($username, $connection); //create the database @@ -162,4 +169,23 @@ class MySQL extends AbstractDatabase { 'dbpassword' => $this->dbPassword, ]); } + + /** + * @param IDBConnection $connection + * @return bool + */ + private function supports4ByteCharset(IDBConnection $connection) { + foreach (['innodb_file_format' => 'Barracuda', 'innodb_large_prefix' => 'ON', 'innodb_file_per_table' => 'ON'] as $var => $val) { + $result = $connection->executeQuery("SHOW VARIABLES LIKE '$var'"); + $rows = $result->fetch(); + $result->closeCursor(); + if ($rows === false) { + return false; + } + if (strcasecmp($rows['Value'], $val) === 0) { + return false; + } + } + return true; + } } From 6d93cb74888ae0da0d2286ca1cd82179ea367513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Fri, 3 Feb 2017 16:24:22 +0100 Subject: [PATCH 2/7] Fix mysql settings --- config/config.sample.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index 84b98550fb..363bdabef0 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -1241,9 +1241,9 @@ $CONFIG = array( * needed: * * [mysqld] - * innodb_large_prefix=true - * innodb_file_format=barracuda - * innodb_file_per_table=true + * innodb_large_prefix=ON + * innodb_file_format=Barracuda + * innodb_file_per_table=ON * * Tables will be created with * * character set: utf8mb4 @@ -1256,8 +1256,6 @@ $CONFIG = array( * https://mariadb.com/kb/en/mariadb/xtradbinnodb-server-system-variables/#innodb_large_prefix * http://www.tocker.ca/2013/10/31/benchmarking-innodb-page-compression-performance.html * http://mechanics.flite.com/blog/2014/07/29/using-innodb-large-prefix-to-avoid-error-1071/ - * - * WARNING: EXPERIMENTAL */ 'mysql.utf8mb4' => false, From 43427e26d78123e66652741d960c14cf40342715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Fri, 3 Feb 2017 16:24:53 +0100 Subject: [PATCH 3/7] Add console command to migrate the charset --- core/Command/Db/ConvertMysqlToMB4.php | 12 +++---- lib/private/DB/MySqlTools.php | 49 +++++++++++++++++++++++++++ lib/private/Setup/MySQL.php | 30 ++++------------ 3 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 lib/private/DB/MySqlTools.php diff --git a/core/Command/Db/ConvertMysqlToMB4.php b/core/Command/Db/ConvertMysqlToMB4.php index 286274753e..38aff8b09d 100644 --- a/core/Command/Db/ConvertMysqlToMB4.php +++ b/core/Command/Db/ConvertMysqlToMB4.php @@ -22,6 +22,7 @@ namespace OC\Core\Command\Db; use Doctrine\DBAL\Platforms\MySqlPlatform; +use OC\DB\MySqlTools; use OC\Migration\ConsoleOutput; use OC\Repair\Collation; use OCP\IConfig; @@ -71,18 +72,17 @@ class ConvertMysqlToMB4 extends Command { return 1; } - $oldValue = $this->config->getSystemValue('mysql.utf8mb4', false); - // enable charset - $this->config->setSystemValue('mysql.utf8mb4', true); - - if (!$this->connection->supports4ByteText()) { + $tools = new MySqlTools(); + if (!$tools->supports4ByteCharset($this->connection)) { $url = $this->urlGenerator->linkToDocs('admin-mysql-utf8mb4'); $output->writeln("The database is not properly setup to use the charset utf8mb4."); $output->writeln("For more information please read the documentation at $url"); - $this->config->setSystemValue('mysql.utf8mb4', $oldValue); return 1; } + // enable charset + $this->config->setSystemValue('mysql.utf8mb4', true); + // run conversion $coll = new Collation($this->config, $this->logger, $this->connection, false); $coll->run(new ConsoleOutput($output)); diff --git a/lib/private/DB/MySqlTools.php b/lib/private/DB/MySqlTools.php new file mode 100644 index 0000000000..32f1de887c --- /dev/null +++ b/lib/private/DB/MySqlTools.php @@ -0,0 +1,49 @@ + + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\DB; + +use OCP\IDBConnection; + +/** +* Various MySQL specific helper functions. +*/ +class MySqlTools { + + /** + * @param Connection $connection + * @return bool + */ + public function supports4ByteCharset(IDBConnection $connection) { + foreach (['innodb_file_format' => 'Barracuda', 'innodb_large_prefix' => 'ON', 'innodb_file_per_table' => 'ON'] as $var => $val) { + $result = $connection->executeQuery("SHOW VARIABLES LIKE '$var'"); + $rows = $result->fetch(); + $result->closeCursor(); + if ($rows === false) { + return false; + } + if (strcasecmp($rows['Value'], $val) !== 0) { + return false; + } + } + return true; + } +} diff --git a/lib/private/Setup/MySQL.php b/lib/private/Setup/MySQL.php index d72a106053..3f3ad6d958 100644 --- a/lib/private/Setup/MySQL.php +++ b/lib/private/Setup/MySQL.php @@ -27,7 +27,7 @@ */ namespace OC\Setup; -use OC\DB\ConnectionFactory; +use OC\DB\MySqlTools; use OCP\IDBConnection; class MySQL extends AbstractDatabase { @@ -38,9 +38,12 @@ class MySQL extends AbstractDatabase { $connection = $this->connect(['dbname' => null]); // detect mb4 - if (is_null($this->config->getSValue('mysql.utf8mb4', null)) && $this->supports4ByteCharset($connection)) { - $this->config->setValue('mysql.utf8mb4', true); - $connection = $this->connect(); + if (is_null($this->config->getValue('mysql.utf8mb4', null))) { + $tools = new MySqlTools(); + if ($tools->supports4ByteCharset($connection)) { + $this->config->setValue('mysql.utf8mb4', true); + $connection = $this->connect(); + } } $this->createSpecificUser($username, $connection); @@ -169,23 +172,4 @@ class MySQL extends AbstractDatabase { 'dbpassword' => $this->dbPassword, ]); } - - /** - * @param IDBConnection $connection - * @return bool - */ - private function supports4ByteCharset(IDBConnection $connection) { - foreach (['innodb_file_format' => 'Barracuda', 'innodb_large_prefix' => 'ON', 'innodb_file_per_table' => 'ON'] as $var => $val) { - $result = $connection->executeQuery("SHOW VARIABLES LIKE '$var'"); - $rows = $result->fetch(); - $result->closeCursor(); - if ($rows === false) { - return false; - } - if (strcasecmp($rows['Value'], $val) === 0) { - return false; - } - } - return true; - } } From 86ccc8949f27f69f7ad895d345561567512a223a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Tue, 21 Mar 2017 13:08:14 +0100 Subject: [PATCH 4/7] MySQL 4 byte is now detected during setup in any case. config.sample.php was updated to explicitly state that there detection in place to set mysql.utf8mb4 --- config/config.sample.php | 8 ++++++-- lib/private/Setup/MySQL.php | 10 ++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index 363bdabef0..69567f0bce 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -1234,8 +1234,12 @@ $CONFIG = array( 'sqlite.journal_mode' => 'DELETE', /** - * If this setting is set to true MySQL can handle 4 byte characters instead of - * 3 byte characters + * If requirements are met (see below) this setting is set to true during setup + * and MySQL can handle 4 byte characters instead of 3 byte characters. + * + * If you want to convert a 3-byte setup into a 4-byte setup please run the + * migration command: + * ./occ db:convert-mysql-charset * * MySQL requires a special setup for longer indexes (> 767 bytes) which are * needed: diff --git a/lib/private/Setup/MySQL.php b/lib/private/Setup/MySQL.php index 3f3ad6d958..8290d67550 100644 --- a/lib/private/Setup/MySQL.php +++ b/lib/private/Setup/MySQL.php @@ -38,12 +38,10 @@ class MySQL extends AbstractDatabase { $connection = $this->connect(['dbname' => null]); // detect mb4 - if (is_null($this->config->getValue('mysql.utf8mb4', null))) { - $tools = new MySqlTools(); - if ($tools->supports4ByteCharset($connection)) { - $this->config->setValue('mysql.utf8mb4', true); - $connection = $this->connect(); - } + $tools = new MySqlTools(); + if ($tools->supports4ByteCharset($connection)) { + $this->config->setValue('mysql.utf8mb4', true); + $connection = $this->connect(); } $this->createSpecificUser($username, $connection); From 99d60f43dcd1decb9a3cbd17270842617af6fbcd Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 23 Mar 2017 11:12:58 +0100 Subject: [PATCH 5/7] config.sample.php doc update for MySQL 4-Byte unicode --- config/config.sample.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index 69567f0bce..c0e897a845 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -1234,13 +1234,16 @@ $CONFIG = array( 'sqlite.journal_mode' => 'DELETE', /** - * If requirements are met (see below) this setting is set to true during setup + * During setup, if requirements are met (see below), this setting is set to true * and MySQL can handle 4 byte characters instead of 3 byte characters. * - * If you want to convert a 3-byte setup into a 4-byte setup please run the - * migration command: - * ./occ db:convert-mysql-charset - * + * If you want to convert an existing 3-byte setup into a 4-byte setup please + * set the parameters in MySQL as mentioned below run the migration command: + * ./occ db:convert-mysql-charset + * The config setting will be set automatically after a successful run. + * + * Consult the documentation for more details. + * * MySQL requires a special setup for longer indexes (> 767 bytes) which are * needed: * From 0aa5ddff13aa2ad39fc3e5114bdf4d554aaff200 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Thu, 27 Apr 2017 08:41:49 +0200 Subject: [PATCH 6/7] Fix autoloading Signed-off-by: Joas Schilling --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 8a883938b5..02dee593a7 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -488,6 +488,7 @@ return array( 'OC\\DB\\MigrationException' => $baseDir . '/lib/private/DB/MigrationException.php', 'OC\\DB\\Migrator' => $baseDir . '/lib/private/DB/Migrator.php', 'OC\\DB\\MySQLMigrator' => $baseDir . '/lib/private/DB/MySQLMigrator.php', + 'OC\\DB\\MySqlTools' => $baseDir . '/lib/private/DB/MySqlTools.php', 'OC\\DB\\NoCheckMigrator' => $baseDir . '/lib/private/DB/NoCheckMigrator.php', 'OC\\DB\\OCSqlitePlatform' => $baseDir . '/lib/private/DB/OCSqlitePlatform.php', 'OC\\DB\\OracleConnection' => $baseDir . '/lib/private/DB/OracleConnection.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 7076157262..754360466c 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -518,6 +518,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\DB\\MigrationException' => __DIR__ . '/../../..' . '/lib/private/DB/MigrationException.php', 'OC\\DB\\Migrator' => __DIR__ . '/../../..' . '/lib/private/DB/Migrator.php', 'OC\\DB\\MySQLMigrator' => __DIR__ . '/../../..' . '/lib/private/DB/MySQLMigrator.php', + 'OC\\DB\\MySqlTools' => __DIR__ . '/../../..' . '/lib/private/DB/MySqlTools.php', 'OC\\DB\\NoCheckMigrator' => __DIR__ . '/../../..' . '/lib/private/DB/NoCheckMigrator.php', 'OC\\DB\\OCSqlitePlatform' => __DIR__ . '/../../..' . '/lib/private/DB/OCSqlitePlatform.php', 'OC\\DB\\OracleConnection' => __DIR__ . '/../../..' . '/lib/private/DB/OracleConnection.php', From 3afad7fe407ad6c5ef772d0e89f3f3cc06111a36 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 8 May 2017 11:34:47 +0200 Subject: [PATCH 7/7] Fix docs Signed-off-by: Joas Schilling --- config/config.sample.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.sample.php b/config/config.sample.php index c0e897a845..aa0be98d8f 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -1238,7 +1238,7 @@ $CONFIG = array( * and MySQL can handle 4 byte characters instead of 3 byte characters. * * If you want to convert an existing 3-byte setup into a 4-byte setup please - * set the parameters in MySQL as mentioned below run the migration command: + * set the parameters in MySQL as mentioned below and run the migration command: * ./occ db:convert-mysql-charset * The config setting will be set automatically after a successful run. *