From 972e560e7274cf25021b1a5095206640b063789a Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 30 Jul 2015 10:57:16 +0200 Subject: [PATCH 01/14] Adding tests for 4 byte unicode characters * success on SQLite and Postgres * failure on MySQL due to the limited charset that only supports up to 3 bytes --- tests/data/db_structure.xml | 15 +++++++++++++++ tests/lib/DB/LegacyDBTest.php | 31 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/tests/data/db_structure.xml b/tests/data/db_structure.xml index 371da94483..b155114f2e 100644 --- a/tests/data/db_structure.xml +++ b/tests/data/db_structure.xml @@ -293,4 +293,19 @@ + + + *dbprefix*text_table + + + + textfield + text + false + 255 + + + +
+ diff --git a/tests/lib/DB/LegacyDBTest.php b/tests/lib/DB/LegacyDBTest.php index 7aeeb3dd1f..2c91121c02 100644 --- a/tests/lib/DB/LegacyDBTest.php +++ b/tests/lib/DB/LegacyDBTest.php @@ -46,6 +46,11 @@ class LegacyDBTest extends \Test\TestCase { */ private $table5; + /** + * @var string + */ + private $text_table; + protected function setUp() { parent::setUp(); @@ -63,6 +68,7 @@ class LegacyDBTest extends \Test\TestCase { $this->table3 = $this->test_prefix.'vcategory'; $this->table4 = $this->test_prefix.'decimal'; $this->table5 = $this->test_prefix.'uniconst'; + $this->text_table = $this->test_prefix.'text_table'; } protected function tearDown() { @@ -390,4 +396,29 @@ class LegacyDBTest extends \Test\TestCase { $result = $query->execute(array('%ba%')); $this->assertCount(1, $result->fetchAll()); } + + /** + * @dataProvider insertAndSelectDataProvider + */ + public function testInsertAndSelectData($expected) { + $table = "*PREFIX*{$this->text_table}"; + + $query = OC_DB::prepare("INSERT INTO `$table` (`textfield`) VALUES (?)"); + $result = $query->execute(array($expected)); + $this->assertEquals(1, $result); + + $actual = OC_DB::prepare("SELECT `textfield` FROM `$table`")->execute()->fetchOne(); + $this->assertSame($expected, $actual); + } + + public function insertAndSelectDataProvider() { + return [ + ['abcdefghijklmnopqrstuvwxyzABCDEFGHIKLMNOPQRSTUVWXYZ'], + ['0123456789'], + ['äöüÄÖÜß!"§$%&/()=?#\'+*~°^`´'], + ['²³¼½¬{[]}\\'], + ['♡⚗'], + ['💩'], # :hankey: on github + ]; + } } From cc28f82b369c2e8ebf2d0b4390379b9cda4af40b Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 30 Jul 2015 13:57:04 +0200 Subject: [PATCH 02/14] Add config option to update charset of mysql to utf8mb4 * fully optional * requires additional options set in the database --- config/config.sample.php | 29 ++++++++++++++++++++++++++++ core/register_command.php | 2 +- lib/private/DB/AdapterMySQL.php | 3 ++- lib/private/DB/ConnectionFactory.php | 7 +++++++ lib/private/DB/MDB2SchemaReader.php | 14 +++++++++++++- lib/private/DB/MDB2SchemaWriter.php | 6 +++++- lib/private/Repair/Collation.php | 7 +++++-- lib/private/Server.php | 2 +- lib/private/Setup/MySQL.php | 5 +++-- 9 files changed, 66 insertions(+), 9 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index 3aa0f353c5..df1e2d16fc 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -129,6 +129,7 @@ $CONFIG = array( */ 'dbtableprefix' => '', + /** * Indicates whether the Nextcloud instance was installed successfully; ``true`` * indicates a successful installation, and ``false`` indicates an unsuccessful @@ -1079,6 +1080,34 @@ $CONFIG = array( */ 'sqlite.journal_mode' => 'DELETE', +/** + * If this setting is set to true MySQL can handle 4 byte characters instead of + * 3 byte characters + * + * MySQL requires a special setup for longer indexes (> 767 bytes) which are + * needed: + * + * [mysqld] + * innodb_large_prefix=true + * innodb_file_format=barracuda + * innodb_file_per_table=true + * + * Tables will be created with + * * character set: utf8mb4 + * * collation: utf8mb4_bin + * * row_format: compressed + * + * See: + * https://dev.mysql.com/doc/refman/5.7/en/charset-unicode-utf8mb4.html + * https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_large_prefix + * https://mariadb.com/kb/en/mariadb/xtradbinnodb-server-system-variables/#innodb_large_prefix + * http://www.tocker.ca/2013/10/31/benchmarking-innodb-page-compression-performance.html + * http://mechanics.flite.com/blog/2014/07/29/using-innodb-large-prefix-to-avoid-error-1071/ + * + * WARNING: EXPERIMENTAL + */ +'mysql.utf8mb4' => false, + /** * Database types that are supported for installation. * diff --git a/core/register_command.php b/core/register_command.php index a6da3cbd89..fa85aea895 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -83,7 +83,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) { $application->add(new OC\Core\Command\Config\System\SetConfig(\OC::$server->getSystemConfig())); $application->add(new OC\Core\Command\Db\GenerateChangeScript()); - $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory())); + $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig()))); $application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager())); diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php index 3e2fceda8d..0c0c6b3102 100644 --- a/lib/private/DB/AdapterMySQL.php +++ b/lib/private/DB/AdapterMySQL.php @@ -39,7 +39,8 @@ class AdapterMySQL extends Adapter { } public function fixupStatement($statement) { - $statement = str_replace(' ILIKE ', ' COLLATE utf8_general_ci LIKE ', $statement); + $characterSet = \OC::$server->getConfig()->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; + $statement = str_replace(' ILIKE ', ' COLLATE ' . $characterSet . '_general_ci LIKE ', $statement); return $statement; } } diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php index b2c356edef..a7aae32f67 100644 --- a/lib/private/DB/ConnectionFactory.php +++ b/lib/private/DB/ConnectionFactory.php @@ -28,6 +28,7 @@ namespace OC\DB; use Doctrine\DBAL\Event\Listeners\OracleSessionInit; use Doctrine\DBAL\Event\Listeners\SQLSessionInit; use Doctrine\DBAL\Event\Listeners\MysqlSessionInit; +use OC\SystemConfig; /** * Takes care of creating and configuring Doctrine connections. @@ -64,6 +65,12 @@ class ConnectionFactory { ), ); + public function __construct(SystemConfig $systemConfig) { + if($systemConfig->getValue('mysql.utf8mb4', false)) { + $defaultConnectionParams['mysql']['charset'] = 'utf8mb4'; + } + } + /** * @brief Get default connection parameters for a given DBMS. * @param string $type DBMS type diff --git a/lib/private/DB/MDB2SchemaReader.php b/lib/private/DB/MDB2SchemaReader.php index 3f183c9723..c198bb31e0 100644 --- a/lib/private/DB/MDB2SchemaReader.php +++ b/lib/private/DB/MDB2SchemaReader.php @@ -33,6 +33,7 @@ namespace OC\DB; use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\SchemaConfig; +use Doctrine\DBAL\Platforms\MySqlPlatform; use OCP\IConfig; class MDB2SchemaReader { @@ -54,12 +55,16 @@ class MDB2SchemaReader { /** @var \Doctrine\DBAL\Schema\SchemaConfig $schemaConfig */ protected $schemaConfig; + /** @var IConfig */ + protected $config; + /** * @param \OCP\IConfig $config * @param \Doctrine\DBAL\Platforms\AbstractPlatform $platform */ public function __construct(IConfig $config, AbstractPlatform $platform) { $this->platform = $platform; + $this->config = $config; $this->DBNAME = $config->getSystemValue('dbname', 'owncloud'); $this->DBTABLEPREFIX = $config->getSystemValue('dbtableprefix', 'oc_'); @@ -118,8 +123,15 @@ class MDB2SchemaReader { $name = str_replace('*dbprefix*', $this->DBTABLEPREFIX, $name); $name = $this->platform->quoteIdentifier($name); $table = $schema->createTable($name); - $table->addOption('collate', 'utf8_bin'); $table->setSchemaConfig($this->schemaConfig); + + if($this->platform instanceof MySqlPlatform && $this->config->getSystemValue('mysql.utf8mb4', false)) { + $table->addOption('charset', 'utf8mb4'); + $table->addOption('collate', 'utf8mb4_bin'); + $table->addOption('row_format', 'compressed'); + } else { + $table->addOption('collate', 'utf8_bin'); + } break; case 'create': case 'overwrite': diff --git a/lib/private/DB/MDB2SchemaWriter.php b/lib/private/DB/MDB2SchemaWriter.php index 26e32b036f..7664b4359a 100644 --- a/lib/private/DB/MDB2SchemaWriter.php +++ b/lib/private/DB/MDB2SchemaWriter.php @@ -43,7 +43,11 @@ class MDB2SchemaWriter { $xml->addChild('name', $config->getSystemValue('dbname', 'owncloud')); $xml->addChild('create', 'true'); $xml->addChild('overwrite', 'false'); - $xml->addChild('charset', 'utf8'); + if($config->getSystemValue('dbtype', 'sqlite') === 'mysql' && $config->getSystemValue('mysql.utf8mb4', false)) { + $xml->addChild('charset', 'utf8mb4'); + } else { + $xml->addChild('charset', 'utf8'); + } // FIX ME: bloody work around if ($config->getSystemValue('dbtype', 'sqlite') === 'oci') { diff --git a/lib/private/Repair/Collation.php b/lib/private/Repair/Collation.php index 74c4ca2e6a..c19b8eea5e 100644 --- a/lib/private/Repair/Collation.php +++ b/lib/private/Repair/Collation.php @@ -61,10 +61,12 @@ class Collation implements IRepairStep { return; } + $characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; + $tables = $this->getAllNonUTF8BinTables($this->connection); foreach ($tables as $table) { $output->info("Change collation for $table ..."); - $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin;'); + $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;'); $query->execute(); } } @@ -75,11 +77,12 @@ class Collation implements IRepairStep { */ protected function getAllNonUTF8BinTables($connection) { $dbName = $this->config->getSystemValue("dbname"); + $characterSet = $this->config->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; $rows = $connection->fetchAll( "SELECT DISTINCT(TABLE_NAME) AS `table`" . " FROM INFORMATION_SCHEMA . COLUMNS" . " WHERE TABLE_SCHEMA = ?" . - " AND (COLLATION_NAME <> 'utf8_bin' OR CHARACTER_SET_NAME <> 'utf8')" . + " AND (COLLATION_NAME <> '" . $characterSet . "_bin' OR CHARACTER_SET_NAME <> '" . $characterSet . "')" . " AND TABLE_NAME LIKE \"*PREFIX*%\"", array($dbName) ); diff --git a/lib/private/Server.php b/lib/private/Server.php index 291714b23d..063ff4a3d3 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -407,8 +407,8 @@ class Server extends ServerContainer implements IServerContainer { return new CredentialsManager($c->getCrypto(), $c->getDatabaseConnection()); }); $this->registerService('DatabaseConnection', function (Server $c) { - $factory = new \OC\DB\ConnectionFactory(); $systemConfig = $c->getSystemConfig(); + $factory = new \OC\DB\ConnectionFactory($systemConfig); $type = $systemConfig->getValue('dbtype', 'sqlite'); if (!$factory->isValidType($type)) { throw new \OC\DatabaseException('Invalid database type'); diff --git a/lib/private/Setup/MySQL.php b/lib/private/Setup/MySQL.php index 4ad6926c2d..c022616d8b 100644 --- a/lib/private/Setup/MySQL.php +++ b/lib/private/Setup/MySQL.php @@ -58,8 +58,9 @@ class MySQL extends AbstractDatabase { try{ $name = $this->dbName; $user = $this->dbUser; - //we can't use OC_BD functions here because we need to connect as the administrative user. - $query = "CREATE DATABASE IF NOT EXISTS `$name` CHARACTER SET utf8 COLLATE utf8_bin;"; + //we can't use OC_DB functions here because we need to connect as the administrative user. + $characterSet = \OC::$server->getSystemConfig()->getValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; + $query = "CREATE DATABASE IF NOT EXISTS `$name` CHARACTER SET $characterSet COLLATE ${characterSet}_bin;"; $connection->executeUpdate($query); } catch (\Exception $ex) { $this->logger->error('Database creation failed: {error}', [ From 296a3274cf9fccc5fea1f067bb5cb9cc71450734 Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 30 Jul 2015 14:57:17 +0200 Subject: [PATCH 03/14] only disable unicode test on mysql --- tests/lib/DB/LegacyDBTest.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/lib/DB/LegacyDBTest.php b/tests/lib/DB/LegacyDBTest.php index 2c91121c02..d28dfd1448 100644 --- a/tests/lib/DB/LegacyDBTest.php +++ b/tests/lib/DB/LegacyDBTest.php @@ -400,7 +400,7 @@ class LegacyDBTest extends \Test\TestCase { /** * @dataProvider insertAndSelectDataProvider */ - public function testInsertAndSelectData($expected) { + public function testInsertAndSelectData($expected, $skipOnMysql) { $table = "*PREFIX*{$this->text_table}"; $query = OC_DB::prepare("INSERT INTO `$table` (`textfield`) VALUES (?)"); @@ -408,17 +408,21 @@ class LegacyDBTest extends \Test\TestCase { $this->assertEquals(1, $result); $actual = OC_DB::prepare("SELECT `textfield` FROM `$table`")->execute()->fetchOne(); + $config = \OC::$server->getConfig(); + if($skipOnMysql && $config->getSystemValue('dbtype', 'sqlite') === 'mysql' && $config->getSystemValue('mysql.utf8mb4', false) === false) { + return; + } $this->assertSame($expected, $actual); } public function insertAndSelectDataProvider() { return [ - ['abcdefghijklmnopqrstuvwxyzABCDEFGHIKLMNOPQRSTUVWXYZ'], - ['0123456789'], - ['äöüÄÖÜß!"§$%&/()=?#\'+*~°^`´'], - ['²³¼½¬{[]}\\'], - ['♡⚗'], - ['💩'], # :hankey: on github + ['abcdefghijklmnopqrstuvwxyzABCDEFGHIKLMNOPQRSTUVWXYZ', false], + ['0123456789', false], + ['äöüÄÖÜß!"§$%&/()=?#\'+*~°^`´', false], + ['²³¼½¬{[]}\\', false], + ['♡⚗', false], + ['💩', true], # :hankey: on github ]; } } From a7245ea08284568f9a449a11b726048dcec06d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Tue, 9 Feb 2016 17:25:12 +0100 Subject: [PATCH 04/14] Fixing ctor call --- core/register_command.php | 2 +- lib/private/DB/ConnectionFactory.php | 6 +++--- lib/private/Server.php | 2 +- lib/private/Setup/AbstractDatabase.php | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/register_command.php b/core/register_command.php index fa85aea895..89b0cf31ef 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -83,7 +83,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) { $application->add(new OC\Core\Command\Config\System\SetConfig(\OC::$server->getSystemConfig())); $application->add(new OC\Core\Command\Db\GenerateChangeScript()); - $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig()))); + $application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getConfig()))); $application->add(new OC\Core\Command\Encryption\Disable(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Encryption\Enable(\OC::$server->getConfig(), \OC::$server->getEncryptionManager())); diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php index a7aae32f67..8a1ed60b25 100644 --- a/lib/private/DB/ConnectionFactory.php +++ b/lib/private/DB/ConnectionFactory.php @@ -28,7 +28,7 @@ namespace OC\DB; use Doctrine\DBAL\Event\Listeners\OracleSessionInit; use Doctrine\DBAL\Event\Listeners\SQLSessionInit; use Doctrine\DBAL\Event\Listeners\MysqlSessionInit; -use OC\SystemConfig; +use OCP\IConfig; /** * Takes care of creating and configuring Doctrine connections. @@ -65,8 +65,8 @@ class ConnectionFactory { ), ); - public function __construct(SystemConfig $systemConfig) { - if($systemConfig->getValue('mysql.utf8mb4', false)) { + public function __construct(IConfig $config) { + if($config->getSystemValue('mysql.utf8mb4', false)) { $defaultConnectionParams['mysql']['charset'] = 'utf8mb4'; } } diff --git a/lib/private/Server.php b/lib/private/Server.php index 063ff4a3d3..11558118d5 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -408,7 +408,7 @@ class Server extends ServerContainer implements IServerContainer { }); $this->registerService('DatabaseConnection', function (Server $c) { $systemConfig = $c->getSystemConfig(); - $factory = new \OC\DB\ConnectionFactory($systemConfig); + $factory = new \OC\DB\ConnectionFactory($c->getConfig()); $type = $systemConfig->getValue('dbtype', 'sqlite'); if (!$factory->isValidType($type)) { throw new \OC\DatabaseException('Invalid database type'); diff --git a/lib/private/Setup/AbstractDatabase.php b/lib/private/Setup/AbstractDatabase.php index 310f74d4c0..47c3e5ee1c 100644 --- a/lib/private/Setup/AbstractDatabase.php +++ b/lib/private/Setup/AbstractDatabase.php @@ -134,7 +134,7 @@ abstract class AbstractDatabase { } $connectionParams = array_merge($connectionParams, $configOverwrite); - $cf = new ConnectionFactory(); + $cf = new ConnectionFactory($this->config); return $cf->getConnection($this->config->getSystemValue('dbtype', 'sqlite'), $connectionParams); } From 9c3f066dabe57af6f5fedb1579f724c58adb8f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Tue, 9 Feb 2016 17:38:12 +0100 Subject: [PATCH 05/14] Adding docker based unit test execution for mysql utf8mb4 --- autotest.sh | 27 ++++++++++++++++++++++++++- tests/docker/mysqlmb4.config.php | 5 +++++ tests/docker/mysqlmb4/mb4.cnf | 5 +++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/docker/mysqlmb4.config.php create mode 100644 tests/docker/mysqlmb4/mb4.cnf diff --git a/autotest.sh b/autotest.sh index eca3d81c04..8674fb74c3 100755 --- a/autotest.sh +++ b/autotest.sh @@ -21,7 +21,7 @@ ADMINLOGIN=admin$EXECUTOR_NUMBER BASEDIR=$PWD PRIMARY_STORAGE_CONFIGS="local swift" -DBCONFIGS="sqlite mysql mariadb pgsql oci" +DBCONFIGS="sqlite mysql mariadb pgsql oci mysqlmb4" # $PHP_EXE is run through 'which' and as such e.g. 'php' or 'hhvm' is usually # sufficient. Due to the behaviour of 'which', $PHP_EXE may also be a path @@ -209,6 +209,31 @@ function execute_tests { exit 1 fi fi + if [ "$DB" == "mysqlmb4" ] ; then + echo "Fire up the mysql docker" + DOCKER_CONTAINER_ID=$(docker run \ + -v tests/docker/mysqlmb4:/etc/mysql/conf.d \ + -e MYSQL_ROOT_PASSWORD=owncloud \ + -e MYSQL_USER="$DATABASEUSER" \ + -e MYSQL_PASSWORD=owncloud \ + -e MYSQL_DATABASE="$DATABASENAME" \ + -d mysql:5.7) + + DATABASEHOST=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "$DOCKER_CONTAINER_ID") + + echo "Waiting for MySQL(utf8mb4) initialisation ..." + + if ! apps/files_external/tests/env/wait-for-connection $DATABASEHOST 3306 60; then + echo "[ERROR] Waited 60 seconds, no response" >&2 + exit 1 + fi + sleep 1 + + echo "MySQL(utf8mb4) is up." + _DB="mysql" + + cp tests/docker/mysqlmb4.config.php config + fi if [ "$DB" == "mariadb" ] ; then if [ ! -z "$USEDOCKER" ] ; then echo "Fire up the mariadb docker" diff --git a/tests/docker/mysqlmb4.config.php b/tests/docker/mysqlmb4.config.php new file mode 100644 index 0000000000..4e78272fab --- /dev/null +++ b/tests/docker/mysqlmb4.config.php @@ -0,0 +1,5 @@ + true, +); diff --git a/tests/docker/mysqlmb4/mb4.cnf b/tests/docker/mysqlmb4/mb4.cnf new file mode 100644 index 0000000000..00333eab10 --- /dev/null +++ b/tests/docker/mysqlmb4/mb4.cnf @@ -0,0 +1,5 @@ + +[mysqld] +innodb_large_prefix=true +innodb_file_format=barracuda +innodb_file_per_table=true From d294ef23dde27b22c1c96e576b50753a724e4189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Wed, 20 Apr 2016 12:14:13 +0200 Subject: [PATCH 06/14] fix docker --- autotest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autotest.sh b/autotest.sh index 8674fb74c3..4364e41fa3 100755 --- a/autotest.sh +++ b/autotest.sh @@ -212,7 +212,7 @@ function execute_tests { if [ "$DB" == "mysqlmb4" ] ; then echo "Fire up the mysql docker" DOCKER_CONTAINER_ID=$(docker run \ - -v tests/docker/mysqlmb4:/etc/mysql/conf.d \ + -v $BASEDIR/tests/docker/mysqlmb4:/etc/mysql/conf.d \ -e MYSQL_ROOT_PASSWORD=owncloud \ -e MYSQL_USER="$DATABASEUSER" \ -e MYSQL_PASSWORD=owncloud \ From 9356a0e5830a82ddcf48d52ffc48a8fb8f7b68a7 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 18 Oct 2016 10:22:02 +0200 Subject: [PATCH 07/14] Correctly save and pass on the charset Signed-off-by: Joas Schilling --- lib/private/DB/ConnectionFactory.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/private/DB/ConnectionFactory.php b/lib/private/DB/ConnectionFactory.php index 8a1ed60b25..adb180da0c 100644 --- a/lib/private/DB/ConnectionFactory.php +++ b/lib/private/DB/ConnectionFactory.php @@ -67,7 +67,7 @@ class ConnectionFactory { public function __construct(IConfig $config) { if($config->getSystemValue('mysql.utf8mb4', false)) { - $defaultConnectionParams['mysql']['charset'] = 'utf8mb4'; + $this->defaultConnectionParams['mysql']['charset'] = 'utf8mb4'; } } @@ -106,7 +106,9 @@ class ConnectionFactory { case 'mysql': // Send "SET NAMES utf8". Only required on PHP 5.3 below 5.3.6. // See http://stackoverflow.com/questions/4361459/php-pdo-charset-set-names#4361485 - $eventManager->addEventSubscriber(new MysqlSessionInit); + $eventManager->addEventSubscriber(new MysqlSessionInit( + $this->defaultConnectionParams['mysql']['charset'] + )); $eventManager->addEventSubscriber( new SQLSessionInit("SET SESSION AUTOCOMMIT=1")); break; From d0a3d17912cbbf31e6a5dd93aa190c48283de1cc Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Tue, 18 Oct 2016 10:31:05 +0200 Subject: [PATCH 08/14] add 4 byte mysql test run Signed-off-by: Morris Jobke --- .drone.yml | 22 ++++++++++++++++++++++ autotest.sh | 32 +++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/.drone.yml b/.drone.yml index c54907cc5d..025d223f2d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -148,6 +148,14 @@ pipeline: matrix: DB: postgres PHP: 5.6 + mysqlmb4-php5.6: + image: nextcloudci/php5.6:php5.6-2 + commands: + - NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh mysqlmb4 + when: + matrix: + DB: mysqlmb4 + PHP: 5.6 integration-capabilities_features: image: nextcloudci/integration-php7.0:integration-php7.0-1 commands: @@ -368,6 +376,8 @@ matrix: PHP: 5.6 - DB: postgres PHP: 5.6 + - DB: mysqlmb4 + PHP: 5.6 services: cache: @@ -390,3 +400,15 @@ services: when: matrix: DB: mysql + mysqlmb4: + image: mysql + environment: + - MYSQL_ROOT_PASSWORD=owncloud + - MYSQL_USER=oc_autotest + - MYSQL_PASSWORD=owncloud + - MYSQL_DATABASE=oc_autotest + volumes: + - /drone/src/github.com/nextcloud/server/tests/docker/mysqlmb4:/etc/mysql/conf.d + when: + matrix: + DB: mysqlmb4 diff --git a/autotest.sh b/autotest.sh index 4364e41fa3..c77562e760 100755 --- a/autotest.sh +++ b/autotest.sh @@ -210,16 +210,30 @@ function execute_tests { fi fi if [ "$DB" == "mysqlmb4" ] ; then - echo "Fire up the mysql docker" - DOCKER_CONTAINER_ID=$(docker run \ - -v $BASEDIR/tests/docker/mysqlmb4:/etc/mysql/conf.d \ - -e MYSQL_ROOT_PASSWORD=owncloud \ - -e MYSQL_USER="$DATABASEUSER" \ - -e MYSQL_PASSWORD=owncloud \ - -e MYSQL_DATABASE="$DATABASENAME" \ - -d mysql:5.7) + if [ ! -z "$USEDOCKER" ] ; then + echo "Fire up the mysql docker" + DOCKER_CONTAINER_ID=$(docker run \ + -v $BASEDIR/tests/docker/mysqlmb4:/etc/mysql/conf.d \ + -e MYSQL_ROOT_PASSWORD=owncloud \ + -e MYSQL_USER="$DATABASEUSER" \ + -e MYSQL_PASSWORD=owncloud \ + -e MYSQL_DATABASE="$DATABASENAME" \ + -d mysql:5.7) - DATABASEHOST=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "$DOCKER_CONTAINER_ID") + DATABASEHOST=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "$DOCKER_CONTAINER_ID") + + else + if [ -z "$DRONE" ] ; then # no need to drop the DB when we are on CI + if [ "mysql" != "$(mysql --version | grep -o mysql)" ] ; then + echo "Your mysql binary is not provided by mysql" + echo "To use the docker container set the USEDOCKER environment variable" + exit -1 + fi + mysql -u "$DATABASEUSER" -powncloud -e "DROP DATABASE IF EXISTS $DATABASENAME" -h $DATABASEHOST || true + else + DATABASEHOST=127.0.0.1 + fi + fi echo "Waiting for MySQL(utf8mb4) initialisation ..." From 15bbe0210670d5b23b265628209b9b3de50d0fd6 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 18 Oct 2016 11:10:55 +0200 Subject: [PATCH 09/14] Ignore failures of collation change in the pre update step Signed-off-by: Joas Schilling --- lib/private/Repair.php | 3 ++- lib/private/Repair/Collation.php | 38 +++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 2ba118b9c3..7a5ef9fbd9 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -129,6 +129,7 @@ class Repair implements IOutput{ */ public static function getRepairSteps() { return [ + new Collation(\OC::$server->getConfig(), \OC::$server->getLogger(), \OC::$server->getDatabaseConnection(), false), new RepairMimeTypes(\OC::$server->getConfig()), new RepairLegacyStorages(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()), new AssetCache(), @@ -179,7 +180,7 @@ class Repair implements IOutput{ $connection = \OC::$server->getDatabaseConnection(); $steps = [ new InnoDB(), - new Collation(\OC::$server->getConfig(), $connection), + new Collation(\OC::$server->getConfig(), \OC::$server->getLogger(), $connection, true), new SqliteAutoincrement($connection), new SearchLuceneTables(), ]; diff --git a/lib/private/Repair/Collation.php b/lib/private/Repair/Collation.php index c19b8eea5e..54de1a719b 100644 --- a/lib/private/Repair/Collation.php +++ b/lib/private/Repair/Collation.php @@ -24,28 +24,38 @@ namespace OC\Repair; +use Doctrine\DBAL\Exception\DriverException; use Doctrine\DBAL\Platforms\MySqlPlatform; +use OCP\IConfig; +use OCP\IDBConnection; +use OCP\ILogger; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; class Collation implements IRepairStep { - /** - * @var \OCP\IConfig - */ + /** @var IConfig */ protected $config; - /** - * @var \OC\DB\Connection - */ + /** @var ILogger */ + protected $logger; + + /** @var IDBConnection */ protected $connection; + /** @var bool */ + protected $ignoreFailures; + /** - * @param \OCP\IConfig $config - * @param \OC\DB\Connection $connection + * @param IConfig $config + * @param ILogger $logger + * @param IDBConnection $connection + * @param bool $ignoreFailures */ - public function __construct($config, $connection) { + public function __construct(IConfig $config, ILogger $logger, IDBConnection $connection, $ignoreFailures) { $this->connection = $connection; $this->config = $config; + $this->logger = $logger; + $this->ignoreFailures = $ignoreFailures; } public function getName() { @@ -67,7 +77,15 @@ class Collation implements IRepairStep { foreach ($tables as $table) { $output->info("Change collation for $table ..."); $query = $this->connection->prepare('ALTER TABLE `' . $table . '` CONVERT TO CHARACTER SET ' . $characterSet . ' COLLATE ' . $characterSet . '_bin;'); - $query->execute(); + try { + $query->execute(); + } catch (DriverException $e) { + // Just log this + $this->logger->logException($e); + if (!$this->ignoreFailures) { + throw $e; + } + } } } From 17a2723948830e30713d012f0739c336af2a12f1 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 18 Oct 2016 11:17:13 +0200 Subject: [PATCH 10/14] Fix the test Signed-off-by: Joas Schilling --- tests/lib/DB/LegacyDBTest.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/lib/DB/LegacyDBTest.php b/tests/lib/DB/LegacyDBTest.php index d28dfd1448..f3de570c52 100644 --- a/tests/lib/DB/LegacyDBTest.php +++ b/tests/lib/DB/LegacyDBTest.php @@ -400,18 +400,18 @@ class LegacyDBTest extends \Test\TestCase { /** * @dataProvider insertAndSelectDataProvider */ - public function testInsertAndSelectData($expected, $skipOnMysql) { + public function testInsertAndSelectData($expected, $throwsOnMysqlWithoutUTF8MB4) { $table = "*PREFIX*{$this->text_table}"; + $config = \OC::$server->getConfig(); $query = OC_DB::prepare("INSERT INTO `$table` (`textfield`) VALUES (?)"); + if ($throwsOnMysqlWithoutUTF8MB4 && $config->getSystemValue('dbtype', 'sqlite') === 'mysql' && $config->getSystemValue('mysql.utf8mb4', false) === false) { + $this->markTestSkipped('MySQL requires UTF8mb4 to store value: ' . $expected); + } $result = $query->execute(array($expected)); $this->assertEquals(1, $result); $actual = OC_DB::prepare("SELECT `textfield` FROM `$table`")->execute()->fetchOne(); - $config = \OC::$server->getConfig(); - if($skipOnMysql && $config->getSystemValue('dbtype', 'sqlite') === 'mysql' && $config->getSystemValue('mysql.utf8mb4', false) === false) { - return; - } $this->assertSame($expected, $actual); } From 64c9ef96c4a8b205e32cdc1a38038aef18dcf4cf Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 18 Oct 2016 11:37:49 +0200 Subject: [PATCH 11/14] Fix like queries in the QueryBuilder Signed-off-by: Joas Schilling --- lib/private/DB/AdapterMySQL.php | 15 +++++++++++++-- .../MySqlExpressionBuilder.php | 19 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/private/DB/AdapterMySQL.php b/lib/private/DB/AdapterMySQL.php index 0c0c6b3102..aa784bb83d 100644 --- a/lib/private/DB/AdapterMySQL.php +++ b/lib/private/DB/AdapterMySQL.php @@ -27,6 +27,9 @@ namespace OC\DB; class AdapterMySQL extends Adapter { + /** @var string */ + protected $charset; + /** * @param string $tableName */ @@ -39,8 +42,16 @@ class AdapterMySQL extends Adapter { } public function fixupStatement($statement) { - $characterSet = \OC::$server->getConfig()->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8'; - $statement = str_replace(' ILIKE ', ' COLLATE ' . $characterSet . '_general_ci LIKE ', $statement); + $statement = str_replace(' ILIKE ', ' COLLATE ' . $this->getCharset() . '_general_ci LIKE ', $statement); return $statement; } + + protected function getCharset() { + if (!$this->charset) { + $params = $this->conn->getParams(); + $this->charset = isset($params['charset']) ? $params['charset'] : 'utf8'; + } + + return $this->charset; + } } diff --git a/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php b/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php index 66d8851632..17f7fd5aa4 100644 --- a/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php +++ b/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php @@ -24,18 +24,31 @@ namespace OC\DB\QueryBuilder\ExpressionBuilder; -use OC\DB\QueryBuilder\QueryFunction; -use OCP\DB\QueryBuilder\IQueryBuilder; +use OC\DB\Connection; +use OCP\IDBConnection; class MySqlExpressionBuilder extends ExpressionBuilder { + /** @var string */ + protected $charset; + + /** + * @param \OCP\IDBConnection|Connection $connection + */ + public function __construct(IDBConnection $connection) { + parent::__construct($connection); + + $params = $connection->getParams(); + $this->charset = isset($params['charset']) ? $params['charset'] : 'utf8'; + } + /** * @inheritdoc */ public function iLike($x, $y, $type = null) { $x = $this->helper->quoteColumnName($x); $y = $this->helper->quoteColumnName($y); - return $this->expressionBuilder->comparison($x, ' COLLATE utf8_general_ci LIKE', $y); + return $this->expressionBuilder->comparison($x, ' COLLATE ' . $this->charset . '_general_ci LIKE', $y); } } From b1235a67de9589457af91dfc322d7853199547b0 Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Tue, 18 Oct 2016 12:40:17 +0200 Subject: [PATCH 12/14] test alternative drone syntax for command options Signed-off-by: Morris Jobke --- .drone.yml | 3 +-- autotest.sh | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 025d223f2d..78cdefd82b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -407,8 +407,7 @@ services: - MYSQL_USER=oc_autotest - MYSQL_PASSWORD=owncloud - MYSQL_DATABASE=oc_autotest - volumes: - - /drone/src/github.com/nextcloud/server/tests/docker/mysqlmb4:/etc/mysql/conf.d + command: [ "--innodb_large_prefix=true", "--innodb_file_format=barracuda", "--innodb_file_per_table=true" ] when: matrix: DB: mysqlmb4 diff --git a/autotest.sh b/autotest.sh index c77562e760..61e7426ab9 100755 --- a/autotest.sh +++ b/autotest.sh @@ -218,7 +218,10 @@ function execute_tests { -e MYSQL_USER="$DATABASEUSER" \ -e MYSQL_PASSWORD=owncloud \ -e MYSQL_DATABASE="$DATABASENAME" \ - -d mysql:5.7) + -d mysql:5.7 + --innodb_large_prefix=true + --innodb_file_format=barracuda + --innodb_file_per_table=true) DATABASEHOST=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "$DOCKER_CONTAINER_ID") From 43b7b143f46fd59911ef463d60d158dcc925d4af Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 18 Oct 2016 16:09:48 +0200 Subject: [PATCH 13/14] Fix test of repair step Signed-off-by: Joas Schilling --- tests/lib/Repair/RepairCollationTest.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/lib/Repair/RepairCollationTest.php b/tests/lib/Repair/RepairCollationTest.php index 2e304a74ab..897f772a79 100644 --- a/tests/lib/Repair/RepairCollationTest.php +++ b/tests/lib/Repair/RepairCollationTest.php @@ -1,9 +1,4 @@ * This file is licensed under the Affero General Public License version 3 or @@ -11,6 +6,11 @@ use OCP\Migration\IOutput; * See the COPYING-README file. */ +namespace Test\Repair; + +use OCP\ILogger; +use OCP\Migration\IOutput; + class TestCollationRepair extends \OC\Repair\Collation { /** * @param \Doctrine\DBAL\Connection $connection @@ -50,10 +50,14 @@ class RepairCollationTest extends \Test\TestCase { */ private $config; + /** @var ILogger */ + private $logger; + protected function setUp() { parent::setUp(); $this->connection = \OC::$server->getDatabaseConnection(); + $this->logger = $this->createMock(ILogger::class); $this->config = \OC::$server->getConfig(); if (!$this->connection->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\MySqlPlatform) { $this->markTestSkipped("Test only relevant on MySql"); @@ -63,7 +67,7 @@ class RepairCollationTest extends \Test\TestCase { $this->tableName = $this->getUniqueID($dbPrefix . "_collation_test"); $this->connection->exec("CREATE TABLE $this->tableName(text VARCHAR(16)) COLLATE utf8_unicode_ci"); - $this->repair = new TestCollationRepair($this->config, $this->connection); + $this->repair = new TestCollationRepair($this->config, $this->logger, $this->connection, false); } protected function tearDown() { From 303e0737244eebd055d74e2f6d75747a5cdb26da Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 18 Oct 2016 16:50:25 +0200 Subject: [PATCH 14/14] Do not skip when mysql uses utf8mb4 Signed-off-by: Joas Schilling --- tests/lib/Files/Cache/CacheTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/lib/Files/Cache/CacheTest.php b/tests/lib/Files/Cache/CacheTest.php index 4a2581fbc5..4c4f43d63d 100644 --- a/tests/lib/Files/Cache/CacheTest.php +++ b/tests/lib/Files/Cache/CacheTest.php @@ -113,7 +113,8 @@ class CacheTest extends \Test\TestCase { public function testFolder($folder) { if(strpos($folder, 'F09F9890')) { // 4 byte UTF doesn't work on mysql - if(\OC::$server->getDatabaseConnection()->getDatabasePlatform() instanceof MySqlPlatform) { + $params = \OC::$server->getDatabaseConnection()->getParams(); + if(\OC::$server->getDatabaseConnection()->getDatabasePlatform() instanceof MySqlPlatform && $params['charset'] !== 'utf8mb4') { $this->markTestSkipped('MySQL doesn\'t support 4 byte UTF-8'); } }