From 2050517d44267facdbcd9e1d075fdfbd939aec4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julius=20H=C3=A4rtl?= Date: Tue, 3 Nov 2020 17:24:06 +0100 Subject: [PATCH 01/17] Add github action for oci8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julius Härtl --- .github/workflows/oci.yml | 53 +++++++++++++++++++++++++++++++++++++ tests/lib/AppConfigTest.php | 5 ++-- tests/lib/DB/testschema.xml | 2 +- 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/oci.yml diff --git a/.github/workflows/oci.yml b/.github/workflows/oci.yml new file mode 100644 index 0000000000..87cc1d41d4 --- /dev/null +++ b/.github/workflows/oci.yml @@ -0,0 +1,53 @@ +name: "Unit tests" + +on: + push: + +jobs: + phpunit-oci8: + name: "PHPUnit on OCI8" + runs-on: "ubuntu-latest" + + strategy: + matrix: + php-version: + - "7.4" + + services: + oracle: + image: deepdiver/docker-oracle-xe-11g # "wnameless/oracle-xe-11g-r2" + ports: + - "1521:1521" + + steps: + - name: "Checkout" + uses: "actions/checkout@v2" + + - name: Checkout submodules + shell: bash + run: | + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 + + - name: "Install PHP" + uses: "shivammathur/setup-php@v2" + with: + php-version: "${{ matrix.php-version }}" + extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, oci8 + tools: phpunit:8.5.2 + coverage: none + + - name: Set up Nextcloud + run: | + mkdir data + ./occ maintenance:install --verbose --database=oci --database-name=XE --database-host=127.0.0.1 --database-port=1521 --database-user=autotest --database-pass=owncloud --admin-user admin --admin-pass admin + + # Generate instance id by loading index.php + - name: Generate instance id by loading index.php + run: | + php -f index.php + + - name: Run phpunit + run: | + cd tests && phpunit --configuration phpunit-autotest.xml --group DB,SLOWDB diff --git a/tests/lib/AppConfigTest.php b/tests/lib/AppConfigTest.php index ff23454eb8..40a99709bd 100644 --- a/tests/lib/AppConfigTest.php +++ b/tests/lib/AppConfigTest.php @@ -9,6 +9,7 @@ namespace Test; +use OC\AppConfig; use OCP\IConfig; /** @@ -42,7 +43,7 @@ class AppConfigTest extends TestCase { $sql->delete('appconfig'); $sql->execute(); - $this->overwriteService('AppConfig', new \OC\AppConfig($this->connection)); + $this->overwriteService(AppConfig::class, new \OC\AppConfig($this->connection)); $sql = $this->connection->getQueryBuilder(); $sql->insert('appconfig') @@ -132,7 +133,7 @@ class AppConfigTest extends TestCase { $sql->execute(); } - $this->restoreService('AppConfig'); + $this->restoreService(AppConfig::class); parent::tearDown(); } diff --git a/tests/lib/DB/testschema.xml b/tests/lib/DB/testschema.xml index 5f449c936d..d42dbe8d58 100644 --- a/tests/lib/DB/testschema.xml +++ b/tests/lib/DB/testschema.xml @@ -37,7 +37,7 @@ clobfield - clob + text booleanfield From e313777cd6e0a7255bf6f5c7ed151fa44ac536b0 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Wed, 4 Nov 2020 16:02:58 +0100 Subject: [PATCH 02/17] Use a different column for the primary key as we can not insert it on oracle Signed-off-by: Joas Schilling --- tests/lib/DB/MDB2SchemaReaderTest.php | 30 +++++++++++++++++---------- tests/lib/DB/testschema.xml | 18 +++++++++++++++- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/tests/lib/DB/MDB2SchemaReaderTest.php b/tests/lib/DB/MDB2SchemaReaderTest.php index c57b0f22b6..b3dd98fd6b 100644 --- a/tests/lib/DB/MDB2SchemaReaderTest.php +++ b/tests/lib/DB/MDB2SchemaReaderTest.php @@ -14,6 +14,10 @@ use Doctrine\DBAL\Schema\Schema; use OC\DB\MDB2SchemaReader; use OCP\IConfig; use Test\TestCase; +use Doctrine\DBAL\Types\IntegerType; +use Doctrine\DBAL\Types\TextType; +use Doctrine\DBAL\Types\StringType; +use Doctrine\DBAL\Types\BooleanType; /** * Class MDB2SchemaReaderTest @@ -51,13 +55,13 @@ class MDB2SchemaReaderTest extends TestCase { $this->assertCount(1, $schema->getTables()); $table = $schema->getTable('test_table'); - $this->assertCount(8, $table->getColumns()); + $this->assertCount(9, $table->getColumns()); - $this->assertEquals(4, $table->getColumn('integerfield')->getLength()); - $this->assertTrue($table->getColumn('integerfield')->getAutoincrement()); - $this->assertEquals(0, $table->getColumn('integerfield')->getDefault()); - $this->assertTrue($table->getColumn('integerfield')->getNotnull()); - $this->assertInstanceOf('Doctrine\DBAL\Types\IntegerType', $table->getColumn('integerfield')->getType()); + $this->assertEquals(4, $table->getColumn('id')->getLength()); + $this->assertTrue($table->getColumn('id')->getAutoincrement()); + $this->assertEquals(0, $table->getColumn('id')->getDefault()); + $this->assertTrue($table->getColumn('id')->getNotnull()); + $this->assertInstanceOf(IntegerType::class, $table->getColumn('id')->getType()); $this->assertSame(10, $table->getColumn('integerfield_default')->getDefault()); @@ -65,18 +69,19 @@ class MDB2SchemaReaderTest extends TestCase { $this->assertFalse($table->getColumn('textfield')->getAutoincrement()); $this->assertSame('foo', $table->getColumn('textfield')->getDefault()); $this->assertTrue($table->getColumn('textfield')->getNotnull()); - $this->assertInstanceOf('Doctrine\DBAL\Types\StringType', $table->getColumn('textfield')->getType()); + $this->assertInstanceOf(StringType::class, $table->getColumn('textfield')->getType()); $this->assertNull($table->getColumn('clobfield')->getLength()); $this->assertFalse($table->getColumn('clobfield')->getAutoincrement()); $this->assertNull($table->getColumn('clobfield')->getDefault()); $this->assertFalse($table->getColumn('clobfield')->getNotnull()); - $this->assertInstanceOf('Doctrine\DBAL\Types\TextType', $table->getColumn('clobfield')->getType()); + $this->assertInstanceOf(StringType::class, $table->getColumn('clobfield')->getType()); +// $this->assertInstanceOf(TextType::class, $table->getColumn('clobfield')->getType()); $this->assertNull($table->getColumn('booleanfield')->getLength()); $this->assertFalse($table->getColumn('booleanfield')->getAutoincrement()); $this->assertNull($table->getColumn('booleanfield')->getDefault()); - $this->assertInstanceOf('Doctrine\DBAL\Types\BooleanType', $table->getColumn('booleanfield')->getType()); + $this->assertInstanceOf(BooleanType::class, $table->getColumn('booleanfield')->getType()); $this->assertTrue($table->getColumn('booleanfield_true')->getDefault()); $this->assertFalse($table->getColumn('booleanfield_false')->getDefault()); @@ -84,10 +89,13 @@ class MDB2SchemaReaderTest extends TestCase { $this->assertEquals(12, $table->getColumn('decimalfield_precision_scale')->getPrecision()); $this->assertEquals(2, $table->getColumn('decimalfield_precision_scale')->getScale()); - $this->assertCount(2, $table->getIndexes()); - $this->assertEquals(['integerfield'], $table->getIndex('primary')->getUnquotedColumns()); + $this->assertCount(3, $table->getIndexes()); + $this->assertEquals(['id'], $table->getIndex('primary')->getUnquotedColumns()); $this->assertTrue($table->getIndex('primary')->isPrimary()); $this->assertTrue($table->getIndex('primary')->isUnique()); + $this->assertEquals(['integerfield'], $table->getIndex('index_integerfield')->getUnquotedColumns()); + $this->assertFalse($table->getIndex('index_integerfield')->isPrimary()); + $this->assertTrue($table->getIndex('index_integerfield')->isUnique()); $this->assertEquals(['booleanfield'], $table->getIndex('index_boolean')->getUnquotedColumns()); $this->assertFalse($table->getIndex('index_boolean')->isPrimary()); $this->assertFalse($table->getIndex('index_boolean')->isUnique()); diff --git a/tests/lib/DB/testschema.xml b/tests/lib/DB/testschema.xml index d42dbe8d58..a2b01d8259 100644 --- a/tests/lib/DB/testschema.xml +++ b/tests/lib/DB/testschema.xml @@ -13,7 +13,7 @@ - integerfield + id integer 0 true @@ -21,6 +21,13 @@ 4 1 + + integerfield + integer + 0 + true + 4 + integerfield_default integer @@ -64,6 +71,15 @@ index_primary true true + + id + ascending + + + + + index_integerfield + true integerfield ascending From 74c01430013ff813876c2f096e989c1e9886d762 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Wed, 4 Nov 2020 16:03:45 +0100 Subject: [PATCH 03/17] Easier debugging and spell fix Signed-off-by: Joas Schilling --- tests/lib/DB/ConnectionTest.php | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/lib/DB/ConnectionTest.php b/tests/lib/DB/ConnectionTest.php index be84cb81cb..5a2823ef28 100644 --- a/tests/lib/DB/ConnectionTest.php +++ b/tests/lib/DB/ConnectionTest.php @@ -97,14 +97,17 @@ class ConnectionTest extends \Test\TestCase { $this->assertTableNotExist('table'); } - private function getTextValueByIntergerField($integerField) { + private function getTextValueByIntegerField($integerField) { $builder = $this->connection->getQueryBuilder(); - $query = $builder->select('textfield') + $query = $builder->select('*') ->from('table') ->where($builder->expr()->eq('integerfield', $builder->createNamedParameter($integerField, IQueryBuilder::PARAM_INT))); $result = $query->execute(); - return $result->fetchColumn(); + $row = $result->fetch(); + $result->closeCursor(); + + return $row['textfield'] ?? null; } public function testSetValues() { @@ -116,7 +119,7 @@ class ConnectionTest extends \Test\TestCase { 'clobfield' => 'not_null' ]); - $this->assertEquals('foo', $this->getTextValueByIntergerField(1)); + $this->assertEquals('foo', $this->getTextValueByIntegerField(1)); } public function testSetValuesOverWrite() { @@ -133,7 +136,7 @@ class ConnectionTest extends \Test\TestCase { 'textfield' => 'bar' ]); - $this->assertEquals('bar', $this->getTextValueByIntergerField(1)); + $this->assertEquals('bar', $this->getTextValueByIntegerField(1)); } public function testSetValuesOverWritePrecondition() { @@ -154,7 +157,7 @@ class ConnectionTest extends \Test\TestCase { 'booleanfield' => true ]); - $this->assertEquals('bar', $this->getTextValueByIntergerField(1)); + $this->assertEquals('bar', $this->getTextValueByIntegerField(1)); } From 61c767128c49b8aa8bd681e8ac7edaa235812c5a Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Wed, 4 Nov 2020 16:40:31 +0100 Subject: [PATCH 04/17] Skip the insertIfNotExists() tests on Oracle because it doesn't work with clob Signed-off-by: Joas Schilling --- tests/lib/DB/ConnectionTest.php | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/lib/DB/ConnectionTest.php b/tests/lib/DB/ConnectionTest.php index 5a2823ef28..fa4d5fe005 100644 --- a/tests/lib/DB/ConnectionTest.php +++ b/tests/lib/DB/ConnectionTest.php @@ -203,6 +203,10 @@ class ConnectionTest extends \Test\TestCase { } public function testInsertIfNotExist() { + if (\OC::$server->getConfig()->getSystemValue('dbtype', 'sqlite') === 'oci') { + self::markTestSkipped('Insert if not exist does not work with clob on oracle'); + } + $this->makeTestTable(); $categoryEntries = [ ['user' => 'test', 'category' => 'Family', 'expectedResult' => 1], @@ -232,6 +236,10 @@ class ConnectionTest extends \Test\TestCase { } public function testInsertIfNotExistNull() { + if (\OC::$server->getConfig()->getSystemValue('dbtype', 'sqlite') === 'oci') { + self::markTestSkipped('Insert if not exist does not work with clob on oracle'); + } + $this->makeTestTable(); $categoryEntries = [ ['addressbookid' => 123, 'fullname' => null, 'expectedResult' => 1], @@ -255,6 +263,10 @@ class ConnectionTest extends \Test\TestCase { } public function testInsertIfNotExistDonTOverwrite() { + if (\OC::$server->getConfig()->getSystemValue('dbtype', 'sqlite') === 'oci') { + self::markTestSkipped('Insert if not exist does not work with clob on oracle'); + } + $this->makeTestTable(); $fullName = 'fullname test'; $uri = 'uri_1'; @@ -291,6 +303,10 @@ class ConnectionTest extends \Test\TestCase { } public function testInsertIfNotExistsViolating() { + if (\OC::$server->getConfig()->getSystemValue('dbtype', 'sqlite') === 'oci') { + self::markTestSkipped('Insert if not exist does not work with clob on oracle'); + } + $this->makeTestTable(); $result = $this->connection->insertIfNotExist('*PREFIX*table', [ @@ -321,6 +337,10 @@ class ConnectionTest extends \Test\TestCase { * @param array $compareKeys */ public function testInsertIfNotExistsViolatingUnique($compareKeys) { + if (\OC::$server->getConfig()->getSystemValue('dbtype', 'sqlite') === 'oci') { + self::markTestSkipped('Insert if not exist does not work with clob on oracle'); + } + $this->makeTestTable(); $result = $this->connection->insertIfNotExist('*PREFIX*table', [ From 39131519b2f425e50f929eedb6fe81c5991ed1b9 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 6 Nov 2020 14:12:40 +0100 Subject: [PATCH 05/17] Make the test pass on repeating calls Signed-off-by: Joas Schilling --- apps/oauth2/tests/Db/ClientMapperTest.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/oauth2/tests/Db/ClientMapperTest.php b/apps/oauth2/tests/Db/ClientMapperTest.php index ad1612da6b..e4a71fc104 100644 --- a/apps/oauth2/tests/Db/ClientMapperTest.php +++ b/apps/oauth2/tests/Db/ClientMapperTest.php @@ -40,6 +40,13 @@ class ClientMapperTest extends TestCase { $this->clientMapper = new ClientMapper(\OC::$server->getDatabaseConnection()); } + protected function tearDown(): void { + $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query->delete('oauth2_clients')->execute(); + + parent::tearDown(); + } + public function testGetByIdentifier() { $client = new Client(); $client->setClientIdentifier('MyAwesomeClientIdentifier'); @@ -51,7 +58,6 @@ class ClientMapperTest extends TestCase { $this->assertEquals($client, $this->clientMapper->getByIdentifier('MyAwesomeClientIdentifier')); } - public function testGetByIdentifierNotExisting() { $this->expectException(\OCA\OAuth2\Exceptions\ClientNotFoundException::class); @@ -69,7 +75,6 @@ class ClientMapperTest extends TestCase { $this->assertEquals($client, $this->clientMapper->getByUid($client->getId())); } - public function testGetByUidNotExisting() { $this->expectException(\OCA\OAuth2\Exceptions\ClientNotFoundException::class); From 609c8a9cf85f404ff3d8ed9db23a1cf5852047b4 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 6 Nov 2020 14:13:11 +0100 Subject: [PATCH 06/17] Fix "Invalid fetch style: 12" on Oracle Signed-off-by: Joas Schilling --- apps/dav/tests/unit/DAV/CustomPropertiesBackendTest.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/dav/tests/unit/DAV/CustomPropertiesBackendTest.php b/apps/dav/tests/unit/DAV/CustomPropertiesBackendTest.php index 2633ccf544..86a88b3ccd 100644 --- a/apps/dav/tests/unit/DAV/CustomPropertiesBackendTest.php +++ b/apps/dav/tests/unit/DAV/CustomPropertiesBackendTest.php @@ -110,10 +110,13 @@ class CustomPropertiesBackendTest extends TestCase { ->where($query->expr()->eq('userid', $query->createNamedParameter($user))) ->where($query->expr()->eq('propertypath', $query->createNamedParameter($this->formatPath($path)))); - $result = $query->execute(); - $data = $result->fetchAll(\PDO::FETCH_KEY_PAIR); + $data = []; + while ($row = $result->fetch()) { + $data[$row['propertyname']] = $row['propertyvalue']; + } $result->closeCursor(); + return $data; } From ecbb066dec9d08c371ffd85f3bd387357b1000e7 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 9 Nov 2020 10:38:47 +0100 Subject: [PATCH 07/17] Make sure columns with an empty default are nullable for Oracle Signed-off-by: Joas Schilling --- apps/dav/appinfo/info.xml | 2 +- .../composer/composer/autoload_classmap.php | 1 + .../dav/composer/composer/autoload_static.php | 1 + .../Version1012Date20190808122342.php | 2 +- .../Version1016Date20201109085907.php | 60 +++++++++++++++++ .../Version13000Date20170718121200.php | 2 +- .../Version14000Date20180710092004.php | 1 + .../Version15000Date20180926101451.php | 4 +- .../Version15000Date20181015062942.php | 2 +- .../Version16000Date20190207141427.php | 2 +- .../Version16000Date20190428150708.php | 2 +- .../Version20000Date20201109081915.php | 64 +++++++++++++++++++ lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + version.php | 2 +- 15 files changed, 138 insertions(+), 9 deletions(-) create mode 100644 apps/dav/lib/Migration/Version1016Date20201109085907.php create mode 100644 core/Migrations/Version20000Date20201109081915.php diff --git a/apps/dav/appinfo/info.xml b/apps/dav/appinfo/info.xml index 53825f5507..d9a5761230 100644 --- a/apps/dav/appinfo/info.xml +++ b/apps/dav/appinfo/info.xml @@ -5,7 +5,7 @@ WebDAV WebDAV endpoint WebDAV endpoint - 1.17.0 + 1.17.1 agpl owncloud.org DAV diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index aaa5e82da0..e0684f0c79 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -230,6 +230,7 @@ return array( 'OCA\\DAV\\Migration\\Version1011Date20190725113607' => $baseDir . '/../lib/Migration/Version1011Date20190725113607.php', 'OCA\\DAV\\Migration\\Version1011Date20190806104428' => $baseDir . '/../lib/Migration/Version1011Date20190806104428.php', 'OCA\\DAV\\Migration\\Version1012Date20190808122342' => $baseDir . '/../lib/Migration/Version1012Date20190808122342.php', + 'OCA\\DAV\\Migration\\Version1016Date20201109085907' => $baseDir . '/../lib/Migration/Version1016Date20201109085907.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningNode.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php', 'OCA\\DAV\\RootCollection' => $baseDir . '/../lib/RootCollection.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index 0dc34f5a31..32b5e03a3f 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -245,6 +245,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\Version1011Date20190725113607' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190725113607.php', 'OCA\\DAV\\Migration\\Version1011Date20190806104428' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190806104428.php', 'OCA\\DAV\\Migration\\Version1012Date20190808122342' => __DIR__ . '/..' . '/../lib/Migration/Version1012Date20190808122342.php', + 'OCA\\DAV\\Migration\\Version1016Date20201109085907' => __DIR__ . '/..' . '/../lib/Migration/Version1016Date20201109085907.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningNode.php', 'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php', 'OCA\\DAV\\RootCollection' => __DIR__ . '/..' . '/../lib/RootCollection.php', diff --git a/apps/dav/lib/Migration/Version1012Date20190808122342.php b/apps/dav/lib/Migration/Version1012Date20190808122342.php index 7aa51a224e..51929d1a2f 100644 --- a/apps/dav/lib/Migration/Version1012Date20190808122342.php +++ b/apps/dav/lib/Migration/Version1012Date20190808122342.php @@ -69,7 +69,7 @@ class Version1012Date20190808122342 extends SimpleMigrationStep { 'length' => 11, ]); $table->addColumn('is_recurring', Types::SMALLINT, [ - 'notnull' => true, + 'notnull' => false, 'length' => 1, ]); $table->addColumn('uid', Types::STRING, [ diff --git a/apps/dav/lib/Migration/Version1016Date20201109085907.php b/apps/dav/lib/Migration/Version1016Date20201109085907.php new file mode 100644 index 0000000000..a43c8ae06a --- /dev/null +++ b/apps/dav/lib/Migration/Version1016Date20201109085907.php @@ -0,0 +1,60 @@ + + * + * @author Joas Schilling + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\DAV\Migration; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version1016Date20201109085907 extends SimpleMigrationStep { + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + $result = $this->ensureColumnIsNullable($schema, 'calendar_reminders', 'is_recurring'); + + return $result ? $schema : null; + } + + protected function ensureColumnIsNullable(ISchemaWrapper $schema, string $tableName, string $columnName): bool { + $table = $schema->getTable($tableName); + $column = $table->getColumn($columnName); + + if ($column->getNotnull()) { + $column->setNotnull(false); + return true; + } + + return false; + } +} diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index d86477b5ab..2627e50bdb 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -814,7 +814,7 @@ class Version13000Date20170718121200 extends SimpleMigrationStep { if (!$schema->hasTable('credentials')) { $table = $schema->createTable('credentials'); $table->addColumn('user', 'string', [ - 'notnull' => true, + 'notnull' => false, 'length' => 64, ]); $table->addColumn('identifier', 'string', [ diff --git a/core/Migrations/Version14000Date20180710092004.php b/core/Migrations/Version14000Date20180710092004.php index 141b0c1afb..767d8ee5f7 100644 --- a/core/Migrations/Version14000Date20180710092004.php +++ b/core/Migrations/Version14000Date20180710092004.php @@ -43,6 +43,7 @@ class Version14000Date20180710092004 extends SimpleMigrationStep { if (!$table->hasColumn('password_by_talk')) { $table->addColumn('password_by_talk', Types::BOOLEAN, [ 'default' => 0, + 'notnull' => false, ]); } diff --git a/core/Migrations/Version15000Date20180926101451.php b/core/Migrations/Version15000Date20180926101451.php index a5f37cc912..e03a11318c 100644 --- a/core/Migrations/Version15000Date20180926101451.php +++ b/core/Migrations/Version15000Date20180926101451.php @@ -45,8 +45,8 @@ class Version15000Date20180926101451 extends SimpleMigrationStep { $table = $schema->getTable('authtoken'); $table->addColumn('password_invalid','boolean', [ - 'notnull' => true, - 'default' => false, + 'default' => 0, + 'notnull' => false, ]); return $schema; diff --git a/core/Migrations/Version15000Date20181015062942.php b/core/Migrations/Version15000Date20181015062942.php index eaeaf0dd26..23c2a90474 100644 --- a/core/Migrations/Version15000Date20181015062942.php +++ b/core/Migrations/Version15000Date20181015062942.php @@ -45,7 +45,7 @@ class Version15000Date20181015062942 extends SimpleMigrationStep { $table = $schema->getTable('share'); $table->addColumn('hide_download', 'smallint', [ - 'notnull' => true, + 'notnull' => false, 'length' => 1, 'default' => 0, ]); diff --git a/core/Migrations/Version16000Date20190207141427.php b/core/Migrations/Version16000Date20190207141427.php index e6235efe20..63c6c871ea 100644 --- a/core/Migrations/Version16000Date20190207141427.php +++ b/core/Migrations/Version16000Date20190207141427.php @@ -102,7 +102,7 @@ class Version16000Date20190207141427 extends SimpleMigrationStep { 'default' => '', ]); $table->addColumn('access', Types::SMALLINT, [ - 'notnull' => true, + 'notnull' => false, 'default' => 0, ]); diff --git a/core/Migrations/Version16000Date20190428150708.php b/core/Migrations/Version16000Date20190428150708.php index 546d4c19e1..dc33b08035 100644 --- a/core/Migrations/Version16000Date20190428150708.php +++ b/core/Migrations/Version16000Date20190428150708.php @@ -49,7 +49,7 @@ class Version16000Date20190428150708 extends SimpleMigrationStep { if ($schema->hasTable('collres_accesscache')) { $table = $schema->getTable('collres_accesscache'); $table->addColumn('access', Types::BOOLEAN, [ - 'notnull' => true, + 'notnull' => false, 'default' => false ]); } diff --git a/core/Migrations/Version20000Date20201109081915.php b/core/Migrations/Version20000Date20201109081915.php new file mode 100644 index 0000000000..720f54d8e8 --- /dev/null +++ b/core/Migrations/Version20000Date20201109081915.php @@ -0,0 +1,64 @@ + + * + * @author Joas Schilling + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version20000Date20201109081915 extends SimpleMigrationStep { + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + $result = $this->ensureColumnIsNullable($schema, 'share', 'password_by_talk'); + $result = $this->ensureColumnIsNullable($schema, 'share', 'hide_download') || $result; + $result = $this->ensureColumnIsNullable($schema, 'credentials', 'user') || $result; + $result = $this->ensureColumnIsNullable($schema, 'authtoken', 'password_invalid') || $result; + $result = $this->ensureColumnIsNullable($schema, 'collres_accesscache', 'access') || $result; + + return $result ? $schema : null; + } + + protected function ensureColumnIsNullable(ISchemaWrapper $schema, string $tableName, string $columnName): bool { + $table = $schema->getTable($tableName); + $column = $table->getColumn($columnName); + + if ($column->getNotnull()) { + $column->setNotnull(false); + return true; + } + + return false; + } +} diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 5c94e2f39f..bfa087bf5f 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -922,6 +922,7 @@ return array( 'OC\\Core\\Migrations\\Version18000Date20191014105105' => $baseDir . '/core/Migrations/Version18000Date20191014105105.php', 'OC\\Core\\Migrations\\Version18000Date20191204114856' => $baseDir . '/core/Migrations/Version18000Date20191204114856.php', 'OC\\Core\\Migrations\\Version19000Date20200211083441' => $baseDir . '/core/Migrations/Version19000Date20200211083441.php', + 'OC\\Core\\Migrations\\Version20000Date20201109081915' => $baseDir . '/core/Migrations/Version20000Date20201109081915.php', 'OC\\Core\\Notification\\RemoveLinkSharesNotifier' => $baseDir . '/core/Notification/RemoveLinkSharesNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => $baseDir . '/lib/private/DB/Adapter.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 48c311bfbb..82143b4b04 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -951,6 +951,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Core\\Migrations\\Version18000Date20191014105105' => __DIR__ . '/../../..' . '/core/Migrations/Version18000Date20191014105105.php', 'OC\\Core\\Migrations\\Version18000Date20191204114856' => __DIR__ . '/../../..' . '/core/Migrations/Version18000Date20191204114856.php', 'OC\\Core\\Migrations\\Version19000Date20200211083441' => __DIR__ . '/../../..' . '/core/Migrations/Version19000Date20200211083441.php', + 'OC\\Core\\Migrations\\Version20000Date20201109081915' => __DIR__ . '/../../..' . '/core/Migrations/Version20000Date20201109081915.php', 'OC\\Core\\Notification\\RemoveLinkSharesNotifier' => __DIR__ . '/../../..' . '/core/Notification/RemoveLinkSharesNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => __DIR__ . '/../../..' . '/lib/private/DB/Adapter.php', diff --git a/version.php b/version.php index 280a04eec0..efe744daba 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version = [21, 0, 0, 2]; +$OC_Version = [21, 0, 0, 3]; // The human readable string $OC_VersionString = '21.0.0 alpha'; From 31e243be744aa726178a5ecb15649e7675a049d4 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 9 Nov 2020 10:40:55 +0100 Subject: [PATCH 08/17] Fix public calendars as they are stored with null on oracle Signed-off-by: Joas Schilling --- apps/dav/lib/CalDAV/CalDavBackend.php | 46 ++++++++++++++++++++------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/apps/dav/lib/CalDAV/CalDavBackend.php b/apps/dav/lib/CalDAV/CalDavBackend.php index 4ff51a5f52..84294971b4 100644 --- a/apps/dav/lib/CalDAV/CalDavBackend.php +++ b/apps/dav/lib/CalDAV/CalDavBackend.php @@ -242,8 +242,13 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription $principalUri = $this->convertPrincipal($principalUri, true); $query = $this->db->getQueryBuilder(); $query->select($query->func()->count('*')) - ->from('calendars') - ->where($query->expr()->eq('principaluri', $query->createNamedParameter($principalUri))); + ->from('calendars'); + + if ($principalUri === '') { + $query->where($query->expr()->emptyString('principaluri')); + } else { + $query->where($query->expr()->eq('principaluri', $query->createNamedParameter($principalUri))); + } if ($excludeBirthday) { $query->andWhere($query->expr()->neq('uri', $query->createNamedParameter(BirthdayService::BIRTHDAY_CALENDAR_URI))); @@ -293,13 +298,21 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription // Making fields a comma-delimited list $query = $this->db->getQueryBuilder(); - $query->select($fields)->from('calendars') - ->where($query->expr()->eq('principaluri', $query->createNamedParameter($principalUri))) - ->orderBy('calendarorder', 'ASC'); - $stmt = $query->execute(); + $query->select($fields) + ->from('calendars') + ->orderBy('calendarorder', 'ASC'); + + if ($principalUri === '') { + $query->where($query->expr()->emptyString('principaluri')); + } else { + $query->where($query->expr()->eq('principaluri', $query->createNamedParameter($principalUri))); + } + + $result = $query->execute(); $calendars = []; - while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) { + while ($row = $result->fetch()) { + $row['principaluri'] = (string) $row['principaluri']; $components = []; if ($row['components']) { $components = explode(',',$row['components']); @@ -326,8 +339,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription $calendars[$calendar['id']] = $calendar; } } - - $stmt->closeCursor(); + $result->closeCursor(); // query for shared calendars $principals = $this->principalBackend->getGroupMembership($principalUriOriginal, true); @@ -347,17 +359,19 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription $fields[] = 'a.transparent'; $fields[] = 's.access'; $query = $this->db->getQueryBuilder(); - $result = $query->select($fields) + $query->select($fields) ->from('dav_shares', 's') ->join('s', 'calendars', 'a', $query->expr()->eq('s.resourceid', 'a.id')) ->where($query->expr()->in('s.principaluri', $query->createParameter('principaluri'))) ->andWhere($query->expr()->eq('s.type', $query->createParameter('type'))) ->setParameter('type', 'calendar') - ->setParameter('principaluri', $principals, \Doctrine\DBAL\Connection::PARAM_STR_ARRAY) - ->execute(); + ->setParameter('principaluri', $principals, \Doctrine\DBAL\Connection::PARAM_STR_ARRAY); + + $result = $query->execute(); $readOnlyPropertyName = '{' . \OCA\DAV\DAV\Sharing\Plugin::NS_OWNCLOUD . '}read-only'; while ($row = $result->fetch()) { + $row['principaluri'] = (string) $row['principaluri']; if ($row['principaluri'] === $principalUri) { continue; } @@ -428,6 +442,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription $stmt = $query->execute(); $calendars = []; while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) { + $row['principaluri'] = (string) $row['principaluri']; $components = []; if ($row['components']) { $components = explode(',',$row['components']); @@ -497,6 +512,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription ->execute(); while ($row = $result->fetch()) { + $row['principaluri'] = (string) $row['principaluri']; list(, $name) = Uri\split($row['principaluri']); $row['displayname'] = $row['displayname'] . "($name)"; $components = []; @@ -563,6 +579,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription throw new NotFound('Node with name \'' . $uri . '\' could not be found'); } + $row['principaluri'] = (string) $row['principaluri']; list(, $name) = Uri\split($row['principaluri']); $row['displayname'] = $row['displayname'] . ' ' . "($name)"; $components = []; @@ -619,6 +636,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription return null; } + $row['principaluri'] = (string) $row['principaluri']; $components = []; if ($row['components']) { $components = explode(',',$row['components']); @@ -669,6 +687,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription return null; } + $row['principaluri'] = (string) $row['principaluri']; $components = []; if ($row['components']) { $components = explode(',',$row['components']); @@ -718,6 +737,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription return null; } + $row['principaluri'] = (string) $row['principaluri']; $subscription = [ 'id' => $row['id'], 'uri' => $row['uri'], @@ -965,6 +985,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription 'classification' => (int)$row['classification'] ]; } + $stmt->closeCursor(); return $result; } @@ -995,6 +1016,7 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription ->andWhere($query->expr()->eq('calendartype', $query->createNamedParameter($calendarType))); $stmt = $query->execute(); $row = $stmt->fetch(\PDO::FETCH_ASSOC); + $stmt->closeCursor(); if (!$row) { return null; From cffad62771a1640675e92fdc6a85ff013bac2ec4 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 9 Nov 2020 16:26:09 +0100 Subject: [PATCH 09/17] Empty string is returned as null, but empty string in file cache is the root and exists Signed-off-by: Joas Schilling --- lib/private/Files/Cache/Cache.php | 6 +++++- lib/private/Share20/DefaultShareProvider.php | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index e74cc86d9c..1d3b4da3bc 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -998,7 +998,11 @@ class Cache implements ICache { $path = $result->fetchColumn(); $result->closeCursor(); - return $path === false ? null : $path; + if ($path === false) { + return null; + } + + return (string) $path; } /** diff --git a/lib/private/Share20/DefaultShareProvider.php b/lib/private/Share20/DefaultShareProvider.php index e43529086f..b4ac3b3b84 100644 --- a/lib/private/Share20/DefaultShareProvider.php +++ b/lib/private/Share20/DefaultShareProvider.php @@ -874,6 +874,11 @@ class DefaultShareProvider implements IShareProvider { $cursor = $qb->execute(); while ($data = $cursor->fetch()) { + if ($data['fileid'] && $data['path'] === null) { + $data['path'] = (string) $data['path']; + $data['name'] = (string) $data['name']; + $data['checksum'] = (string) $data['checksum']; + } if ($this->isAccessibleResult($data)) { $shares[] = $this->createShare($data); } @@ -1004,7 +1009,7 @@ class DefaultShareProvider implements IShareProvider { ->setShareType((int)$data['share_type']) ->setPermissions((int)$data['permissions']) ->setTarget($data['file_target']) - ->setNote($data['note']) + ->setNote((string)$data['note']) ->setMailSend((bool)$data['mail_send']) ->setStatus((int)$data['accepted']) ->setLabel($data['label']); From 1aa9c9164d73be7aabb5a2ff18e8999c55e33a48 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 9 Nov 2020 17:33:05 +0100 Subject: [PATCH 10/17] Fix comparing the empty string for global credentials Signed-off-by: Joas Schilling --- lib/private/DB/Connection.php | 16 +++++++++++----- lib/private/Security/CredentialsManager.php | 21 +++++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index 6835226357..0ea7163794 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -332,11 +332,17 @@ class Connection extends ReconnectWrapper implements IDBConnection { $where = $updateQb->expr()->andX(); $whereValues = array_merge($keys, $updatePreconditionValues); foreach ($whereValues as $name => $value) { - $where->add($updateQb->expr()->eq( - $name, - $updateQb->createNamedParameter($value, $this->getType($value)), - $this->getType($value) - )); + if ($value === '') { + $where->add($updateQb->expr()->emptyString( + $name + )); + } else { + $where->add($updateQb->expr()->eq( + $name, + $updateQb->createNamedParameter($value, $this->getType($value)), + $this->getType($value) + )); + } } $updateQb->where($where); $affected = $updateQb->execute(); diff --git a/lib/private/Security/CredentialsManager.php b/lib/private/Security/CredentialsManager.php index a40a7e1d88..20af25ae10 100644 --- a/lib/private/Security/CredentialsManager.php +++ b/lib/private/Security/CredentialsManager.php @@ -81,9 +81,13 @@ class CredentialsManager implements ICredentialsManager { $qb = $this->dbConnection->getQueryBuilder(); $qb->select('credentials') ->from(self::DB_TABLE) - ->where($qb->expr()->eq('user', $qb->createNamedParameter((string)$userId))) - ->andWhere($qb->expr()->eq('identifier', $qb->createNamedParameter($identifier))) - ; + ->where($qb->expr()->eq('identifier', $qb->createNamedParameter($identifier))); + + if ($userId === '') { + $qb->andWhere($qb->expr()->emptyString('user')); + } else { + $qb->andWhere($qb->expr()->eq('user', $qb->createNamedParameter((string)$userId))); + } $qResult = $qb->execute(); $result = $qResult->fetch(); @@ -107,9 +111,14 @@ class CredentialsManager implements ICredentialsManager { public function delete($userId, $identifier) { $qb = $this->dbConnection->getQueryBuilder(); $qb->delete(self::DB_TABLE) - ->where($qb->expr()->eq('user', $qb->createNamedParameter((string)$userId))) - ->andWhere($qb->expr()->eq('identifier', $qb->createNamedParameter($identifier))) - ; + ->where($qb->expr()->eq('identifier', $qb->createNamedParameter($identifier))); + + if ($userId === '') { + $qb->andWhere($qb->expr()->emptyString('user')); + } else { + $qb->andWhere($qb->expr()->eq('user', $qb->createNamedParameter((string)$userId))); + } + return $qb->execute(); } From fbda2d1d253b99a4e83970e023ede42faad87966 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 9 Nov 2020 17:33:32 +0100 Subject: [PATCH 11/17] Don't try to update on NotNullConstraintViolationException, only on unique or foreign key Signed-off-by: Joas Schilling --- lib/private/DB/Connection.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/private/DB/Connection.php b/lib/private/DB/Connection.php index 0ea7163794..ebb3900f3e 100644 --- a/lib/private/DB/Connection.php +++ b/lib/private/DB/Connection.php @@ -39,6 +39,7 @@ use Doctrine\DBAL\Configuration; use Doctrine\DBAL\DBALException; use Doctrine\DBAL\Driver; use Doctrine\DBAL\Exception\ConstraintViolationException; +use Doctrine\DBAL\Exception\NotNullConstraintViolationException; use Doctrine\DBAL\Platforms\MySqlPlatform; use Doctrine\DBAL\Schema\Schema; use OC\DB\QueryBuilder\QueryBuilder; @@ -322,6 +323,8 @@ class Connection extends ReconnectWrapper implements IDBConnection { }, array_merge($keys, $values)) ); return $insertQb->execute(); + } catch (NotNullConstraintViolationException $e) { + throw $e; } catch (ConstraintViolationException $e) { // value already exists, try update $updateQb = $this->getQueryBuilder(); From 5b5aebbf66e7559aac73eb82b236b052b6a1ae3e Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Nov 2020 09:33:29 +0100 Subject: [PATCH 12/17] Replace the credentials table with one that can have empty user Primary key columns on Oracle can not have empty strings Signed-off-by: Joas Schilling --- .../Version13000Date20170718121200.php | 32 +++--- .../Version20000Date20201109081915.php | 2 +- .../Version20000Date20201109081918.php | 107 ++++++++++++++++++ .../Version20000Date20201109081919.php | 52 +++++++++ lib/composer/composer/autoload_classmap.php | 2 + lib/composer/composer/autoload_static.php | 2 + lib/private/Security/CredentialsManager.php | 2 +- version.php | 2 +- 8 files changed, 182 insertions(+), 19 deletions(-) create mode 100644 core/Migrations/Version20000Date20201109081918.php create mode 100644 core/Migrations/Version20000Date20201109081919.php diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index 2627e50bdb..ba5f72588a 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -811,22 +811,22 @@ class Version13000Date20170718121200 extends SimpleMigrationStep { $table->addUniqueIndex(['user_id', 'object_type', 'object_id'], 'comments_marker_index'); } - if (!$schema->hasTable('credentials')) { - $table = $schema->createTable('credentials'); - $table->addColumn('user', 'string', [ - 'notnull' => false, - 'length' => 64, - ]); - $table->addColumn('identifier', 'string', [ - 'notnull' => true, - 'length' => 64, - ]); - $table->addColumn('credentials', 'text', [ - 'notnull' => false, - ]); - $table->setPrimaryKey(['user', 'identifier']); - $table->addIndex(['user'], 'credentials_user'); - } +// if (!$schema->hasTable('credentials')) { +// $table = $schema->createTable('credentials'); +// $table->addColumn('user', 'string', [ +// 'notnull' => false, +// 'length' => 64, +// ]); +// $table->addColumn('identifier', 'string', [ +// 'notnull' => true, +// 'length' => 64, +// ]); +// $table->addColumn('credentials', 'text', [ +// 'notnull' => false, +// ]); +// $table->setPrimaryKey(['user', 'identifier']); +// $table->addIndex(['user'], 'credentials_user'); +// } if (!$schema->hasTable('admin_sections')) { $table = $schema->createTable('admin_sections'); diff --git a/core/Migrations/Version20000Date20201109081915.php b/core/Migrations/Version20000Date20201109081915.php index 720f54d8e8..d19c9ca0bb 100644 --- a/core/Migrations/Version20000Date20201109081915.php +++ b/core/Migrations/Version20000Date20201109081915.php @@ -43,7 +43,7 @@ class Version20000Date20201109081915 extends SimpleMigrationStep { $result = $this->ensureColumnIsNullable($schema, 'share', 'password_by_talk'); $result = $this->ensureColumnIsNullable($schema, 'share', 'hide_download') || $result; - $result = $this->ensureColumnIsNullable($schema, 'credentials', 'user') || $result; +// $result = $this->ensureColumnIsNullable($schema, 'credentials', 'user') || $result; $result = $this->ensureColumnIsNullable($schema, 'authtoken', 'password_invalid') || $result; $result = $this->ensureColumnIsNullable($schema, 'collres_accesscache', 'access') || $result; diff --git a/core/Migrations/Version20000Date20201109081918.php b/core/Migrations/Version20000Date20201109081918.php new file mode 100644 index 0000000000..8e685b200a --- /dev/null +++ b/core/Migrations/Version20000Date20201109081918.php @@ -0,0 +1,107 @@ + + * + * @author Joas Schilling + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use Doctrine\DBAL\Types\Type; +use OCP\DB\ISchemaWrapper; +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version20000Date20201109081918 extends SimpleMigrationStep { + + /** @var IDBConnection */ + protected $connection; + + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + $table = $schema->createTable('storages_credentials'); + $table->addColumn('id', Type::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('user', Type::STRING, [ + 'notnull' => false, + 'length' => 64, + ]); + $table->addColumn('identifier', Type::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('credentials', Type::TEXT, [ + 'notnull' => false, + ]); + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['user', 'identifier'], 'stocred_ui'); + $table->addIndex(['user'], 'stocred_user'); + + return $schema; + } + + /** + * {@inheritDoc} + * + * @since 13.0.0 + */ + public function postSchemaChange(IOutput $output, \Closure $schemaClosure, array $options): void { + if (!$this->connection->tableExists('credentials')) { + return; + } + + $query = $this->connection->getQueryBuilder(); + $query->select('*') + ->from('credentials'); + + $insert = $this->connection->getQueryBuilder(); + $insert->insert('storages_credentials') + ->setValue('user', $insert->createNamedParameter('user')) + ->setValue('identifier', $insert->createNamedParameter('identifier')) + ->setValue('credentials', $insert->createNamedParameter('credentials')); + + $result = $query->execute(); + while ($row = $result->fetch()) { + $insert->setParameter('user', (string) $row['user']) + ->setParameter('identifier', (string) $row['identifier']) + ->setParameter('credentials', (string) $row['credentials']); + $insert->execute(); + } + $result->closeCursor(); + } +} diff --git a/core/Migrations/Version20000Date20201109081919.php b/core/Migrations/Version20000Date20201109081919.php new file mode 100644 index 0000000000..bfc7080ab4 --- /dev/null +++ b/core/Migrations/Version20000Date20201109081919.php @@ -0,0 +1,52 @@ + + * + * @author Joas Schilling + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version20000Date20201109081919 extends SimpleMigrationStep { + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if ($schema->hasTable('credentials')) { + $schema->dropTable('credentials'); + return $schema; + } + + return null; + } +} diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index bfa087bf5f..16f3b4c9b0 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -923,6 +923,8 @@ return array( 'OC\\Core\\Migrations\\Version18000Date20191204114856' => $baseDir . '/core/Migrations/Version18000Date20191204114856.php', 'OC\\Core\\Migrations\\Version19000Date20200211083441' => $baseDir . '/core/Migrations/Version19000Date20200211083441.php', 'OC\\Core\\Migrations\\Version20000Date20201109081915' => $baseDir . '/core/Migrations/Version20000Date20201109081915.php', + 'OC\\Core\\Migrations\\Version20000Date20201109081918' => $baseDir . '/core/Migrations/Version20000Date20201109081918.php', + 'OC\\Core\\Migrations\\Version20000Date20201109081919' => $baseDir . '/core/Migrations/Version20000Date20201109081919.php', 'OC\\Core\\Notification\\RemoveLinkSharesNotifier' => $baseDir . '/core/Notification/RemoveLinkSharesNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => $baseDir . '/lib/private/DB/Adapter.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 82143b4b04..1b8e31bba3 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -952,6 +952,8 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Core\\Migrations\\Version18000Date20191204114856' => __DIR__ . '/../../..' . '/core/Migrations/Version18000Date20191204114856.php', 'OC\\Core\\Migrations\\Version19000Date20200211083441' => __DIR__ . '/../../..' . '/core/Migrations/Version19000Date20200211083441.php', 'OC\\Core\\Migrations\\Version20000Date20201109081915' => __DIR__ . '/../../..' . '/core/Migrations/Version20000Date20201109081915.php', + 'OC\\Core\\Migrations\\Version20000Date20201109081918' => __DIR__ . '/../../..' . '/core/Migrations/Version20000Date20201109081918.php', + 'OC\\Core\\Migrations\\Version20000Date20201109081919' => __DIR__ . '/../../..' . '/core/Migrations/Version20000Date20201109081919.php', 'OC\\Core\\Notification\\RemoveLinkSharesNotifier' => __DIR__ . '/../../..' . '/core/Notification/RemoveLinkSharesNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => __DIR__ . '/../../..' . '/lib/private/DB/Adapter.php', diff --git a/lib/private/Security/CredentialsManager.php b/lib/private/Security/CredentialsManager.php index 20af25ae10..2a480ef710 100644 --- a/lib/private/Security/CredentialsManager.php +++ b/lib/private/Security/CredentialsManager.php @@ -35,7 +35,7 @@ use OCP\Security\ICrypto; * @package OC\Security */ class CredentialsManager implements ICredentialsManager { - public const DB_TABLE = 'credentials'; + public const DB_TABLE = 'storages_credentials'; /** @var ICrypto */ protected $crypto; diff --git a/version.php b/version.php index efe744daba..dc48f4bc52 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version = [21, 0, 0, 3]; +$OC_Version = [21, 0, 0, 5]; // The human readable string $OC_VersionString = '21.0.0 alpha'; From d5df033ede1f2b19b87c365b1f808130023edf17 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Nov 2020 09:34:57 +0100 Subject: [PATCH 13/17] Create primary keys on all tables and add a command to create the afterwards Signed-off-by: Joas Schilling --- .../Version1010Date20200630191755.php | 3 +- .../lib/Controller/CheckSetupController.php | 11 ++ .../Controller/CheckSetupControllerTest.php | 5 + core/Application.php | 58 ++++++ core/Command/Db/AddMissingPrimaryKeys.php | 181 ++++++++++++++++++ .../Version13000Date20170718121200.php | 6 +- .../Version16000Date20190207141427.php | 6 +- .../Version17000Date20190514105811.php | 3 +- .../Version18000Date20191204114856.php | 1 - core/js/setupchecks.js | 15 ++ core/js/tests/specs/setupchecksSpec.js | 16 ++ lib/composer/composer/autoload_classmap.php | 2 + lib/composer/composer/autoload_static.php | 2 + .../DB/MissingPrimaryKeyInformation.php | 42 ++++ lib/public/IDBConnection.php | 2 + version.php | 2 +- 16 files changed, 347 insertions(+), 8 deletions(-) create mode 100644 core/Command/Db/AddMissingPrimaryKeys.php create mode 100644 lib/private/DB/MissingPrimaryKeyInformation.php diff --git a/apps/federatedfilesharing/lib/Migration/Version1010Date20200630191755.php b/apps/federatedfilesharing/lib/Migration/Version1010Date20200630191755.php index 53348d8f86..bf0d0f8eec 100644 --- a/apps/federatedfilesharing/lib/Migration/Version1010Date20200630191755.php +++ b/apps/federatedfilesharing/lib/Migration/Version1010Date20200630191755.php @@ -52,7 +52,8 @@ class Version1010Date20200630191755 extends SimpleMigrationStep { 'notnull' => true, 'length' => 4, ]); - $table->addUniqueIndex(['share_id'], 'share_id_index'); + $table->setPrimaryKey(['share_id'], 'federated_res_pk'); +// $table->addUniqueIndex(['share_id'], 'share_id_index'); } return $schema; } diff --git a/apps/settings/lib/Controller/CheckSetupController.php b/apps/settings/lib/Controller/CheckSetupController.php index 0f9dd84feb..6984b0f5ae 100644 --- a/apps/settings/lib/Controller/CheckSetupController.php +++ b/apps/settings/lib/Controller/CheckSetupController.php @@ -49,6 +49,7 @@ use OC\AppFramework\Http; use OC\DB\Connection; use OC\DB\MissingColumnInformation; use OC\DB\MissingIndexInformation; +use OC\DB\MissingPrimaryKeyInformation; use OC\DB\SchemaWrapper; use OC\IntegrityCheck\Checker; use OC\Lock\NoopLockingProvider; @@ -452,6 +453,15 @@ Raw output return $indexInfo->getListOfMissingIndexes(); } + protected function hasMissingPrimaryKeys(): array { + $info = new MissingPrimaryKeyInformation(); + // Dispatch event so apps can also hint for pending index updates if needed + $event = new GenericEvent($info); + $this->dispatcher->dispatch(IDBConnection::CHECK_MISSING_PRIMARY_KEYS_EVENT, $event); + + return $info->getListOfMissingPrimaryKeys(); + } + protected function hasMissingColumns(): array { $indexInfo = new MissingColumnInformation(); // Dispatch event so apps can also hint for pending index updates if needed @@ -722,6 +732,7 @@ Raw output 'phpOpcacheDocumentation' => $this->urlGenerator->linkToDocs('admin-php-opcache'), 'isSettimelimitAvailable' => $this->isSettimelimitAvailable(), 'hasFreeTypeSupport' => $this->hasFreeTypeSupport(), + 'missingPrimaryKeys' => $this->hasMissingPrimaryKeys(), 'missingIndexes' => $this->hasMissingIndexes(), 'missingColumns' => $this->hasMissingColumns(), 'isSqliteUsed' => $this->isSqliteUsed(), diff --git a/apps/settings/tests/Controller/CheckSetupControllerTest.php b/apps/settings/tests/Controller/CheckSetupControllerTest.php index dbc3fc7d0d..43ec984041 100644 --- a/apps/settings/tests/Controller/CheckSetupControllerTest.php +++ b/apps/settings/tests/Controller/CheckSetupControllerTest.php @@ -165,6 +165,7 @@ class CheckSetupControllerTest extends TestCase { 'isOpcacheProperlySetup', 'hasFreeTypeSupport', 'hasMissingIndexes', + 'hasMissingPrimaryKeys', 'isSqliteUsed', 'isPHPMailerUsed', 'hasOpcacheLoaded', @@ -448,6 +449,9 @@ class CheckSetupControllerTest extends TestCase { $this->checkSetupController ->method('hasMissingIndexes') ->willReturn([]); + $this->checkSetupController + ->method('hasMissingPrimaryKeys') + ->willReturn([]); $this->checkSetupController ->method('isSqliteUsed') ->willReturn(false); @@ -587,6 +591,7 @@ class CheckSetupControllerTest extends TestCase { 'isSqliteUsed' => false, 'databaseConversionDocumentation' => 'http://docs.example.org/server/go.php?to=admin-db-conversion', 'missingIndexes' => [], + 'missingPrimaryKeys' => [], 'missingColumns' => [], 'isMemoryLimitSufficient' => true, 'appDirsWithDifferentOwner' => [], diff --git a/core/Application.php b/core/Application.php index a452ca9677..d9d6b92a2a 100644 --- a/core/Application.php +++ b/core/Application.php @@ -42,6 +42,7 @@ use OC\Authentication\Notifications\Notifier as AuthenticationNotifier; use OC\Core\Notification\RemoveLinkSharesNotifier; use OC\DB\MissingColumnInformation; use OC\DB\MissingIndexInformation; +use OC\DB\MissingPrimaryKeyInformation; use OC\DB\SchemaWrapper; use OCP\AppFramework\App; use OCP\EventDispatcher\IEventDispatcher; @@ -181,6 +182,63 @@ class Application extends App { } ); + $oldEventDispatcher->addListener(IDBConnection::CHECK_MISSING_PRIMARY_KEYS_EVENT, + function (GenericEvent $event) use ($container) { + /** @var MissingPrimaryKeyInformation $subject */ + $subject = $event->getSubject(); + + $schema = new SchemaWrapper($container->query(IDBConnection::class)); + + if ($schema->hasTable('federated_reshares')) { + $table = $schema->getTable('federated_reshares'); + + if (!$table->hasPrimaryKey()) { + $subject->addHintForMissingSubject($table->getName()); + } + } + + if ($schema->hasTable('systemtag_object_mapping')) { + $table = $schema->getTable('systemtag_object_mapping'); + + if (!$table->hasPrimaryKey()) { + $subject->addHintForMissingSubject($table->getName()); + } + } + + if ($schema->hasTable('comments_read_markers')) { + $table = $schema->getTable('comments_read_markers'); + + if (!$table->hasPrimaryKey()) { + $subject->addHintForMissingSubject($table->getName()); + } + } + + if ($schema->hasTable('collres_resources')) { + $table = $schema->getTable('collres_resources'); + + if (!$table->hasPrimaryKey()) { + $subject->addHintForMissingSubject($table->getName()); + } + } + + if ($schema->hasTable('collres_accesscache')) { + $table = $schema->getTable('collres_accesscache'); + + if (!$table->hasPrimaryKey()) { + $subject->addHintForMissingSubject($table->getName()); + } + } + + if ($schema->hasTable('filecache_extended')) { + $table = $schema->getTable('filecache_extended'); + + if (!$table->hasPrimaryKey()) { + $subject->addHintForMissingSubject($table->getName()); + } + } + } + ); + $oldEventDispatcher->addListener(IDBConnection::CHECK_MISSING_COLUMNS_EVENT, function (GenericEvent $event) use ($container) { /** @var MissingColumnInformation $subject */ diff --git a/core/Command/Db/AddMissingPrimaryKeys.php b/core/Command/Db/AddMissingPrimaryKeys.php new file mode 100644 index 0000000000..a51249b1b8 --- /dev/null +++ b/core/Command/Db/AddMissingPrimaryKeys.php @@ -0,0 +1,181 @@ + + * + * @author Bjoern Schiessle + * @author Joas Schilling + * @author Mario Danic + * @author Morris Jobke + * @author Robin Appelman + * @author Roeland Jago Douma + * @author Thomas Citharel + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Command\Db; + +use OC\DB\SchemaWrapper; +use OCP\IDBConnection; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Symfony\Component\EventDispatcher\GenericEvent; + +/** + * Class AddMissingPrimaryKeys + * + * if you added primary keys to the database, this is the right place to add + * your update routine for existing instances + * + * @package OC\Core\Command\Db + */ +class AddMissingPrimaryKeys extends Command { + + /** @var IDBConnection */ + private $connection; + + /** @var EventDispatcherInterface */ + private $dispatcher; + + public function __construct(IDBConnection $connection, EventDispatcherInterface $dispatcher) { + parent::__construct(); + + $this->connection = $connection; + $this->dispatcher = $dispatcher; + } + + protected function configure() { + $this + ->setName('db:add-missing-primary-keys') + ->setDescription('Add missing primary keys to the database tables'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $this->addCorePrimaryKeys($output); + + // Dispatch event so apps can also update indexes if needed + $event = new GenericEvent($output); + $this->dispatcher->dispatch(IDBConnection::ADD_MISSING_PRIMARY_KEYS_EVENT, $event); + return 0; + } + + /** + * add missing indices to the share table + * + * @param OutputInterface $output + * @throws \Doctrine\DBAL\Schema\SchemaException + */ + private function addCorePrimaryKeys(OutputInterface $output) { + $output->writeln('Check primary keys.'); + + $schema = new SchemaWrapper($this->connection); + $updated = false; + + if ($schema->hasTable('federated_reshares')) { + $table = $schema->getTable('federated_reshares'); + if (!$table->hasPrimaryKey()) { + $output->writeln('Adding primary key to the federated_reshares table, this can take some time...'); + $table->setPrimaryKey(['share_id'], 'federated_res_pk'); + if ($table->hasIndex('share_id_index')) { + $table->dropIndex('share_id_index'); + } + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('federated_reshares table updated successfully.'); + } + } + + if ($schema->hasTable('systemtag_object_mapping')) { + $table = $schema->getTable('systemtag_object_mapping'); + if (!$table->hasPrimaryKey()) { + $output->writeln('Adding primary key to the systemtag_object_mapping table, this can take some time...'); + $table->setPrimaryKey(['objecttype', 'objectid', 'systemtagid'], 'som_pk'); + if ($table->hasIndex('mapping')) { + $table->dropIndex('mapping'); + } + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('systemtag_object_mapping table updated successfully.'); + } + } + + if ($schema->hasTable('comments_read_markers')) { + $table = $schema->getTable('comments_read_markers'); + if (!$table->hasPrimaryKey()) { + $output->writeln('Adding primary key to the comments_read_markers table, this can take some time...'); + $table->setPrimaryKey(['user_id', 'object_type', 'object_id'], 'crm_pk'); + if ($table->hasIndex('comments_marker_index')) { + $table->dropIndex('comments_marker_index'); + } + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('comments_read_markers table updated successfully.'); + } + } + + if ($schema->hasTable('collres_resources')) { + $table = $schema->getTable('collres_resources'); + if (!$table->hasPrimaryKey()) { + $output->writeln('Adding primary key to the collres_resources table, this can take some time...'); + $table->setPrimaryKey(['collection_id', 'resource_type', 'resource_id'], 'crr_pk'); + if ($table->hasIndex('collres_unique_res')) { + $table->dropIndex('collres_unique_res'); + } + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('collres_resources table updated successfully.'); + } + } + + if ($schema->hasTable('collres_accesscache')) { + $table = $schema->getTable('collres_accesscache'); + if (!$table->hasPrimaryKey()) { + $output->writeln('Adding primary key to the collres_accesscache table, this can take some time...'); + $table->setPrimaryKey(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'cra_pk'); + if ($table->hasIndex('collres_unique_user')) { + $table->dropIndex('collres_unique_user'); + } + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('collres_accesscache table updated successfully.'); + } + } + + if ($schema->hasTable('filecache_extended')) { + $table = $schema->getTable('filecache_extended'); + if (!$table->hasPrimaryKey()) { + $output->writeln('Adding primary key to the filecache_extended table, this can take some time...'); + $table->setPrimaryKey(['fileid'], 'fce_pk'); + if ($table->hasIndex('fce_fileid_idx')) { + $table->dropIndex('fce_fileid_idx'); + } + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('filecache_extended table updated successfully.'); + } + } + + if (!$updated) { + $output->writeln('Done.'); + } + } +} diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index ba5f72588a..2a47bb52cf 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -673,7 +673,8 @@ class Version13000Date20170718121200 extends SimpleMigrationStep { 'default' => 0, 'unsigned' => true, ]); - $table->addUniqueIndex(['objecttype', 'objectid', 'systemtagid'], 'mapping'); + $table->setPrimaryKey(['objecttype', 'objectid', 'systemtagid'], 'som_pk'); +// $table->addUniqueIndex(['objecttype', 'objectid', 'systemtagid'], 'mapping'); } if (!$schema->hasTable('systemtag_group')) { @@ -808,7 +809,8 @@ class Version13000Date20170718121200 extends SimpleMigrationStep { 'default' => '', ]); $table->addIndex(['object_type', 'object_id'], 'comments_marker_object_index'); - $table->addUniqueIndex(['user_id', 'object_type', 'object_id'], 'comments_marker_index'); + $table->setPrimaryKey(['user_id', 'object_type', 'object_id'], 'crm_pk'); +// $table->addUniqueIndex(['user_id', 'object_type', 'object_id'], 'comments_marker_index'); } // if (!$schema->hasTable('credentials')) { diff --git a/core/Migrations/Version16000Date20190207141427.php b/core/Migrations/Version16000Date20190207141427.php index 63c6c871ea..77a41dc3c4 100644 --- a/core/Migrations/Version16000Date20190207141427.php +++ b/core/Migrations/Version16000Date20190207141427.php @@ -77,7 +77,8 @@ class Version16000Date20190207141427 extends SimpleMigrationStep { 'length' => 64, ]); - $table->addUniqueIndex(['collection_id', 'resource_type', 'resource_id'], 'collres_unique_res'); + $table->setPrimaryKey(['collection_id', 'resource_type', 'resource_id'], 'crr_pk'); +// $table->addUniqueIndex(['collection_id', 'resource_type', 'resource_id'], 'collres_unique_res'); } if (!$schema->hasTable('collres_accesscache')) { @@ -106,7 +107,8 @@ class Version16000Date20190207141427 extends SimpleMigrationStep { 'default' => 0, ]); - $table->addUniqueIndex(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'collres_unique_user'); + $table->setPrimaryKey(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'cra_pk'); +// $table->addUniqueIndex(['user_id', 'collection_id', 'resource_type', 'resource_id'], 'collres_unique_user'); $table->addIndex(['user_id', 'resource_type', 'resource_id'], 'collres_user_res'); $table->addIndex(['user_id', 'collection_id'], 'collres_user_coll'); } diff --git a/core/Migrations/Version17000Date20190514105811.php b/core/Migrations/Version17000Date20190514105811.php index 910d821607..bd6ab1bc8a 100644 --- a/core/Migrations/Version17000Date20190514105811.php +++ b/core/Migrations/Version17000Date20190514105811.php @@ -67,7 +67,8 @@ class Version17000Date20190514105811 extends SimpleMigrationStep { 'length' => 20, 'default' => 0, ]); - $table->addUniqueIndex(['fileid'], 'fce_fileid_idx'); + $table->setPrimaryKey(['fileid'], 'fce_pk'); +// $table->addUniqueIndex(['fileid'], 'fce_fileid_idx'); $table->addIndex(['creation_time'], 'fce_ctime_idx'); $table->addIndex(['upload_time'], 'fce_utime_idx'); } diff --git a/core/Migrations/Version18000Date20191204114856.php b/core/Migrations/Version18000Date20191204114856.php index e9bf1221db..0e5f3cbe27 100644 --- a/core/Migrations/Version18000Date20191204114856.php +++ b/core/Migrations/Version18000Date20191204114856.php @@ -58,7 +58,6 @@ class Version18000Date20191204114856 extends SimpleMigrationStep { 'length' => 4000, ]); - return $schema; } } diff --git a/core/js/setupchecks.js b/core/js/setupchecks.js index e8bcc72cb3..214f148fa9 100644 --- a/core/js/setupchecks.js +++ b/core/js/setupchecks.js @@ -357,6 +357,21 @@ type: OC.SetupChecks.MESSAGE_TYPE_INFO }) } + if (data.missingPrimaryKeys.length > 0) { + var listOfMissingPrimaryKeys = ""; + data.missingPrimaryKeys.forEach(function(element){ + listOfMissingPrimaryKeys += "
  • "; + listOfMissingPrimaryKeys += t('core', 'Missing primary key on table "{tableName}".', element); + listOfMissingPrimaryKeys += "
    • "; + }); + messages.push({ + msg: t( + 'core', + 'The database is missing some primary keys. Due to the fact that adding primary keys on big tables could take some time they were not added automatically. By running "occ db:add-missing-primary-keys" those missing primary keys could be added manually while the instance keeps running.' + ) + "
      " + listOfMissingPrimaryKeys + "
    ", + type: OC.SetupChecks.MESSAGE_TYPE_INFO + }) + } if (data.missingColumns.length > 0) { var listOfMissingColumns = ""; data.missingColumns.forEach(function(element){ diff --git a/core/js/tests/specs/setupchecksSpec.js b/core/js/tests/specs/setupchecksSpec.js index f4c81c6bf7..3b54a3ff66 100644 --- a/core/js/tests/specs/setupchecksSpec.js +++ b/core/js/tests/specs/setupchecksSpec.js @@ -240,6 +240,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -294,6 +295,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -349,6 +351,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -402,6 +405,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -453,6 +457,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -504,6 +509,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -557,6 +563,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -608,6 +615,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: false, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -659,6 +667,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -731,6 +740,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -783,6 +793,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -835,6 +846,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -887,6 +899,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: false, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -938,6 +951,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -989,6 +1003,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { @@ -1041,6 +1056,7 @@ describe('OC.SetupChecks tests', function() { isSettimelimitAvailable: true, hasFreeTypeSupport: true, missingIndexes: [], + missingPrimaryKeys: [], missingColumns: [], cronErrors: [], cronInfo: { diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 16f3b4c9b0..b88f877b92 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -799,6 +799,7 @@ return array( 'OC\\Core\\Command\\Config\\System\\SetConfig' => $baseDir . '/core/Command/Config/System/SetConfig.php', 'OC\\Core\\Command\\Db\\AddMissingColumns' => $baseDir . '/core/Command/Db/AddMissingColumns.php', 'OC\\Core\\Command\\Db\\AddMissingIndices' => $baseDir . '/core/Command/Db/AddMissingIndices.php', + 'OC\\Core\\Command\\Db\\AddMissingPrimaryKeys' => $baseDir . '/core/Command/Db/AddMissingPrimaryKeys.php', 'OC\\Core\\Command\\Db\\ConvertFilecacheBigInt' => $baseDir . '/core/Command/Db/ConvertFilecacheBigInt.php', 'OC\\Core\\Command\\Db\\ConvertMysqlToMB4' => $baseDir . '/core/Command/Db/ConvertMysqlToMB4.php', 'OC\\Core\\Command\\Db\\ConvertType' => $baseDir . '/core/Command/Db/ConvertType.php', @@ -942,6 +943,7 @@ return array( 'OC\\DB\\Migrator' => $baseDir . '/lib/private/DB/Migrator.php', 'OC\\DB\\MissingColumnInformation' => $baseDir . '/lib/private/DB/MissingColumnInformation.php', 'OC\\DB\\MissingIndexInformation' => $baseDir . '/lib/private/DB/MissingIndexInformation.php', + 'OC\\DB\\MissingPrimaryKeyInformation' => $baseDir . '/lib/private/DB/MissingPrimaryKeyInformation.php', 'OC\\DB\\MySQLMigrator' => $baseDir . '/lib/private/DB/MySQLMigrator.php', 'OC\\DB\\MySqlTools' => $baseDir . '/lib/private/DB/MySqlTools.php', 'OC\\DB\\OCSqlitePlatform' => $baseDir . '/lib/private/DB/OCSqlitePlatform.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 1b8e31bba3..2c0b5b1932 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -828,6 +828,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Core\\Command\\Config\\System\\SetConfig' => __DIR__ . '/../../..' . '/core/Command/Config/System/SetConfig.php', 'OC\\Core\\Command\\Db\\AddMissingColumns' => __DIR__ . '/../../..' . '/core/Command/Db/AddMissingColumns.php', 'OC\\Core\\Command\\Db\\AddMissingIndices' => __DIR__ . '/../../..' . '/core/Command/Db/AddMissingIndices.php', + 'OC\\Core\\Command\\Db\\AddMissingPrimaryKeys' => __DIR__ . '/../../..' . '/core/Command/Db/AddMissingPrimaryKeys.php', 'OC\\Core\\Command\\Db\\ConvertFilecacheBigInt' => __DIR__ . '/../../..' . '/core/Command/Db/ConvertFilecacheBigInt.php', 'OC\\Core\\Command\\Db\\ConvertMysqlToMB4' => __DIR__ . '/../../..' . '/core/Command/Db/ConvertMysqlToMB4.php', 'OC\\Core\\Command\\Db\\ConvertType' => __DIR__ . '/../../..' . '/core/Command/Db/ConvertType.php', @@ -971,6 +972,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\DB\\Migrator' => __DIR__ . '/../../..' . '/lib/private/DB/Migrator.php', 'OC\\DB\\MissingColumnInformation' => __DIR__ . '/../../..' . '/lib/private/DB/MissingColumnInformation.php', 'OC\\DB\\MissingIndexInformation' => __DIR__ . '/../../..' . '/lib/private/DB/MissingIndexInformation.php', + 'OC\\DB\\MissingPrimaryKeyInformation' => __DIR__ . '/../../..' . '/lib/private/DB/MissingPrimaryKeyInformation.php', 'OC\\DB\\MySQLMigrator' => __DIR__ . '/../../..' . '/lib/private/DB/MySQLMigrator.php', 'OC\\DB\\MySqlTools' => __DIR__ . '/../../..' . '/lib/private/DB/MySqlTools.php', 'OC\\DB\\OCSqlitePlatform' => __DIR__ . '/../../..' . '/lib/private/DB/OCSqlitePlatform.php', diff --git a/lib/private/DB/MissingPrimaryKeyInformation.php b/lib/private/DB/MissingPrimaryKeyInformation.php new file mode 100644 index 0000000000..c6b81ddba4 --- /dev/null +++ b/lib/private/DB/MissingPrimaryKeyInformation.php @@ -0,0 +1,42 @@ + + * + * @author Morris Jobke + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\DB; + +class MissingPrimaryKeyInformation { + private $listOfMissingPrimaryKeys = []; + + public function addHintForMissingSubject(string $tableName) { + $this->listOfMissingPrimaryKeys[] = [ + 'tableName' => $tableName, + ]; + } + + public function getListOfMissingPrimaryKeys(): array { + return $this->listOfMissingPrimaryKeys; + } +} diff --git a/lib/public/IDBConnection.php b/lib/public/IDBConnection.php index 8e20b5503b..11fa301ab8 100644 --- a/lib/public/IDBConnection.php +++ b/lib/public/IDBConnection.php @@ -50,6 +50,8 @@ use OCP\DB\QueryBuilder\IQueryBuilder; interface IDBConnection { public const ADD_MISSING_INDEXES_EVENT = self::class . '::ADD_MISSING_INDEXES'; public const CHECK_MISSING_INDEXES_EVENT = self::class . '::CHECK_MISSING_INDEXES'; + public const ADD_MISSING_PRIMARY_KEYS_EVENT = self::class . '::ADD_MISSING_PRIMARY_KEYS'; + public const CHECK_MISSING_PRIMARY_KEYS_EVENT = self::class . '::CHECK_MISSING_PRIMARY_KEYS'; public const ADD_MISSING_COLUMNS_EVENT = self::class . '::ADD_MISSING_COLUMNS'; public const CHECK_MISSING_COLUMNS_EVENT = self::class . '::CHECK_MISSING_COLUMNS'; diff --git a/version.php b/version.php index dc48f4bc52..2915ada69d 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version = [21, 0, 0, 5]; +$OC_Version = [21, 0, 0, 6]; // The human readable string $OC_VersionString = '21.0.0 alpha'; From 264d54b388a60d36107dae40521099036266fe8f Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Nov 2020 15:43:34 +0100 Subject: [PATCH 14/17] Fix CS Signed-off-by: Joas Schilling --- apps/dav/lib/Migration/Version1016Date20201109085907.php | 1 + core/Migrations/Version20000Date20201109081915.php | 1 + core/Migrations/Version20000Date20201109081918.php | 1 + core/Migrations/Version20000Date20201109081919.php | 1 + 4 files changed, 4 insertions(+) diff --git a/apps/dav/lib/Migration/Version1016Date20201109085907.php b/apps/dav/lib/Migration/Version1016Date20201109085907.php index a43c8ae06a..fd8996d5e2 100644 --- a/apps/dav/lib/Migration/Version1016Date20201109085907.php +++ b/apps/dav/lib/Migration/Version1016Date20201109085907.php @@ -1,4 +1,5 @@ Date: Tue, 10 Nov 2020 15:55:06 +0100 Subject: [PATCH 15/17] Update baseline, I'm sorry Signed-off-by: Joas Schilling --- build/psalm-baseline.xml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index b4b164f783..e033bab7b3 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -1700,6 +1700,7 @@ $files_list + 'Share has no read permission' $freeSpace @@ -2031,9 +2032,10 @@ - + IDBConnection::CHECK_MISSING_COLUMNS_EVENT IDBConnection::CHECK_MISSING_INDEXES_EVENT + IDBConnection::CHECK_MISSING_PRIMARY_KEYS_EVENT $lastCronRun @@ -2049,7 +2051,8 @@ $lastCronRun 0 - + + dispatch dispatch dispatch @@ -2728,6 +2731,14 @@ dispatch + + + IDBConnection::ADD_MISSING_PRIMARY_KEYS_EVENT + + + dispatch + + $chunk * $chunkSize @@ -3506,7 +3517,9 @@ $e->getCode() - + + \Doctrine\DBAL\Driver\Statement + $statement @@ -3599,7 +3612,9 @@ - + + parent::__construct($params, $driver, $config, $eventManager) + @@ -5284,7 +5299,6 @@ $appName $appName \OC_User::getUser() - \OC_User::getUser() Util::addScript('dist/unified-search', null, true) From a524e83be0d298f5cecdbb11d9c87dfef68a6a9c Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Nov 2020 18:49:02 +0100 Subject: [PATCH 16/17] Fix naming of jobs and steps Signed-off-by: Joas Schilling --- .github/workflows/oci.yml | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/oci.yml b/.github/workflows/oci.yml index 87cc1d41d4..5b545461e5 100644 --- a/.github/workflows/oci.yml +++ b/.github/workflows/oci.yml @@ -1,17 +1,24 @@ -name: "Unit tests" +name: PHPUnit on: + pull_request: push: + branches: + - master + - stable* jobs: phpunit-oci8: - name: "PHPUnit on OCI8" - runs-on: "ubuntu-latest" + runs-on: ubuntu-latest strategy: + # do not stop on another job's failure + fail-fast: false matrix: - php-version: - - "7.4" + php-versions: [ '7.4' ] + databases: [ 'oci' ] + + name: php${{ matrix.php-versions }}-${{ matrix.databases }} services: oracle: @@ -20,8 +27,8 @@ jobs: - "1521:1521" steps: - - name: "Checkout" - uses: "actions/checkout@v2" + - name: Checkout server + uses: actions/checkout@v2 - name: Checkout submodules shell: bash @@ -30,10 +37,10 @@ jobs: git submodule sync --recursive git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 - - name: "Install PHP" - uses: "shivammathur/setup-php@v2" + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@v2 with: - php-version: "${{ matrix.php-version }}" + php-version: ${{ matrix.php-versions }} extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, oci8 tools: phpunit:8.5.2 coverage: none @@ -42,12 +49,8 @@ jobs: run: | mkdir data ./occ maintenance:install --verbose --database=oci --database-name=XE --database-host=127.0.0.1 --database-port=1521 --database-user=autotest --database-pass=owncloud --admin-user admin --admin-pass admin - - # Generate instance id by loading index.php - - name: Generate instance id by loading index.php - run: | php -f index.php - - name: Run phpunit - run: | - cd tests && phpunit --configuration phpunit-autotest.xml --group DB,SLOWDB + - name: PHPUnit + working-directory: tests + run: phpunit --configuration phpunit-autotest.xml --group DB,SLOWDB From 0c695d97ece66d50d4a131205cf724ed49e8152a Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Nov 2020 19:21:08 +0100 Subject: [PATCH 17/17] Fix unique key in test table Signed-off-by: Joas Schilling --- tests/lib/DB/ConnectionTest.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/lib/DB/ConnectionTest.php b/tests/lib/DB/ConnectionTest.php index fa4d5fe005..cab6133c3c 100644 --- a/tests/lib/DB/ConnectionTest.php +++ b/tests/lib/DB/ConnectionTest.php @@ -220,13 +220,15 @@ class ConnectionTest extends \Test\TestCase { ['user' => 'test2', 'category' => 'Coworkers', 'expectedResult' => 1], ]; + $row = 0; foreach ($categoryEntries as $entry) { $result = $this->connection->insertIfNotExist('*PREFIX*table', [ 'textfield' => $entry['user'], 'clobfield' => $entry['category'], - ]); - $this->assertEquals($entry['expectedResult'], $result); + 'integerfield' => $row++, + ], ['textfield', 'clobfield']); + $this->assertEquals($entry['expectedResult'], $result, json_encode($entry)); } $query = $this->connection->prepare('SELECT * FROM `*PREFIX*table`'); @@ -247,13 +249,15 @@ class ConnectionTest extends \Test\TestCase { ['addressbookid' => 123, 'fullname' => 'test', 'expectedResult' => 1], ]; + $row = 0; foreach ($categoryEntries as $entry) { $result = $this->connection->insertIfNotExist('*PREFIX*table', [ 'integerfield_default' => $entry['addressbookid'], 'clobfield' => $entry['fullname'], - ]); - $this->assertEquals($entry['expectedResult'], $result); + 'integerfield' => $row++, + ], ['integerfield_default', 'clobfield']); + $this->assertEquals($entry['expectedResult'], $result, json_encode($entry)); } $query = $this->connection->prepare('SELECT * FROM `*PREFIX*table`');