From e57d8d6521f25cf6523c1a880ad22727674f3ae0 Mon Sep 17 00:00:00 2001 From: Arthur Schiwon Date: Fri, 8 Jan 2021 11:54:07 +0100 Subject: [PATCH] respect DB limits limit per statement and query Signed-off-by: Arthur Schiwon --- .../user_ldap/lib/Mapping/AbstractMapping.php | 51 ++++++++++++++++--- .../tests/Mapping/AbstractMappingTest.php | 19 +++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/apps/user_ldap/lib/Mapping/AbstractMapping.php b/apps/user_ldap/lib/Mapping/AbstractMapping.php index 0daa4f1504..d46010efc3 100644 --- a/apps/user_ldap/lib/Mapping/AbstractMapping.php +++ b/apps/user_ldap/lib/Mapping/AbstractMapping.php @@ -27,6 +27,7 @@ namespace OCA\User_LDAP\Mapping; use OC\DB\QueryBuilder\QueryBuilder; +use OCP\DB\QueryBuilder\IQueryBuilder; /** * Class AbstractMapping @@ -188,19 +189,57 @@ abstract class AbstractMapping { return $this->cache[$fdn]; } - public function getListOfIdsByDn(array $fdns): array { + protected function prepareListOfIdsQuery(array $dnList): IQueryBuilder { $qb = $this->dbc->getQueryBuilder(); $qb->select('owncloud_name', 'ldap_dn') ->from($this->getTableName(false)) - ->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdns, QueryBuilder::PARAM_STR_ARRAY))); - $stmt = $qb->execute(); + ->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($dnList, QueryBuilder::PARAM_STR_ARRAY))); + return $qb; + } - $results = $stmt->fetchAll(\Doctrine\DBAL\FetchMode::ASSOCIATIVE); - foreach ($results as $key => $entry) { - unset($results[$key]); + protected function collectResultsFromListOfIdsQuery(IQueryBuilder $qb, array &$results): void { + $stmt = $qb->execute(); + while ($entry = $stmt->fetch(\Doctrine\DBAL\FetchMode::ASSOCIATIVE)) { $results[$entry['ldap_dn']] = $entry['owncloud_name']; $this->cache[$entry['ldap_dn']] = $entry['owncloud_name']; } + $stmt->closeCursor(); + } + + public function getListOfIdsByDn(array $fdns): array { + $totalDBParamLimit = 65000; + $sliceSize = 1000; + $maxSlices = $totalDBParamLimit / $sliceSize; + $results = []; + + $slice = 1; + $fdnsSlice = count($fdns) > $sliceSize ? array_slice($fdns, 0, $sliceSize) : $fdns; + $qb = $this->prepareListOfIdsQuery($fdnsSlice); + + while (isset($fdnsSlice[999])) { + // Oracle does not allow more than 1000 values in the IN list, + // but allows slicing + $slice++; + $fdnsSlice = array_slice($fdns, $sliceSize * ($slice - 1), $sliceSize); + + if (!isset($qb)) { + $qb = $this->prepareListOfIdsQuery($fdnsSlice); + continue; + } + + if (!empty($fdnsSlice)) { + $qb->orWhere($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdnsSlice, QueryBuilder::PARAM_STR_ARRAY))); + } + + if ($slice % $maxSlices === 0) { + $this->collectResultsFromListOfIdsQuery($qb, $results); + unset($qb); + } + } + + if (isset($qb)) { + $this->collectResultsFromListOfIdsQuery($qb, $results); + } return $results; } diff --git a/apps/user_ldap/tests/Mapping/AbstractMappingTest.php b/apps/user_ldap/tests/Mapping/AbstractMappingTest.php index 079c2e21b1..35259345f2 100644 --- a/apps/user_ldap/tests/Mapping/AbstractMappingTest.php +++ b/apps/user_ldap/tests/Mapping/AbstractMappingTest.php @@ -281,4 +281,23 @@ abstract class AbstractMappingTest extends \Test\TestCase { $results = $mapper->getList(1, 1); $this->assertSame(1, count($results)); } + + public function testGetListOfIdsByDn() { + /** @var AbstractMapping $mapper */ + list($mapper,) = $this->initTest(); + + $listOfDNs = []; + for ($i = 0; $i < 66640; $i++) { + // Postgres has a limit of 65535 values in a single IN list + $name = 'as_' . $i; + $dn = 'uid=' . $name . ',dc=example,dc=org'; + $listOfDNs[] = $dn; + if ($i % 20 === 0) { + $mapper->map($dn, $name, 'fake-uuid-' . $i); + } + } + + $result = $mapper->getListOfIdsByDn($listOfDNs); + $this->assertSame(66640 / 20, count($result)); + } }