mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

respect DB limits limit per statement and query 

Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>
This commit is contained in:
Arthur Schiwon 2021-01-08 11:54:07 +01:00
parent ca9783f634
commit e57d8d6521
No known key found for this signature in database
GPG Key ID: 7424F1874854DF23
2 changed files with 64 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
apps/user_ldap/lib/Mapping/AbstractMapping.php
View File

@ -27,6 +27,7 @@
namespace OCA\User_LDAP\Mapping; namespace OCA\User_LDAP\Mapping;
use OC\DB\QueryBuilder\QueryBuilder; use OC\DB\QueryBuilder\QueryBuilder;
use OCP\DB\QueryBuilder\IQueryBuilder;
/** /**
* Class AbstractMapping * Class AbstractMapping
@ -188,19 +189,57 @@ abstract class AbstractMapping {
return $this->cache[$fdn]; return $this->cache[$fdn];
} }
public function getListOfIdsByDn(array $fdns): array { protected function prepareListOfIdsQuery(array $dnList): IQueryBuilder {
$qb = $this->dbc->getQueryBuilder(); $qb = $this->dbc->getQueryBuilder();
$qb->select('owncloud_name', 'ldap_dn') $qb->select('owncloud_name', 'ldap_dn')
->from($this->getTableName(false)) ->from($this->getTableName(false))
->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdns, QueryBuilder::PARAM_STR_ARRAY))); ->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($dnList, QueryBuilder::PARAM_STR_ARRAY)));
$stmt = $qb->execute(); return $qb;
}
$results = $stmt->fetchAll(\Doctrine\DBAL\FetchMode::ASSOCIATIVE); protected function collectResultsFromListOfIdsQuery(IQueryBuilder $qb, array &$results): void {
foreach ($results as $key => $entry) { $stmt = $qb->execute();
unset($results[$key]); while ($entry = $stmt->fetch(\Doctrine\DBAL\FetchMode::ASSOCIATIVE)) {
$results[$entry['ldap_dn']] = $entry['owncloud_name']; $results[$entry['ldap_dn']] = $entry['owncloud_name'];
$this->cache[$entry['ldap_dn']] = $entry['owncloud_name']; $this->cache[$entry['ldap_dn']] = $entry['owncloud_name'];
} }
$stmt->closeCursor();
}
public function getListOfIdsByDn(array $fdns): array {
$totalDBParamLimit = 65000;
$sliceSize = 1000;
$maxSlices = $totalDBParamLimit / $sliceSize;
$results = [];
$slice = 1;
$fdnsSlice = count($fdns) > $sliceSize ? array_slice($fdns, 0, $sliceSize) : $fdns;
$qb = $this->prepareListOfIdsQuery($fdnsSlice);
while (isset($fdnsSlice[999])) {
// Oracle does not allow more than 1000 values in the IN list,
// but allows slicing
$slice++;
$fdnsSlice = array_slice($fdns, $sliceSize * ($slice - 1), $sliceSize);
if (!isset($qb)) {
$qb = $this->prepareListOfIdsQuery($fdnsSlice);
continue;
}
if (!empty($fdnsSlice)) {
$qb->orWhere($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdnsSlice, QueryBuilder::PARAM_STR_ARRAY)));
}
if ($slice % $maxSlices === 0) {
$this->collectResultsFromListOfIdsQuery($qb, $results);
unset($qb);
}
}
if (isset($qb)) {
$this->collectResultsFromListOfIdsQuery($qb, $results);
}
return $results; return $results;
} }

19
apps/user_ldap/tests/Mapping/AbstractMappingTest.php
View File

@ -281,4 +281,23 @@ abstract class AbstractMappingTest extends \Test\TestCase {
$results = $mapper->getList(1, 1); $results = $mapper->getList(1, 1);
$this->assertSame(1, count($results)); $this->assertSame(1, count($results));
} }
public function testGetListOfIdsByDn() {
/** @var AbstractMapping $mapper */
list($mapper,) = $this->initTest();
$listOfDNs = [];
for ($i = 0; $i < 66640; $i++) {
// Postgres has a limit of 65535 values in a single IN list
$name = 'as_' . $i;
$dn = 'uid=' . $name . ',dc=example,dc=org';
$listOfDNs[] = $dn;
if ($i % 20 === 0) {
$mapper->map($dn, $name, 'fake-uuid-' . $i);
}
}
$result = $mapper->getListOfIdsByDn($listOfDNs);
$this->assertSame(66640 / 20, count($result));
}
} }