mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

respect DB limits limit per statement and query 

Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>
This commit is contained in:
Arthur Schiwon 2021-01-08 11:54:07 +01:00
parent 1ef9a3f039
commit b0ea654653
No known key found for this signature in database
GPG Key ID: 7424F1874854DF23
2 changed files with 64 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
apps/user_ldap/lib/Mapping/AbstractMapping.php
View File

@ -28,6 +28,7 @@
namespace OCA\User_LDAP\Mapping;
use OC\DB\QueryBuilder\QueryBuilder;
use OCP\DB\QueryBuilder\IQueryBuilder;
/**
* Class AbstractMapping
@ -189,19 +190,57 @@ abstract class AbstractMapping {
return $this->cache[$fdn];
}
public function getListOfIdsByDn(array $fdns): array {
protected function prepareListOfIdsQuery(array $dnList): IQueryBuilder {
$qb = $this->dbc->getQueryBuilder();
$qb->select('owncloud_name', 'ldap_dn')
->from($this->getTableName(false))
->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdns, QueryBuilder::PARAM_STR_ARRAY)));
$stmt = $qb->execute();
->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($dnList, QueryBuilder::PARAM_STR_ARRAY)));
return $qb;
}
$results = $stmt->fetchAll(\Doctrine\DBAL\FetchMode::ASSOCIATIVE);
foreach ($results as $key => $entry) {
unset($results[$key]);
protected function collectResultsFromListOfIdsQuery(IQueryBuilder $qb, array &$results): void {
$stmt = $qb->execute();
while ($entry = $stmt->fetch(\Doctrine\DBAL\FetchMode::ASSOCIATIVE)) {
$results[$entry['ldap_dn']] = $entry['owncloud_name'];
$this->cache[$entry['ldap_dn']] = $entry['owncloud_name'];
}
$stmt->closeCursor();
}
public function getListOfIdsByDn(array $fdns): array {
$totalDBParamLimit = 65000;
$sliceSize = 1000;
$maxSlices = $totalDBParamLimit / $sliceSize;
$results = [];
$slice = 1;
$fdnsSlice = count($fdns) > $sliceSize ? array_slice($fdns, 0, $sliceSize) : $fdns;
$qb = $this->prepareListOfIdsQuery($fdnsSlice);
while (isset($fdnsSlice[999])) {
// Oracle does not allow more than 1000 values in the IN list,
// but allows slicing
$slice++;
$fdnsSlice = array_slice($fdns, $sliceSize * ($slice - 1), $sliceSize);
if (!isset($qb)) {
$qb = $this->prepareListOfIdsQuery($fdnsSlice);
continue;
}
if (!empty($fdnsSlice)) {
$qb->orWhere($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdnsSlice, QueryBuilder::PARAM_STR_ARRAY)));
}
if ($slice % $maxSlices === 0) {
$this->collectResultsFromListOfIdsQuery($qb, $results);
unset($qb);
}
}
if (isset($qb)) {
$this->collectResultsFromListOfIdsQuery($qb, $results);
}
return $results;
}

19
apps/user_ldap/tests/Mapping/AbstractMappingTest.php
View File

@ -281,4 +281,23 @@ abstract class AbstractMappingTest extends \Test\TestCase {
$results = $mapper->getList(1, 1);
$this->assertSame(1, count($results));
}
public function testGetListOfIdsByDn() {
/** @var AbstractMapping $mapper */
list($mapper,) = $this->initTest();
$listOfDNs = [];
for ($i = 0; $i < 66640; $i++) {
// Postgres has a limit of 65535 values in a single IN list
$name = 'as_' . $i;
$dn = 'uid=' . $name . ',dc=example,dc=org';
$listOfDNs[] = $dn;
if ($i % 20 === 0) {
$mapper->map($dn, $name, 'fake-uuid-' . $i);
}
}
$result = $mapper->getListOfIdsByDn($listOfDNs);
$this->assertSame(66640 / 20, count($result));
}
}