mbk-lab
/
nextcloud
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

split of query building bits from searchhelper 

Signed-off-by: Robin Appelman <robin@icewind.nl>
This commit is contained in:
Robin Appelman 2021-05-26 15:50:35 +02:00
parent 139fac9526
commit acf974c1cb
No known key found for this signature in database
GPG Key ID: 42B69D8A64526EFB
5 changed files with 257 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/composer/composer/autoload_classmap.php
View File

@ -1057,6 +1057,7 @@ return array(
'OC\\Files\\Cache\\Propagator' => $baseDir . '/lib/private/Files/Cache/Propagator.php',
'OC\\Files\\Cache\\QuerySearchHelper' => $baseDir . '/lib/private/Files/Cache/QuerySearchHelper.php',
'OC\\Files\\Cache\\Scanner' => $baseDir . '/lib/private/Files/Cache/Scanner.php',
'OC\\Files\\Cache\\SearchBuilder' => $baseDir . '/lib/private/Files/Cache/SearchBuilder.php',
'OC\\Files\\Cache\\Storage' => $baseDir . '/lib/private/Files/Cache/Storage.php',
'OC\\Files\\Cache\\StorageGlobal' => $baseDir . '/lib/private/Files/Cache/StorageGlobal.php',
'OC\\Files\\Cache\\Updater' => $baseDir . '/lib/private/Files/Cache/Updater.php',

1
lib/composer/composer/autoload_static.php
View File

@ -1086,6 +1086,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Files\\Cache\\Propagator' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Propagator.php',
'OC\\Files\\Cache\\QuerySearchHelper' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/QuerySearchHelper.php',
'OC\\Files\\Cache\\Scanner' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Scanner.php',
'OC\\Files\\Cache\\SearchBuilder' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/SearchBuilder.php',
'OC\\Files\\Cache\\Storage' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Storage.php',
'OC\\Files\\Cache\\StorageGlobal' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/StorageGlobal.php',
'OC\\Files\\Cache\\Updater' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Updater.php',

203
lib/private/Files/Cache/QuerySearchHelper.php
View File

@ -28,41 +28,15 @@ namespace OC\Files\Cache;
use OC\Files\Search\SearchBinaryOperator;
use OC\SystemConfig;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Cache\ICache;
use OCP\Files\Cache\ICacheEntry;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use OCP\Files\Search\ISearchOperator;
use OCP\Files\Search\ISearchOrder;
use OCP\Files\Search\ISearchQuery;
use OCP\IDBConnection;
use OCP\ILogger;
/**
* Tools for transforming search queries into database queries
*/
class QuerySearchHelper {
protected static $searchOperatorMap = [
ISearchComparison::COMPARE_LIKE => 'iLike',
ISearchComparison::COMPARE_EQUAL => 'eq',
ISearchComparison::COMPARE_GREATER_THAN => 'gt',
ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte',
ISearchComparison::COMPARE_LESS_THAN => 'lt',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte',
];
protected static $searchOperatorNegativeMap = [
ISearchComparison::COMPARE_LIKE => 'notLike',
ISearchComparison::COMPARE_EQUAL => 'neq',
ISearchComparison::COMPARE_GREATER_THAN => 'lte',
ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt',
ISearchComparison::COMPARE_LESS_THAN => 'gte',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lt',
];
public const TAG_FAVORITE = '_$!<Favorite>!$_';
/** @var IMimeTypeLoader */
private $mimetypeLoader;
@ -72,6 +46,8 @@ class QuerySearchHelper {
private $systemConfig;
/** @var ILogger */
private $logger;
/** @var SearchBuilder */
private $searchBuilder;
public function __construct(
IMimeTypeLoader $mimetypeLoader,
@ -83,172 +59,7 @@ class QuerySearchHelper {
$this->connection = $connection;
$this->systemConfig = $systemConfig;
$this->logger = $logger;
}
/**
* Whether or not the tag tables should be joined to complete the search
*
* @param ISearchOperator $operator
* @return boolean
*/
public function shouldJoinTags(ISearchOperator $operator) {
if ($operator instanceof ISearchBinaryOperator) {
return array_reduce($operator->getArguments(), function ($shouldJoin, ISearchOperator $operator) {
return $shouldJoin || $this->shouldJoinTags($operator);
}, false);
} elseif ($operator instanceof ISearchComparison) {
return $operator->getField() === 'tagname' || $operator->getField() === 'favorite';
}
return false;
}
/**
* @param IQueryBuilder $builder
* @param ISearchOperator $operator
*/
public function searchOperatorArrayToDBExprArray(IQueryBuilder $builder, array $operators) {
return array_filter(array_map(function ($operator) use ($builder) {
return $this->searchOperatorToDBExpr($builder, $operator);
}, $operators));
}
public function searchOperatorToDBExpr(IQueryBuilder $builder, ISearchOperator $operator) {
$expr = $builder->expr();
if ($operator instanceof ISearchBinaryOperator) {
if (count($operator->getArguments()) === 0) {
return null;
}
switch ($operator->getType()) {
case ISearchBinaryOperator::OPERATOR_NOT:
$negativeOperator = $operator->getArguments()[0];
if ($negativeOperator instanceof ISearchComparison) {
return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap);
} else {
throw new \InvalidArgumentException('Binary operators inside "not" is not supported');
}
// no break
case ISearchBinaryOperator::OPERATOR_AND:
return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments()));
case ISearchBinaryOperator::OPERATOR_OR:
return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments()));
default:
throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType());
}
} elseif ($operator instanceof ISearchComparison) {
return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap);
} else {
throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator));
}
}
private function searchComparisonToDBExpr(IQueryBuilder $builder, ISearchComparison $comparison, array $operatorMap) {
$this->validateComparison($comparison);
[$field, $value, $type] = $this->getOperatorFieldAndValue($comparison);
if (isset($operatorMap[$type])) {
$queryOperator = $operatorMap[$type];
return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value));
} else {
throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType());
}
}
private function getOperatorFieldAndValue(ISearchComparison $operator) {
$field = $operator->getField();
$value = $operator->getValue();
$type = $operator->getType();
if ($field === 'mimetype') {
if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) {
$value = (int)$this->mimetypeLoader->getId($value);
} elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) {
// transform "mimetype='foo/%'" to "mimepart='foo'"
if (preg_match('|(.+)/%|', $value, $matches)) {
$field = 'mimepart';
$value = (int)$this->mimetypeLoader->getId($matches[1]);
$type = ISearchComparison::COMPARE_EQUAL;
} elseif (strpos($value, '%') !== false) {
throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported');
} else {
$field = 'mimetype';
$value = (int)$this->mimetypeLoader->getId($value);
$type = ISearchComparison::COMPARE_EQUAL;
}
}
} elseif ($field === 'favorite') {
$field = 'tag.category';
$value = self::TAG_FAVORITE;
} elseif ($field === 'tagname') {
$field = 'tag.category';
} elseif ($field === 'fileid') {
$field = 'file.fileid';
} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL) {
$field = 'path_hash';
$value = md5((string)$value);
}
return [$field, $value, $type];
}
private function validateComparison(ISearchComparison $operator) {
$types = [
'mimetype' => 'string',
'mtime' => 'integer',
'name' => 'string',
'path' => 'string',
'size' => 'integer',
'tagname' => 'string',
'favorite' => 'boolean',
'fileid' => 'integer',
'storage' => 'integer',
];
$comparisons = [
'mimetype' => ['eq', 'like'],
'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'name' => ['eq', 'like'],
'path' => ['eq', 'like'],
'size' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'tagname' => ['eq', 'like'],
'favorite' => ['eq'],
'fileid' => ['eq'],
'storage' => ['eq'],
];
if (!isset($types[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField());
}
$type = $types[$operator->getField()];
if (gettype($operator->getValue()) !== $type) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
}
if (!in_array($operator->getType(), $comparisons[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType());
}
}
private function getParameterForValue(IQueryBuilder $builder, $value) {
if ($value instanceof \DateTime) {
$value = $value->getTimestamp();
}
if (is_numeric($value)) {
$type = IQueryBuilder::PARAM_INT;
} else {
$type = IQueryBuilder::PARAM_STR;
}
return $builder->createNamedParameter($value, $type);
}
/**
* @param IQueryBuilder $query
* @param ISearchOrder[] $orders
*/
public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders) {
foreach ($orders as $order) {
$field = $order->getField();
if ($field === 'fileid') {
$field = 'file.fileid';
}
$query->addOrderBy($field, $order->getDirection());
}
$this->searchBuilder = new SearchBuilder($this->mimetypeLoader);
}
protected function getQueryBuilder() {
@ -289,7 +100,7 @@ class QuerySearchHelper {
$query = $builder->selectFileCache('file');
if ($this->shouldJoinTags($searchQuery->getSearchOperation())) {
if ($this->searchBuilder->shouldJoinTags($searchQuery->getSearchOperation())) {
$user = $searchQuery->getUser();
if ($user === null) {
throw new \InvalidArgumentException("Searching by tag requires the user to be set in the query");
@ -304,7 +115,7 @@ class QuerySearchHelper {
->andWhere($builder->expr()->eq('tag.uid', $builder->createNamedParameter($user->getUID())));
}
$searchExpr = $this->searchOperatorToDBExpr($builder, $searchQuery->getSearchOperation());
$searchExpr = $this->searchBuilder->searchOperatorToDBExpr($builder, $searchQuery->getSearchOperation());
if ($searchExpr) {
$query->andWhere($searchExpr);
}
@ -312,9 +123,9 @@ class QuerySearchHelper {
$storageFilters = array_values(array_map(function (ICache $cache) {
return $cache->getQueryFilterForStorage();
}, $caches));
$query->andWhere($this->searchOperatorToDBExpr($builder, new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $storageFilters)));
$query->andWhere($this->searchBuilder->searchOperatorToDBExpr($builder, new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_OR, $storageFilters)));
$this->addSearchOrdersToQuery($query, $searchQuery->getOrder());
$this->searchBuilder->addSearchOrdersToQuery($query, $searchQuery->getOrder());
if ($searchQuery->getLimit()) {
$query->setMaxResults($searchQuery->getLimit());

235
lib/private/Files/Cache/SearchBuilder.php Normal file
View File

@ -0,0 +1,235 @@
<?php
/**
* @copyright Copyright (c) 2017 Robin Appelman <robin@icewind.nl>
*
* @author Christoph Wurst <christoph@winzerhof-wurst.at>
* @author Robin Appelman <robin@icewind.nl>
* @author Roeland Jago Douma <roeland@famdouma.nl>
* @author Tobias Kaminsky <tobias@kaminsky.me>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\Cache;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use OCP\Files\Search\ISearchOperator;
use OCP\Files\Search\ISearchOrder;
/**
* Tools for transforming search queries into database queries
*/
class SearchBuilder {
protected static $searchOperatorMap = [
ISearchComparison::COMPARE_LIKE => 'iLike',
ISearchComparison::COMPARE_EQUAL => 'eq',
ISearchComparison::COMPARE_GREATER_THAN => 'gt',
ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'gte',
ISearchComparison::COMPARE_LESS_THAN => 'lt',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte',
];
protected static $searchOperatorNegativeMap = [
ISearchComparison::COMPARE_LIKE => 'notLike',
ISearchComparison::COMPARE_EQUAL => 'neq',
ISearchComparison::COMPARE_GREATER_THAN => 'lte',
ISearchComparison::COMPARE_GREATER_THAN_EQUAL => 'lt',
ISearchComparison::COMPARE_LESS_THAN => 'gte',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lt',
];
public const TAG_FAVORITE = '_$!<Favorite>!$_';
/** @var IMimeTypeLoader */
private $mimetypeLoader;
public function __construct(
IMimeTypeLoader $mimetypeLoader
) {
$this->mimetypeLoader = $mimetypeLoader;
}
/**
* Whether or not the tag tables should be joined to complete the search
*
* @param ISearchOperator $operator
* @return boolean
*/
public function shouldJoinTags(ISearchOperator $operator) {
if ($operator instanceof ISearchBinaryOperator) {
return array_reduce($operator->getArguments(), function ($shouldJoin, ISearchOperator $operator) {
return $shouldJoin || $this->shouldJoinTags($operator);
}, false);
} elseif ($operator instanceof ISearchComparison) {
return $operator->getField() === 'tagname' || $operator->getField() === 'favorite';
}
return false;
}
/**
* @param IQueryBuilder $builder
* @param ISearchOperator $operator
*/
public function searchOperatorArrayToDBExprArray(IQueryBuilder $builder, array $operators) {
return array_filter(array_map(function ($operator) use ($builder) {
return $this->searchOperatorToDBExpr($builder, $operator);
}, $operators));
}
public function searchOperatorToDBExpr(IQueryBuilder $builder, ISearchOperator $operator) {
$expr = $builder->expr();
if ($operator instanceof ISearchBinaryOperator) {
if (count($operator->getArguments()) === 0) {
return null;
}
switch ($operator->getType()) {
case ISearchBinaryOperator::OPERATOR_NOT:
$negativeOperator = $operator->getArguments()[0];
if ($negativeOperator instanceof ISearchComparison) {
return $this->searchComparisonToDBExpr($builder, $negativeOperator, self::$searchOperatorNegativeMap);
} else {
throw new \InvalidArgumentException('Binary operators inside "not" is not supported');
}
// no break
case ISearchBinaryOperator::OPERATOR_AND:
return call_user_func_array([$expr, 'andX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments()));
case ISearchBinaryOperator::OPERATOR_OR:
return call_user_func_array([$expr, 'orX'], $this->searchOperatorArrayToDBExprArray($builder, $operator->getArguments()));
default:
throw new \InvalidArgumentException('Invalid operator type: ' . $operator->getType());
}
} elseif ($operator instanceof ISearchComparison) {
return $this->searchComparisonToDBExpr($builder, $operator, self::$searchOperatorMap);
} else {
throw new \InvalidArgumentException('Invalid operator type: ' . get_class($operator));
}
}
private function searchComparisonToDBExpr(IQueryBuilder $builder, ISearchComparison $comparison, array $operatorMap) {
$this->validateComparison($comparison);
[$field, $value, $type] = $this->getOperatorFieldAndValue($comparison);
if (isset($operatorMap[$type])) {
$queryOperator = $operatorMap[$type];
return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value));
} else {
throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType());
}
}
private function getOperatorFieldAndValue(ISearchComparison $operator) {
$field = $operator->getField();
$value = $operator->getValue();
$type = $operator->getType();
if ($field === 'mimetype') {
$value = (string)$value;
if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) {
$value = (int)$this->mimetypeLoader->getId($value);
} elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) {
// transform "mimetype='foo/%'" to "mimepart='foo'"
if (preg_match('|(.+)/%|', $value, $matches)) {
$field = 'mimepart';
$value = (int)$this->mimetypeLoader->getId($matches[1]);
$type = ISearchComparison::COMPARE_EQUAL;
} elseif (strpos($value, '%') !== false) {
throw new \InvalidArgumentException('Unsupported query value for mimetype: ' . $value . ', only values in the format "mime/type" or "mime/%" are supported');
} else {
$field = 'mimetype';
$value = (int)$this->mimetypeLoader->getId($value);
$type = ISearchComparison::COMPARE_EQUAL;
}
}
} elseif ($field === 'favorite') {
$field = 'tag.category';
$value = self::TAG_FAVORITE;
} elseif ($field === 'tagname') {
$field = 'tag.category';
} elseif ($field === 'fileid') {
$field = 'file.fileid';
} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL) {
$field = 'path_hash';
$value = md5((string)$value);
}
return [$field, $value, $type];
}
private function validateComparison(ISearchComparison $operator) {
$types = [
'mimetype' => 'string',
'mtime' => 'integer',
'name' => 'string',
'path' => 'string',
'size' => 'integer',
'tagname' => 'string',
'favorite' => 'boolean',
'fileid' => 'integer',
'storage' => 'integer',
];
$comparisons = [
'mimetype' => ['eq', 'like'],
'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'name' => ['eq', 'like'],
'path' => ['eq', 'like'],
'size' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'tagname' => ['eq', 'like'],
'favorite' => ['eq'],
'fileid' => ['eq'],
'storage' => ['eq'],
];
if (!isset($types[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField());
}
$type = $types[$operator->getField()];
if (gettype($operator->getValue()) !== $type) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
}
if (!in_array($operator->getType(), $comparisons[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType());
}
}
private function getParameterForValue(IQueryBuilder $builder, $value) {
if ($value instanceof \DateTime) {
$value = $value->getTimestamp();
}
if (is_numeric($value)) {
$type = IQueryBuilder::PARAM_INT;
} else {
$type = IQueryBuilder::PARAM_STR;
}
return $builder->createNamedParameter($value, $type);
}
/**
* @param IQueryBuilder $query
* @param ISearchOrder[] $orders
*/
public function addSearchOrdersToQuery(IQueryBuilder $query, array $orders) {
foreach ($orders as $order) {
$field = $order->getField();
if ($field === 'fileid') {
$field = 'file.fileid';
}
$query->addOrderBy($field, $order->getDirection());
}
}
}

23
tests/lib/Files/Cache/QuerySearchHelperTest.php → tests/lib/Files/Cache/SearchBuilderTest.php
View File

@ -22,31 +22,28 @@
namespace Test\Files\Cache;
use OC\DB\QueryBuilder\Literal;
use OC\Files\Cache\QuerySearchHelper;
use OC\Files\Cache\SearchBuilder;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OC\SystemConfig;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use OCP\Files\Search\ISearchOperator;
use OCP\IDBConnection;
use OCP\ILogger;
use Test\TestCase;
/**
* @group DB
*/
class QuerySearchHelperTest extends TestCase {
class SearchBuilderTest extends TestCase {
/** @var IQueryBuilder */
private $builder;
/** @var IMimeTypeLoader|\PHPUnit\Framework\MockObject\MockObject */
private $mimetypeLoader;
/** @var QuerySearchHelper */
private $querySearchHelper;
/** @var SearchBuilder */
private $searchBuilder;
/** @var integer */
private $numericStorageId;
@ -78,15 +75,7 @@ class QuerySearchHelperTest extends TestCase {
[6, 'image']
]);
$systemConfig = $this->createMock(SystemConfig::class);
$logger = $this->createMock(ILogger::class);
$this->querySearchHelper = new QuerySearchHelper(
$this->mimetypeLoader,
\OC::$server->get(IDBConnection::class),
$systemConfig,
$logger
);
$this->searchBuilder = new SearchBuilder($this->mimetypeLoader);
$this->numericStorageId = 10000;
$this->builder->select(['fileid'])
@ -145,7 +134,7 @@ class QuerySearchHelperTest extends TestCase {
}
private function search(ISearchOperator $operator) {
$dbOperator = $this->querySearchHelper->searchOperatorToDBExpr($this->builder, $operator);
$dbOperator = $this->searchBuilder->searchOperatorToDBExpr($this->builder, $operator);
$this->builder->andWhere($dbOperator);
$result = $this->builder->execute();