3 namespace Drupal\search;
5 use Drupal\Core\Database\Query\Condition;
6 use Drupal\Component\Utility\Unicode;
7 use Drupal\Core\Database\Query\SelectExtender;
8 use Drupal\Core\Database\Query\SelectInterface;
11 * Search query extender and helper functions.
13 * Performs a query on the full-text search index for a word or words.
15 * This query is used by search plugins that use the search index (not all
16 * search plugins do, as some use a different searching mechanism). It
17 * assumes you have set up a query on the {search_index} table with alias 'i',
18 * and will only work if the user is searching for at least one "positive"
21 * For efficiency, users of this query can run the prepareAndNormalize()
22 * method to figure out if there are any search results, before fully setting
23 * up and calling execute() to execute the query. The scoring expressions are
24 * not needed until the execute() step. However, it's not really necessary
25 * to do this, because this class's execute() method does that anyway.
27 * During both the prepareAndNormalize() and execute() steps, there can be
28 * problems. Call getStatus() to figure out if the query is OK or not.
30 * The query object is given the tag 'search_$type' and can be further
31 * extended with hook_query_alter().
33 class SearchQuery extends SelectExtender {
36 * Indicates no positive keywords were in the search expression.
38 * Positive keywords are words that are searched for, as opposed to negative
39 * keywords, which are words that are excluded. To count as a keyword, a
40 * word must be at least
41 * \Drupal::config('search.settings')->get('index.minimum_word_size')
44 * @see SearchQuery::getStatus()
46 const NO_POSITIVE_KEYWORDS = 1;
49 * Indicates that part of the search expression was ignored.
51 * To prevent Denial of Service attacks, only
52 * \Drupal::config('search.settings')->get('and_or_limit') expressions
53 * (positive keywords, phrases, negative keywords) are allowed; this flag
54 * indicates that expressions existed past that limit and they were removed.
56 * @see SearchQuery::getStatus()
58 const EXPRESSIONS_IGNORED = 2;
61 * Indicates that lower-case "or" was in the search expression.
63 * The word "or" in lower case was found in the search expression. This
64 * probably means someone was trying to do an OR search but used lower-case
65 * instead of upper-case.
67 * @see SearchQuery::getStatus()
69 const LOWER_CASE_OR = 4;
72 * Indicates that no positive keyword matches were found.
74 * @see SearchQuery::getStatus()
76 const NO_KEYWORD_MATCHES = 8;
79 * The keywords and advanced search options that are entered by the user.
83 protected $searchExpression;
86 * The type of search (search type).
88 * This maps to the value of the type column in search_index, and is usually
89 * equal to the machine-readable name of the plugin or the search page.
96 * Parsed-out positive and negative search keys.
100 protected $keys = ['positive' => [], 'negative' => []];
103 * Indicates whether the query conditions are simple or complex (LIKE).
107 protected $simple = TRUE;
110 * Conditions that are used for exact searches.
112 * This is always used for the second step in the query, but is not part of
113 * the preparation step unless $this->simple is FALSE.
115 * @var DatabaseCondition
117 protected $conditions;
120 * Indicates how many matches for a search query are necessary.
124 protected $matches = 0;
127 * Array of positive search words.
129 * These words have to match against {search_index}.word.
133 protected $words = [];
136 * Multiplier to normalize the keyword score.
138 * This value is calculated by the preparation step, and is used as a
139 * multiplier of the word scores to make sure they are between 0 and 1.
143 protected $normalize = 0;
146 * Indicates whether the preparation step has been executed.
150 protected $executedPrepare = FALSE;
153 * A bitmap of status conditions, described in getStatus().
157 * @see SearchQuery::getStatus()
159 protected $status = 0;
162 * The word score expressions.
166 * @see SearchQuery::addScore()
168 protected $scores = [];
171 * Arguments for the score expressions.
175 protected $scoresArguments = [];
178 * The number of 'i.relevance' occurrences in score expressions.
182 protected $relevance_count = 0;
185 * Multipliers for score expressions.
189 protected $multiply = [];
192 * Sets the search query expression.
194 * @param string $expression
195 * A search string, which can contain keywords and options.
196 * @param string $type
197 * The search type. This maps to {search_index}.type in the database.
201 public function searchExpression($expression, $type) {
202 $this->searchExpression = $expression;
206 $this->addTag('search_' . $type);
208 // Initialize conditions and status.
209 $this->conditions = new Condition('AND');
216 * Parses the search query into SQL conditions.
218 * Sets up the following variables:
221 * - $this->conditions
225 protected function parseSearchExpression() {
226 // Matches words optionally prefixed by a - sign. A word in this case is
227 // something between two spaces, optionally quoted.
228 preg_match_all('/ (-?)("[^"]+"|[^" ]+)/i', ' ' . $this->searchExpression, $keywords, PREG_SET_ORDER);
230 if (count($keywords) == 0) {
236 $limit_combinations = \Drupal::config('search.settings')->get('and_or_limit');
237 // The first search expression does not count as AND.
240 foreach ($keywords as $match) {
241 if ($or_count && $and_count + $or_count >= $limit_combinations) {
242 // Ignore all further search expressions to prevent Denial-of-Service
243 // attacks using a high number of AND/OR combinations.
244 $this->status |= SearchQuery::EXPRESSIONS_IGNORED;
248 // Strip off phrase quotes.
250 if ($match[2]{0} == '"') {
251 $match[2] = substr($match[2], 1, -1);
253 $this->simple = FALSE;
256 // Simplify keyword according to indexing rules and external
257 // preprocessors. Use same process as during search indexing, so it
258 // will match search index.
259 $words = search_simplify($match[2]);
260 // Re-explode in case simplification added more words, except when
261 // matching a phrase.
262 $words = $phrase ? [$words] : preg_split('/ /', $words, -1, PREG_SPLIT_NO_EMPTY);
264 if ($match[1] == '-') {
265 $this->keys['negative'] = array_merge($this->keys['negative'], $words);
267 // OR operator: instead of a single keyword, we store an array of all
269 elseif ($match[2] == 'OR' && count($this->keys['positive'])) {
270 $last = array_pop($this->keys['positive']);
271 // Starting a new OR?
272 if (!is_array($last)) {
275 $this->keys['positive'][] = $last;
280 // AND operator: implied, so just ignore it.
281 elseif ($match[2] == 'AND' || $match[2] == 'and') {
287 if ($match[2] == 'or') {
288 // Lower-case "or" instead of "OR" is a warning condition.
289 $this->status |= SearchQuery::LOWER_CASE_OR;
292 // Add to last element (which is an array).
293 $this->keys['positive'][count($this->keys['positive']) - 1] = array_merge($this->keys['positive'][count($this->keys['positive']) - 1], $words);
296 $this->keys['positive'] = array_merge($this->keys['positive'], $words);
303 // Convert keywords into SQL statements.
307 foreach ($this->keys['positive'] as $key) {
308 // Group of ORed terms.
309 if (is_array($key) && count($key)) {
310 // If we had already found one OR, this is another one AND-ed with the
311 // first, meaning it is not a simple query.
313 $this->simple = FALSE;
316 $has_new_scores = FALSE;
317 $queryor = new Condition('OR');
318 foreach ($key as $or) {
319 list($num_new_scores) = $this->parseWord($or);
320 $has_new_scores |= $num_new_scores;
321 $queryor->condition('d.data', "% $or %", 'LIKE');
323 if (count($queryor)) {
324 $this->conditions->condition($queryor);
325 // A group of OR keywords only needs to match once.
326 $this->matches += ($has_new_scores > 0);
329 // Single ANDed term.
332 list($num_new_scores, $num_valid_words) = $this->parseWord($key);
333 $this->conditions->condition('d.data', "% $key %", 'LIKE');
334 if (!$num_valid_words) {
335 $this->simple = FALSE;
337 // Each AND keyword needs to match at least once.
338 $this->matches += $num_new_scores;
341 if ($has_and && $has_or) {
342 $this->simple = FALSE;
346 foreach ($this->keys['negative'] as $key) {
347 $this->conditions->condition('d.data', "% $key %", 'NOT LIKE');
348 $this->simple = FALSE;
353 * Parses a word or phrase for parseQuery().
355 * Splits a phrase into words. Adds its words to $this->words, if it is not
356 * already there. Returns a list containing the number of new words found,
357 * and the total number of words in the phrase.
359 protected function parseWord($word) {
361 $num_valid_words = 0;
363 // Determine the scorewords of this word/phrase.
364 $split = explode(' ', $word);
365 foreach ($split as $s) {
366 $num = is_numeric($s);
367 if ($num || Unicode::strlen($s) >= \Drupal::config('search.settings')->get('index.minimum_word_size')) {
368 if (!isset($this->words[$s])) {
369 $this->words[$s] = $s;
376 // Return matching snippet and number of added words.
377 return [$num_new_scores, $num_valid_words];
381 * Prepares the query and calculates the normalization factor.
383 * After the query is normalized the keywords are weighted to give the results
384 * a relevancy score. The query is ready for execution after this.
386 * Error and warning conditions can apply. Call getStatus() after calling
387 * this method to retrieve them.
390 * TRUE if at least one keyword matched the search index; FALSE if not.
392 public function prepareAndNormalize() {
393 $this->parseSearchExpression();
394 $this->executedPrepare = TRUE;
396 if (count($this->words) == 0) {
397 // Although the query could proceed, there is no point in joining
398 // with other tables and attempting to normalize if there are no
400 $this->status |= SearchQuery::NO_POSITIVE_KEYWORDS;
404 // Build the basic search query: match the entered keywords.
405 $or = new Condition('OR');
406 foreach ($this->words as $word) {
407 $or->condition('i.word', $word);
409 $this->condition($or);
411 // Add keyword normalization information to the query.
412 $this->join('search_total', 't', 'i.word = t.word');
414 ->condition('i.type', $this->type)
418 // If the query is simple, we should have calculated the number of
419 // matching words we need to find, so impose that criterion. For non-
420 // simple queries, this condition could lead to incorrectly deciding not
421 // to continue with the full query.
423 $this->having('COUNT(*) >= :matches', [':matches' => $this->matches]);
426 // Clone the query object to calculate normalization.
427 $normalize_query = clone $this->query;
429 // For complex search queries, add the LIKE conditions; if the query is
430 // simple, we do not need them for normalization.
431 if (!$this->simple) {
432 $normalize_query->join('search_dataset', 'd', 'i.sid = d.sid AND i.type = d.type AND i.langcode = d.langcode');
433 if (count($this->conditions)) {
434 $normalize_query->condition($this->conditions);
438 // Calculate normalization, which is the max of all the search scores for
439 // positive keywords in the query. And note that the query could have other
440 // fields added to it by the user of this extension.
441 $normalize_query->addExpression('SUM(i.score * t.count)', 'calculated_score');
442 $result = $normalize_query
444 ->orderBy('calculated_score', 'DESC')
447 if (isset($result->calculated_score)) {
448 $this->normalize = (float) $result->calculated_score;
451 if ($this->normalize) {
455 // If the normalization value was zero, that indicates there were no
456 // matches to the supplied positive keywords.
457 $this->status |= SearchQuery::NO_KEYWORD_MATCHES;
464 public function preExecute(SelectInterface $query = NULL) {
465 if (!$this->executedPrepare) {
466 $this->prepareAndNormalize();
469 if (!$this->normalize) {
473 return parent::preExecute($query);
477 * Adds a custom score expression to the search query.
479 * Score expressions are used to order search results. If no calls to
480 * addScore() have taken place, a default keyword relevance score will be
481 * used. However, if at least one call to addScore() has taken place, the
482 * keyword relevance score is not automatically added.
484 * Note that you must use this method to add ordering to your searches, and
485 * not call orderBy() directly, when using the SearchQuery extender. This is
486 * because of the two-pass system the SearchQuery class uses to normalize
489 * @param string $score
490 * The score expression, which should evaluate to a number between 0 and 1.
491 * The string 'i.relevance' in a score expression will be replaced by a
492 * measure of keyword relevance between 0 and 1.
493 * @param array $arguments
494 * Query arguments needed to provide values to the score expression.
495 * @param float $multiply
496 * If set, the score is multiplied with this value. However, all scores
497 * with multipliers are then divided by the total of all multipliers, so
498 * that overall, the normalization is maintained.
502 public function addScore($score, $arguments = [], $multiply = FALSE) {
504 $i = count($this->multiply);
505 // Modify the score expression so it is multiplied by the multiplier,
506 // with a divisor to renormalize. Note that the ROUND here is necessary
507 // for PostgreSQL and SQLite in order to ensure that the :multiply_* and
508 // :total_* arguments are treated as a numeric type, because the
509 // PostgreSQL PDO driver sometimes puts values in as strings instead of
510 // numbers in complex expressions like this.
511 $score = "(ROUND(:multiply_$i, 4)) * COALESCE(($score), 0) / (ROUND(:total_$i, 4))";
512 // Add an argument for the multiplier. The :total_$i argument is taken
513 // care of in the execute() method, which is when the total divisor is
515 $arguments[':multiply_' . $i] = $multiply;
516 $this->multiply[] = $multiply;
519 // Search scoring needs a way to include a keyword relevance in the score.
520 // For historical reasons, this is done by putting 'i.relevance' into the
521 // search expression. So, use string replacement to change this to a
522 // calculated query expression, counting the number of occurrences so
523 // in the execute() method we can add arguments.
524 while (($pos = strpos($score, 'i.relevance')) !== FALSE) {
525 $pieces = explode('i.relevance', $score, 2);
526 $score = implode('((ROUND(:normalization_' . $this->relevance_count . ', 4)) * i.score * t.count)', $pieces);
527 $this->relevance_count++;
530 $this->scores[] = $score;
531 $this->scoresArguments += $arguments;
537 * Executes the search.
539 * The complex conditions are applied to the query including score
540 * expressions and ordering.
542 * Error and warning conditions can apply. Call getStatus() after calling
543 * this method to retrieve them.
545 * @return \Drupal\Core\Database\StatementInterface|null
546 * A query result set containing the results of the query.
548 public function execute() {
549 if (!$this->preExecute($this)) {
553 // Add conditions to the query.
554 $this->join('search_dataset', 'd', 'i.sid = d.sid AND i.type = d.type AND i.langcode = d.langcode');
555 if (count($this->conditions)) {
556 $this->condition($this->conditions);
559 // Add default score (keyword relevance) if there are not any defined.
560 if (empty($this->scores)) {
561 $this->addScore('i.relevance');
564 if (count($this->multiply)) {
565 // Re-normalize scores with multipliers by dividing by the total of all
566 // multipliers. The expressions were altered in addScore(), so here just
567 // add the arguments for the total.
568 $sum = array_sum($this->multiply);
569 for ($i = 0; $i < count($this->multiply); $i++) {
570 $this->scoresArguments[':total_' . $i] = $sum;
574 // Add arguments for the keyword relevance normalization number.
575 $normalization = 1.0 / $this->normalize;
576 for ($i = 0; $i < $this->relevance_count; $i++) {
577 $this->scoresArguments[':normalization_' . $i] = $normalization;
580 // Add all scores together to form a query field.
581 $this->addExpression('SUM(' . implode(' + ', $this->scores) . ')', 'calculated_score', $this->scoresArguments);
583 // If an order has not yet been set for this query, add a default order
584 // that sorts by the calculated sum of scores.
585 if (count($this->getOrderBy()) == 0) {
586 $this->orderBy('calculated_score', 'DESC');
589 // Add query metadata.
591 ->addMetaData('normalize', $this->normalize)
592 ->fields('i', ['type', 'sid']);
593 return $this->query->execute();
597 * Builds the default count query for SearchQuery.
599 * Since SearchQuery always uses GROUP BY, we can default to a subquery. We
600 * also add the same conditions as execute() because countQuery() is called
603 public function countQuery() {
604 if (!$this->executedPrepare) {
605 $this->prepareAndNormalize();
608 // Clone the inner query.
609 $inner = clone $this->query;
611 // Add conditions to query.
612 $inner->join('search_dataset', 'd', 'i.sid = d.sid AND i.type = d.type');
613 if (count($this->conditions)) {
614 $inner->condition($this->conditions);
617 // Remove existing fields and expressions, they are not needed for a count
619 $fields =& $inner->getFields();
621 $expressions =& $inner->getExpressions();
624 // Add sid as the only field and count them as a subquery.
625 $count = db_select($inner->fields('i', ['sid']), NULL, ['target' => 'replica']);
627 // Add the COUNT() expression.
628 $count->addExpression('COUNT(*)');
634 * Returns the query status bitmap.
637 * A bitmap indicating query status. Zero indicates there were no problems.
638 * A non-zero value is a combination of one or more of the following flags:
639 * - SearchQuery::NO_POSITIVE_KEYWORDS
640 * - SearchQuery::EXPRESSIONS_IGNORED
641 * - SearchQuery::LOWER_CASE_OR
642 * - SearchQuery::NO_KEYWORD_MATCHES
644 public function getStatus() {
645 return $this->status;