trjhtr

I am putting together a little module for oss release that will let you parse a boolean expression consisting of and/AND/or/OR's (no brackets yet) and output a complete elasticsearch query.

Boolean expression logic:

Right now it uses OR as the basis and puts everything on top of that as ANDs. This means that AND binds left to right.

I lack input on:

1. The quality of the output elasticsearch query - can it be simplified? Are there better approaches?


2. The way I interpret the boolean expression.

def string_to_query(s):
 s = s.lower()
 tokens = [' '.join(t.split()) for t in s.split('or')]
 or_terms = 
 while tokens:
 leaf = tokens.pop()

 and_terms = leaf.split('and')
 if len(and_terms) < 2:
 term = and_terms[0]
 or_terms.extend([
 "match": "Review.Text": 
 "query": term, "operator": "and",
 "match": "Review.Title": 
 "query": term, "operator": "and"
 ])
 else:
 filters = [
 "bool": 
 "should": ["match": "Review.Text": 
 "query": term, "operator": "and",
 "match": "Review.Title": 
 "query": term, "operator": "and"]

 for term in and_terms]
 or_terms.append(
 "bool": 
 "must": filters
 )

 return "query":
 "bool": 
 "should": or_terms
 


query = string_to_query(
 'dog and dog food or cat and cat food'
)

assert query == 
"query": 
 "bool": 
 "should": [
 
 "bool": 
 "must": [
 
 "bool": 
 "should": [
 
 "match": 
 "Review.Text": 
 "operator": "and",
 "query": "cat "
 
 
 ,
 
 "match": 
 "Review.Title": 
 "operator": "and",
 "query": "cat "
 
 
 
 ]
 
 ,
 
 "bool": 
 "should": [
 
 "match": 
 "Review.Text": 
 "operator": "and",
 "query": " cat food"
 
 
 ,
 
 "match": 
 "Review.Title": 
 "operator": "and",
 "query": " cat food"
 
 
 
 ]
 
 
 ]
 
 ,
 
 "bool": 
 "must": [
 
 "bool": 
 "should": [
 
 "match": 
 "Review.Text": 
 "operator": "and",
 "query": "dog "
 
 
 ,
 
 "match": 
 "Review.Title": 
 "operator": "and",
 "query": "dog "
 
 
 
 ]
 
 ,
 
 "bool": 
 "should": [
 
 "match": 
 "Review.Text": 
 "operator": "and",
 "query": " dog food"
 
 
 ,
 
 "match": 
 "Review.Title": 
 "operator": "and",
 "query": " dog food"
 
 
 
 ]
 
 
 ]
 
 
 ]
 

Your usage of split makes your function rather fragile:

>>> string_to_query('doctor and heart')
'query': 'bool': 'should': ['bool': 'must': ['bool': 'should': ['match': 'Review.Text': 'operator': 'and',
 'query': '',
 'match': 'Review.Title': 'operator': 'and',
 'query': ''],
 'bool': 'should': ['match': 'Review.Text': 'operator': 'and',
 'query': ' '
 'heart',
 'match': 'Review.Title': 'operator': 'and',
 'query': ' '
 'heart']],
 'match': 'Review.Text': 'operator': 'and',
 'query': 'doct',
 'match': 'Review.Title': 'operator': 'and',
 'query': 'doct']

Which is equivalent to: "(the empty string AND heart) OR doct" rather than "doctor AND heart".

An other use-case to consider is the use of "and" or "or" as words to search for rather than operators (as "Tom and Jerry", I don't want to search for documents containing "Tom" and "Jerry" separately, but for documents containing the phrase "Tom and Jerry").

Usually, for these kind of problems, an intermediate representation produced by an ad-hoc parser is way better and simpler to convert to the end result. Here I suggest producing a list of lists, since you don't (yet) consider priorisation of clauses using parenthesis. Thus:

[
 [A, B, C],
 [D, E],
 [F],
]

Would be equivalent to "(A and B and C) or (D and E) or F". Which can then easily be converted to the elasticsearch query DSL using simple list comprehensions. The catch, however is that each clause can be complete sentences and must apply to two fields: "Review.Text" and "Review.Title". This is where the multi-match query can simplify the whole writing: each clause A, B, C, D, E, and F would be converted to

'multi_match': 
 'query': clause,
 'type': 'phrase',
 'fields': ['Review.Text', 'Review.Title'],

With all the advantages of the multi-match query such as giving more weight to a single field.

The following rewrite extend the supported syntax to allow double quotes to mean "perfect match":

import re


class ClauseParser:
 def __init__(self, tokenizer, *operators):
 self._tokenizer = tokenizer
 self._operators = set(operators)
 self._found_operator = None

 def __iter__(self):
 for token in self._tokenizer:
 token_value = token.group(0)
 if token.group(2) in self._operators:
 self._found_operator = token_value
 return
 yield token_value

 @property
 def operator(self):
 found_operator = self._found_operator
 self._found_operator = None
 return found_operator


def parser(tokenizer):
 clause_parser = ClauseParser(tokenizer, 'and', 'or')
 current_group = 
 while True:
 current_group.append(' '.join(clause_parser))
 found_operator = clause_parser.operator
 if found_operator != 'and':
 yield current_group
 if found_operator is None:
 return
 current_group = 


def convert_and_clauses(clauses):
 return [
 'multi_match': 
 'query': clause,
 'type': 'phrase',
 'fields': ['Review.Text', 'Review.Title'],
 for clause in clauses
 ]


def string_to_query(phrase):
 tokenizer = re.finditer(r'"([^"]+)"|(w+)', phrase)
 query = list(parser(tokenizer))

 or_clauses = 'bool': 'should': [
 'bool': 'must': convert_and_clauses(clauses)
 for clauses in query
 ]

 return 'query': or_clauses

Example usage:

>>> string_to_query('doctor and heart')
'query': 'bool': 'should': ['bool': 'must': ['multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': 'doctor',
 'type': 'phrase',
 'multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': 'heart',
 'type': 'phrase']]
>>> string_to_query('"Tom and Jerry" or "Road runner and vil coyote"')
'query': 'bool': 'should': ['bool': 'must': ['multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': '"Tom '
 'and '
 'Jerry"',
 'type': 'phrase'],
 'bool': 'must': ['multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': '"Road '
 'runner '
 'and '
 'vil '
 'coyote"',
 'type': 'phrase']]
>>> string_to_query('cat and cat food or dog and dog food')
'query': 'bool': 'should': ['bool': 'must': ['multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': 'cat',
 'type': 'phrase',
 'multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': 'cat '
 'food',
 'type': 'phrase'],
 'bool': 'must': ['multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': 'dog',
 'type': 'phrase',
 'multi_match': 'fields': ['Review.Text',
 'Review.Title'],
 'query': 'dog '
 'food',
 'type': 'phrase']]

I have come up with another solution that yields much simpler results.
It uses query_string searching and the builtin boolean expressions over a set of defined fields:

def string_to_query(s):
 s = s.lower()
 tokens = [' '.join(t.split()) for t in s.split('or')]
 or_terms = 

 while tokens:
 leaf = tokens.pop()

 and_terms = leaf.split('and')
 if len(and_terms) < 2:
 term = and_terms[0]
 or_terms.append('""'.format(term.strip()))
 else:
 and_terms = ['""'.format(term.strip()) for term in and_terms]
 and_string = "( " + " AND ".join(and_terms) + " )"
 or_terms.append(and_string)

 query_string = " OR ".join(or_terms)
 return 
 "query": 
 "query_string": 
 "fields": ["Review.Title", "Review.Text"],
 "query": query_string
 
 
 

query = string_to_query(
 'dog and dog food or cat and cat food'
)

assert query == 
"query": 
 "query_string": 
 "fields": [
 "Review.Title",
 "Review.Text"
 ],
 "query": "( "cat" AND "cat food" ) OR ( "dog" AND "dog food" )"