caio.co/de/cerberus

Add support for negation and phrase queries

This patch adds some smartness to the processing of the fulltext()
field of a SearchQuery. Namely:

* Tokens that start with a `-` are transformed into a NOT query (as
  opposed to the default SHOULD)
* Text between double quotes (`"`) is interpreted as a phrase
  query. So `this query` and `"this query"` are essentially different
Id
d496d002f5c20918c1c707bffb2c99a6eb948d70
Author
Caio
Commit time
2018-12-05T13:47:46+01:00

Modified pom.xml

@@ -67,6 +67,12
</dependency>

<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.immutables</groupId>
<artifactId>value</artifactId>
<version>${immutables.version}</version>

Modified src/main/java/co/caio/cerberus/search/QueryInterpreter.java

@@ -27,26 +27,28
import org.slf4j.LoggerFactory;

class QueryInterpreter {
+ private static final Logger logger = LoggerFactory.getLogger(QueryInterpreter.class);
private final Analyzer analyzer;
private final FacetsConfig facetsConfig;
- private static final Logger logger = LoggerFactory.getLogger(QueryInterpreter.class);
private final MoreLikeThis moreLikeThis;
+ private final FulltextQueryParser queryParser;

QueryInterpreter(MoreLikeThis mlt) {
analyzer = new StandardAnalyzer();
facetsConfig = IndexConfiguration.getFacetsConfig();
moreLikeThis = mlt;
+
+ queryParser = new FulltextQueryParser(analyzer);
}

Query toLuceneQuery(SearchQuery searchQuery) throws IOException {
var queryBuilder = new BooleanQuery.Builder();

if (searchQuery.fulltext().isPresent()) {
- addTermQueries(
- queryBuilder, FULLTEXT, searchQuery.fulltext().get(), BooleanClause.Occur.MUST);
+ queryBuilder.add(queryParser.parse(searchQuery.fulltext().get()), BooleanClause.Occur.MUST);
} else if (searchQuery.similarity().isPresent()) {
queryBuilder.add(
- moreLikeThis.like(IndexField.FULLTEXT, new StringReader(searchQuery.similarity().get())),
+ moreLikeThis.like(FULLTEXT, new StringReader(searchQuery.similarity().get())),
BooleanClause.Occur.MUST);
}

Modified src/test/java/co/caio/cerberus/search/SearcherTest.java

@@ -249,7 +249,7
}

@Test
- void basicSorting() throws Exception {
+ void basicSorting() {
var queryBuilder =
new SearchQuery.Builder().totalTime(SearchQuery.RangedSpec.of(10, 25)).maxResults(50);

@@ -407,5 +407,42
}
}
}
+ }
+
+ @Test
+ void fulltextWithNotQuery() {
+ var builder = new SearchQuery.Builder();
+
+ var withOil = searcher.search(builder.fulltext("oil").build());
+ var withoutOil = searcher.search(builder.fulltext("-oil").build());
+
+ // We expect that the result of searching for documents
+ // matching a term PLUS the results of searching for docs
+ // that do NOT match the same term ends up hitting every
+ // document in the index
+ assertEquals(299, withOil.totalHits() + withoutOil.totalHits());
+
+ // Same for phrases
+ var withYam = searcher.search(builder.fulltext("\"sweet potato\"").build());
+ var withoutYam = searcher.search(builder.fulltext("-\"sweet potato\"").build());
+ assertEquals(299, withOil.totalHits() + withoutOil.totalHits());
+ }
+
+ @Test
+ void phraseQuery() {
+ var builder = new SearchQuery.Builder();
+
+ var termResult = searcher.search(builder.fulltext("sweet potato").build());
+ var phraseResult = searcher.search(builder.fulltext("\"sweet potato\"").build());
+
+ // A phrase query is a LOT more specific than a term-based one
+ assertTrue(termResult.totalHits() > phraseResult.totalHits());
+
+ var termAndPhraseResult =
+ searcher.search(builder.fulltext("sweet \"sweet potato\" potato").build());
+
+ // But querying for a phrase and its terms should match the name
+ // number of documents as just querying for its terms
+ assertEquals(termResult.totalHits(), termAndPhraseResult.totalHits());
}
}

Created src/main/java/co/caio/cerberus/search/FulltextQueryParser.java

@@ -1,0 +1,24
+package co.caio.cerberus.search;
+
+import java.util.Collections;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.queryparser.simple.SimpleQueryParser;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+
+class FulltextQueryParser extends SimpleQueryParser {
+
+ private static final int FEATURES = NOT_OPERATOR | PHRASE_OPERATOR | WHITESPACE_OPERATOR;
+
+ FulltextQueryParser(Analyzer analyzer) {
+ super(analyzer, Collections.singletonMap(IndexField.FULLTEXT, 1f), FEATURES);
+ }
+
+ @Override
+ public Query parse(String input) {
+ if (input.equals("*")) {
+ return new MatchNoDocsQuery("MatchAllDocsQuery() not allowed");
+ }
+ return super.parse(input);
+ }
+}

Created src/test/java/co/caio/cerberus/search/FulltextQueryParserTest.java

@@ -1,0 +1,61
+package co.caio.cerberus.search;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class FulltextQueryParserTest {
+
+ private static FulltextQueryParser parser;
+
+ @BeforeAll
+ static void init() {
+ parser = new FulltextQueryParser(new StandardAnalyzer());
+ }
+
+ @Test
+ void simpleTermQuery() {
+ var query = parser.parse("oil");
+ assertEquals("fulltext:oil", query.toString());
+ }
+
+ @Test
+ void simplePhraseQuery() {
+ var query = parser.parse("\"phrase query\"");
+ assertEquals("fulltext:\"phrase query\"", query.toString());
+ }
+
+ @Test
+ void simpleTermNegation() {
+ var query = parser.parse("-oil");
+ // NOT queries are encoded as (NOT:term AND match_all_docs)
+ assertEquals("-fulltext:oil *:*", query.toString());
+ }
+
+ @Test
+ void simplePhraseNegation() {
+ var query = parser.parse("-\"black pepper\"");
+ assertEquals("-fulltext:\"black pepper\" *:*", query.toString());
+ }
+
+ @Test
+ void implicitGrouping() {
+ var query = parser.parse("-oil salt"); // with salt, without oil
+ assertEquals("(-fulltext:oil *:*) fulltext:salt", query.toString());
+ }
+
+ @Test
+ void disabledAllDocsMatch() {
+ var query = parser.parse("*:*");
+ assertNotEquals("*:*", query.toString());
+ assertEquals(MatchNoDocsQuery.class, query.getClass());
+
+ query = parser.parse("*");
+ assertNotEquals("*:*", query.toString());
+ assertEquals(MatchNoDocsQuery.class, query.getClass());
+ }
+}