org.apache.lucene.expressions.js.JavascriptCompiler Java Examples

/** tests the returned sort values are correct */
public void testSortValues() throws Exception {
  Expression expr = JavascriptCompiler.compile("sqrt(_score)");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES);

  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = (float) Math.sqrt(d.score);
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

public void testBoostsAreAppliedLast() throws Exception {

    SimpleBindings bindings = new SimpleBindings();
    bindings.add("score", DoubleValuesSource.SCORES);
    Expression expr = JavascriptCompiler.compile("ln(score + 4)");

    Query q1 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "text")), expr.getDoubleValuesSource(bindings));
    TopDocs plain = searcher.search(q1, 5);

    Query boosted = new BoostQuery(q1, 2);
    TopDocs afterboost = searcher.search(boosted, 5);
    assertEquals(plain.totalHits.value, afterboost.totalHits.value);
    for (int i = 0; i < 5; i++) {
      assertEquals(plain.scoreDocs[i].doc, afterboost.scoreDocs[i].doc);
      assertEquals(plain.scoreDocs[i].score, afterboost.scoreDocs[i].score / 2, 0.0001);
    }

  }
 

public void testAccessToValueSource() throws Exception {

    FunctionScoreQuery q1 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "a")), DoubleValuesSource.constant(31));
    Query q2 = new FunctionScoreQuery(q1.getWrappedQuery(), q1.getSource());
    QueryUtils.check(q2);
    QueryUtils.checkEqual(q2, q1);

    FunctionScoreQuery q3 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "first")),
            DoubleValuesSource.fromIntField(INT_FIELD));
    Query q4 = new FunctionScoreQuery(q3.getWrappedQuery(), q3.getSource());
    QueryUtils.checkEqual(q3, q4);

    SimpleBindings bindings = new SimpleBindings();
    bindings.add("score", DoubleValuesSource.SCORES);
    Expression expr = JavascriptCompiler.compile("ln(score + 4)");
    FunctionScoreQuery q5 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "text")), expr.getDoubleValuesSource(bindings));
    Query q6 = new FunctionScoreQuery(q5.getWrappedQuery(), q5.getSource());
    QueryUtils.checkEqual(q5, q6);


  }
 

/** tests same binding used more than once in an expression */
public void testTwoOfSameBinding() throws Exception {
  Expression expr = JavascriptCompiler.compile("_score + _score");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES);
  
  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = 2*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

/** Uses variables with $ */
public void testDollarVariable() throws Exception {
  Expression expr = JavascriptCompiler.compile("$0+$score");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("$0", DoubleValuesSource.SCORES);
  bindings.add("$score", DoubleValuesSource.SCORES);

  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = 2*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

/** tests expression referring to another expression */
public void testExpressionRefersToExpression() throws Exception {
  Expression expr1 = JavascriptCompiler.compile("_score");
  Expression expr2 = JavascriptCompiler.compile("2*expr1");
  
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES);
  bindings.add("expr1", expr1);
  
  Sort sort = new Sort(expr2.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = 2*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

private void doTestLotsOfBindings(int n) throws Exception {
  SimpleBindings bindings = new SimpleBindings();    
  StringBuilder sb = new StringBuilder();
  for (int i = 0; i < n; i++) {
    if (i > 0) {
      sb.append("+");
    }
    sb.append("x" + i);
    bindings.add("x" + i, DoubleValuesSource.SCORES);
  }
  
  Expression expr = JavascriptCompiler.compile(sb.toString());
  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  TopFieldDocs td = searcher.search(query, 3, sort, true);
  for (int i = 0; i < 3; i++) {
    FieldDoc d = (FieldDoc) td.scoreDocs[i];
    float expected = n*d.score;
    float actual = ((Double)d.fields[0]).floatValue();
    assertEquals(expected, actual, 0d);
  }
}
 

public void testDoubleValuesSourceTypes() throws Exception {
  Expression expr = JavascriptCompiler.compile("2*popularity + count");
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("popularity", DoubleValuesSource.fromLongField("popularity"));
  bindings.add("count", DoubleValuesSource.fromLongField("count"));
  DoubleValuesSource vs = expr.getDoubleValuesSource(bindings);

  assertEquals(1, reader.leaves().size());
  LeafReaderContext leaf = reader.leaves().get(0);
  DoubleValues values = vs.getValues(leaf, null);

  assertTrue(values.advanceExact(0));
  assertEquals(10, values.doubleValue(), 0);
  assertTrue(values.advanceExact(1));
  assertEquals(41, values.doubleValue(), 0);
  assertTrue(values.advanceExact(2));
  assertEquals(4, values.doubleValue(), 0);
}
 

void assertQuery(Query query, Sort sort) throws Exception {
  int size = TestUtil.nextInt(random(), 1, searcher.getIndexReader().maxDoc() / 5);
  TopDocs expected = searcher.search(query, size, sort, random().nextBoolean());
  
  // make our actual sort, mutating original by replacing some of the 
  // sortfields with equivalent expressions
  
  SortField original[] = sort.getSort();
  SortField mutated[] = new SortField[original.length];
  for (int i = 0; i < mutated.length; i++) {
    if (random().nextInt(3) > 0) {
      SortField s = original[i];
      Expression expr = JavascriptCompiler.compile(s.getField());
      SimpleBindings simpleBindings = new SimpleBindings();
      simpleBindings.add(s.getField(), fromSortField(s));
      boolean reverse = s.getType() == SortField.Type.SCORE || s.getReverse();
      mutated[i] = expr.getSortField(simpleBindings, reverse);
    } else {
      mutated[i] = original[i];
    }
  }
  
  Sort mutatedSort = new Sort(mutated);
  TopDocs actual = searcher.search(query, size, mutatedSort, random().nextBoolean());
  CheckHits.checkEqual(query, expected.scoreDocs, actual.scoreDocs);
  
  if (size < actual.totalHits.value) {
    expected = searcher.searchAfter(expected.scoreDocs[size-1], query, size, sort);
    actual = searcher.searchAfter(actual.scoreDocs[size-1], query, size, mutatedSort);
    CheckHits.checkEqual(query, expected.scoreDocs, actual.scoreDocs);
  }
}
 

public void testDoubleValuesSourceEquals() throws Exception {
  Expression expr = JavascriptCompiler.compile("sqrt(a) + ln(b)");

  SimpleBindings bindings = new SimpleBindings();
  bindings.add("a", DoubleValuesSource.fromIntField("a"));
  bindings.add("b", DoubleValuesSource.fromIntField("b"));

  DoubleValuesSource vs1 = expr.getDoubleValuesSource(bindings);
  // same instance
  assertEquals(vs1, vs1);
  // null
  assertFalse(vs1.equals(null));
  // other object
  assertFalse(vs1.equals("foobar"));
  // same bindings and expression instances
  DoubleValuesSource vs2 = expr.getDoubleValuesSource(bindings);
  assertEquals(vs1.hashCode(), vs2.hashCode());
  assertEquals(vs1, vs2);
  // equiv bindings (different instance)
  SimpleBindings bindings2 = new SimpleBindings();
  bindings2.add("a", DoubleValuesSource.fromIntField("a"));
  bindings2.add("b", DoubleValuesSource.fromIntField("b"));
  DoubleValuesSource vs3 = expr.getDoubleValuesSource(bindings2);
  assertEquals(vs1, vs3);
  // different bindings (same names, different types)
  SimpleBindings bindings3 = new SimpleBindings();
  bindings3.add("a", DoubleValuesSource.fromLongField("a"));
  bindings3.add("b", DoubleValuesSource.fromFloatField("b"));
  DoubleValuesSource vs4 = expr.getDoubleValuesSource(bindings3);
  assertFalse(vs1.equals(vs4));
}
 

public void testRewrite() throws Exception {
  Expression expr = JavascriptCompiler.compile("a");

  ExpressionValueSource rewritingExpressionSource = new ExpressionValueSource(
          new DoubleValuesSource[]{createDoubleValuesSourceMock(true)},
          expr,
          false);
  ExpressionValueSource notRewritingExpressionSource = new ExpressionValueSource(
          new DoubleValuesSource[]{createDoubleValuesSourceMock(false)},
          expr,
          false);

  assertNotSame(rewritingExpressionSource, rewritingExpressionSource.rewrite(null));
  assertSame(notRewritingExpressionSource, notRewritingExpressionSource.rewrite(null));
}
 

public void testCoRecursion4() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("cycle0", JavascriptCompiler.compile("100"));
  bindings.add("cycle1", JavascriptCompiler.compile("100"));
  bindings.add("cycle2", JavascriptCompiler.compile("cycle1 + cycle0 + cycle3"));
  bindings.add("cycle3", JavascriptCompiler.compile("cycle0 + cycle1 + cycle2"));
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    bindings.validate();
  });
  assertTrue(expected.getMessage().contains("Cycle detected"));
}
 

public LongValuesSource fromExpression(String weightExpression, Set<SortField> sortFields) {
  Expression expression = null;
  try {
    expression = JavascriptCompiler.compile(weightExpression);
  } catch (ParseException e) {
    throw new RuntimeException(e);
  }
  SimpleBindings bindings = new SimpleBindings();
  for (SortField sortField : sortFields) {
    bindings.add(sortField.getField(), fromSortField(sortField));
  }
  return expression.getDoubleValuesSource(bindings).toLongValuesSource();
}
 

/** static helper for re-use in sibling trie class */
public static SortField getSortField(final SortField superSort, final SchemaField field) {
  field.checkSortability();
  Expression expr = null;
  try {
    expr = JavascriptCompiler.compile(field.getName() + " % 3");
  } catch (Exception e) {
    throw new RuntimeException("impossible?", e);
  }
  SimpleBindings bindings = new SimpleBindings();
  bindings.add(superSort.getField(), fromSortField(superSort));
  return expr.getSortField(bindings, superSort.getReverse());
}
 

public static Object compile(String scriptSource) {
    // classloader created here
    final SecurityManager sm = System.getSecurityManager();
    if (sm != null) {
        sm.checkPermission(new SpecialPermission());
    }
    return AccessController.doPrivileged(new PrivilegedAction<Expression>() {
        @Override
        public Expression run() {
            try {
                // snapshot our context here, we check on behalf of the expression
                AccessControlContext engineContext = AccessController.getContext();
                ClassLoader loader = getClass().getClassLoader();
                if (sm != null) {
                    loader = new ClassLoader(loader) {
                        @Override
                        protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
                            try {
                                engineContext.checkPermission(new ClassPermission(name));
                            } catch (SecurityException e) {
                                throw new ClassNotFoundException(name, e);
                            }
                            return super.loadClass(name, resolve);
                        }
                    };
                }
                // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here
                return JavascriptCompiler.compile(scriptSource, JavascriptCompiler.DEFAULT_FUNCTIONS, loader);
            } catch (ParseException e) {
                throw convertToScriptException("compile error", scriptSource, scriptSource, e);
            }
        }
    });
}
 

private static DoubleValuesSource buildProductBoostFields() {
    try {
        SimpleBindings bindings = new SimpleBindings();
        bindings.add("score", DoubleValuesSource.SCORES);
        bindings.add("category_boost", DoubleValuesSource.fromIntField(ProductSearchQueryBuilder.PRODUCT_CATEGORY_FIELD + "_boost"));
        bindings.add("instock_boost", DoubleValuesSource.fromIntField(ProductSearchQueryBuilder.PRODUCT_SHOP_INSTOCK_FIELD + "_boost"));
        bindings.add("feature_boost", DoubleValuesSource.fromIntField("featured_boost"));
        return JavascriptCompiler.compile("score * (feature_boost + instock_boost + category_boost)").getDoubleValuesSource(bindings);
    } catch (Exception exp) {
        throw new RuntimeException("Unable to compile PRODUCT_BOOST_FIELDS");
    }
}
 

private static DoubleValuesSource buildProductSkuBoostFields() {
    try {
        SimpleBindings bindings = new SimpleBindings();
        bindings.add("score", DoubleValuesSource.SCORES);
        bindings.add("rank_boost", DoubleValuesSource.fromIntField("rank_boost"));
        bindings.add("instock_boost", DoubleValuesSource.fromIntField(ProductSearchQueryBuilder.PRODUCT_SHOP_INSTOCK_FIELD + "_boost"));
        bindings.add("feature_boost", DoubleValuesSource.fromIntField("featured_boost"));
        return JavascriptCompiler.compile("score * (feature_boost + instock_boost + rank_boost)").getDoubleValuesSource(bindings);
    } catch (Exception exp) {
        throw new RuntimeException("Unable to compile SKU_BOOST_FIELDS");
    }
}
 

/** an example of how to rank by an expression */
public void test() throws Exception {
  // compile an expression:
  Expression expr = JavascriptCompiler.compile("sqrt(_score) + ln(popularity)");
  
  // we use SimpleBindings: which just maps variables to SortField instances
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES);
  bindings.add("popularity", DoubleValuesSource.fromIntField("popularity"));

  // create a sort field and sort by it (reverse order)
  Sort sort = new Sort(expr.getSortField(bindings, true));
  Query query = new TermQuery(new Term("body", "contents"));
  searcher.search(query, 3, sort);
}
 

/** User runs a query and aggregates facets. */
private FacetResult search() throws IOException, ParseException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  // Aggregate categories by an expression that combines the document's score
  // and its popularity field
  Expression expr = JavascriptCompiler.compile("_score * sqrt(popularity)");
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("_score", DoubleValuesSource.SCORES); // the score of the document
  bindings.add("popularity", DoubleValuesSource.fromLongField("popularity")); // the value of the 'popularity' field

  // Aggregates the facet values
  FacetsCollector fc = new FacetsCollector(true);

  // MatchAllDocsQuery is for "browsing" (counts facets
  // for all non-deleted docs in the index); normally
  // you'd use a "normal" query:
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);

  // Retrieve results
  Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, fc, expr.getDoubleValuesSource(bindings));
  FacetResult result = facets.getTopChildren(10, "A");
  
  indexReader.close();
  taxoReader.close();
  
  return result;
}
 

public void testCoRecursion3() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("cycle0", JavascriptCompiler.compile("100"));
  bindings.add("cycle1", JavascriptCompiler.compile("cycle0 + cycle2"));
  bindings.add("cycle2", JavascriptCompiler.compile("cycle0 + cycle1"));
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    bindings.validate();
  });
  assertTrue(expected.getMessage().contains("Cycle detected"));
}
 

public void testCoRecursion2() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("cycle0", JavascriptCompiler.compile("cycle1"));
  bindings.add("cycle1", JavascriptCompiler.compile("cycle2"));
  bindings.add("cycle2", JavascriptCompiler.compile("cycle0"));
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    bindings.validate();
  });
  assertTrue(expected.getMessage().contains("Cycle detected"));
}
 

public void testCoRecursion() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("cycle0", JavascriptCompiler.compile("cycle1"));
  bindings.add("cycle1", JavascriptCompiler.compile("cycle0"));
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    bindings.validate();
  });
  assertTrue(expected.getMessage().contains("Cycle detected"));
}
 

public void testSelfRecursion() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("cycle0", JavascriptCompiler.compile("cycle0"));
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    bindings.validate();
  });
  assertTrue(expected.getMessage().contains("Cycle detected"));
}
 

public void testInvalidExternal2() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("valid", DoubleValuesSource.fromIntField("valid"));
  bindings.add("invalid", JavascriptCompiler.compile("valid + badreference"));
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    bindings.validate();
  });
  assertTrue(expected.getMessage().contains("Invalid reference"));
}
 

public void testInvalidExternal() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("valid", DoubleValuesSource.fromIntField("valid"));
  bindings.add("invalid", JavascriptCompiler.compile("badreference"));
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    bindings.validate();
  });
  assertTrue(expected.getMessage().contains("Invalid reference"));
}
 

public void testValidExternals() throws Exception {
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("valid0", DoubleValuesSource.fromIntField("valid0"));
  bindings.add("valid1", DoubleValuesSource.fromIntField("valid1"));
  bindings.add("valid2", DoubleValuesSource.fromIntField("valid2"));
  bindings.add("_score", DoubleValuesSource.SCORES);
  bindings.add("valide0", JavascriptCompiler.compile("valid0 - valid1 + valid2 + _score"));
  bindings.validate();
  bindings.add("valide1", JavascriptCompiler.compile("valide0 + valid0"));
  bindings.validate();
  bindings.add("valide2", JavascriptCompiler.compile("valide0 * valide1"));
  bindings.validate();
}
 

public void testBasic() throws Exception {

    // create a sort field and sort by it (reverse order)
    Query query = new TermQuery(new Term("body", "contents"));
    IndexReader r = searcher.getIndexReader();

    // Just first pass query
    TopDocs hits = searcher.search(query, 10);
    assertEquals(3, hits.totalHits.value);
    assertEquals("3", r.document(hits.scoreDocs[0].doc).get("id"));
    assertEquals("1", r.document(hits.scoreDocs[1].doc).get("id"));
    assertEquals("2", r.document(hits.scoreDocs[2].doc).get("id"));

    // Now, rescore:

    Expression e = JavascriptCompiler.compile("sqrt(_score) + ln(popularity)");
    SimpleBindings bindings = new SimpleBindings();
    bindings.add("popularity", DoubleValuesSource.fromIntField("popularity"));
    bindings.add("_score", DoubleValuesSource.SCORES);
    Rescorer rescorer = e.getRescorer(bindings);

    hits = rescorer.rescore(searcher, hits, 10);
    assertEquals(3, hits.totalHits.value);
    assertEquals("2", r.document(hits.scoreDocs[0].doc).get("id"));
    assertEquals("1", r.document(hits.scoreDocs[1].doc).get("id"));
    assertEquals("3", r.document(hits.scoreDocs[2].doc).get("id"));

    String expl = rescorer.explain(searcher,
                                   searcher.explain(query, hits.scoreDocs[0].doc),
                                   hits.scoreDocs[0].doc).toString();

    // Confirm the explanation breaks out the individual
    // variables:
    assertTrue(expl.contains("= double(popularity)"));

    // Confirm the explanation includes first pass details:
    assertTrue(expl.contains("= first pass score"));
    assertTrue(expl.contains("body:contents in"));
  }
 

public void testEquals() throws Exception {
  Expression expr = JavascriptCompiler.compile("sqrt(_score) + ln(popularity)");
  
  SimpleBindings bindings = new SimpleBindings();    
  bindings.add("_score", DoubleValuesSource.SCORES);
  bindings.add("popularity", DoubleValuesSource.fromIntField("popularity"));
  
  SimpleBindings otherBindings = new SimpleBindings();
  otherBindings.add("_score", DoubleValuesSource.fromLongField("_score"));
  otherBindings.add("popularity", DoubleValuesSource.fromIntField("popularity"));

  SortField sf1 = expr.getSortField(bindings, true);
  
  // different order
  SortField sf2 = expr.getSortField(bindings, false);
  assertFalse(sf1.equals(sf2));
  
  // different bindings
  sf2 = expr.getSortField(otherBindings, true);
  assertFalse(sf1.equals(sf2));
  
  // different expression
  Expression other = JavascriptCompiler.compile("popularity/2");
  sf2 = other.getSortField(bindings, true);
  assertFalse(sf1.equals(sf2));
  
  // null
  assertFalse(sf1.equals(null));
  
  // same instance:
  assertEquals(sf1, sf1);
}
 

public void testToString() throws Exception {
  Expression expr = JavascriptCompiler.compile("sqrt(_score) + ln(popularity)");
  
  SimpleBindings bindings = new SimpleBindings();    
  bindings.add("_score", DoubleValuesSource.SCORES);
  bindings.add("popularity", DoubleValuesSource.fromIntField("popularity"));
  
  SortField sf = expr.getSortField(bindings, true);
  assertEquals("<expr(sqrt(_score) + ln(popularity))>!", sf.toString());
}
 

private DoubleValuesSource getDistanceValueSource() {
  Expression distance;
  try {
    distance = JavascriptCompiler.compile(
                "haversin(" + ORIGIN_LATITUDE + "," + ORIGIN_LONGITUDE + ",latitude,longitude)");
  } catch (ParseException pe) {
    // Should not happen
    throw new RuntimeException(pe);
  }
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("latitude", DoubleValuesSource.fromDoubleField("latitude"));
  bindings.add("longitude", DoubleValuesSource.fromDoubleField("longitude"));

  return distance.getDoubleValuesSource(bindings);
}