org.apache.lucene.search.DoubleValues Java Examples

public void testFeatureMissingFeatureNameInSegment() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  doc.add(new FeatureField("field", "different_name", 0.5F));
  writer.addDocument(doc);
  writer.commit();
  IndexReader ir = writer.getReader();
  writer.close();
  
  assertEquals(1, ir.leaves().size());
  LeafReaderContext context = ir.leaves().get(0);
  DoubleValuesSource valuesSource = FeatureField.newDoubleValues("field", "name");
  DoubleValues values = valuesSource.getValues(context, null);

  assertFalse(values.advanceExact(0));
  assertFalse(values.advanceExact(1));

  ir.close();
  dir.close();
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  Scorer in = inner.scorer(context);
  if (in == null)
    return null;
  DoubleValues scores = valueSource.getValues(context, DoubleValuesSource.fromScorer(in));
  return new FilterScorer(in) {
    @Override
    public float score() throws IOException {
      if (scores.advanceExact(docID())) {
        double factor = scores.doubleValue();
        if (factor >= 0) {
          return (float) (factor * boost);
        }
      }
      // default: missing value, negative value or NaN
      return 0;
    }
    @Override
    public float getMaxScore(int upTo) throws IOException {
      return Float.POSITIVE_INFINITY;
    }
  };
}
 

private static DoubleValues scores(MatchingDocs hits) {
  return new DoubleValues() {

    int index = -1;

    @Override
    public double doubleValue() throws IOException {
      return hits.scores[index];
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      index++;
      return true;
    }
  };
}
 

private void sumValues(List<MatchingDocs> matchingDocs, boolean keepScores, DoubleValuesSource valueSource) throws IOException {

    IntsRef scratch = new IntsRef();
    for(MatchingDocs hits : matchingDocs) {
      OrdinalsReader.OrdinalsSegmentReader ords = ordinalsReader.getReader(hits.context);
      DoubleValues scores = keepScores ? scores(hits) : null;
      DoubleValues functionValues = valueSource.getValues(hits.context, scores);
      DocIdSetIterator docs = hits.bits.iterator();
      
      int doc;
      while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
        ords.get(doc, scratch);
        if (functionValues.advanceExact(doc)) {
          float value = (float) functionValues.doubleValue();
          for (int i = 0; i < scratch.length; i++) {
            values[scratch.ints[i]] += value;
          }
        }
      }
    }

    rollup();
  }
 

public void testDoubleValuesSourceTypes() throws Exception {
  Expression expr = JavascriptCompiler.compile("2*popularity + count");
  SimpleBindings bindings = new SimpleBindings();
  bindings.add("popularity", DoubleValuesSource.fromLongField("popularity"));
  bindings.add("count", DoubleValuesSource.fromLongField("count"));
  DoubleValuesSource vs = expr.getDoubleValuesSource(bindings);

  assertEquals(1, reader.leaves().size());
  LeafReaderContext leaf = reader.leaves().get(0);
  DoubleValues values = vs.getValues(leaf, null);

  assertTrue(values.advanceExact(0));
  assertEquals(10, values.doubleValue(), 0);
  assertTrue(values.advanceExact(1));
  assertEquals(41, values.doubleValue(), 0);
  assertTrue(values.advanceExact(2));
  assertEquals(4, values.doubleValue(), 0);
}
 

@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
  Terms terms = ctx.reader().terms(term.field());
  TermsEnum te = terms == null ? null : terms.iterator();

  if (te == null || te.seekExact(term.bytes()) == false) {
    return DoubleValues.EMPTY;
  }

  final PostingsEnum pe = te.postings(null);
  assert pe != null;

  return new DoubleValues() {
    @Override
    public double doubleValue() throws IOException {
      return pe.freq();
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      if (pe.docID() > doc)
        return false;
      return pe.docID() == doc || pe.advance(doc) == doc;
    }
  };
}
 

@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
  Map<String, DoubleValues> valuesCache = new HashMap<>();
  DoubleValues[] externalValues = new DoubleValues[expression.variables.length];

  for (int i = 0; i < variables.length; ++i) {
    String externalName = expression.variables[i];
    DoubleValues values = valuesCache.get(externalName);
    if (values == null) {
      values = variables[i].getValues(readerContext, scores);
      if (values == null) {
        throw new RuntimeException("Internal error. External (" + externalName + ") does not exist.");
      }
      valuesCache.put(externalName, values);
    }
    externalValues[i] = zeroWhenUnpositioned(values);
  }

  return new ExpressionFunctionValues(expression, externalValues);
}
 

public void testFeatureMissingFieldInSegment() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  writer.addDocument(doc);
  writer.commit();
  IndexReader ir = writer.getReader();
  writer.close();
  
  assertEquals(1, ir.leaves().size());
  LeafReaderContext context = ir.leaves().get(0);
  DoubleValuesSource valuesSource = FeatureField.newDoubleValues("field", "name");
  DoubleValues values = valuesSource.getValues(context, null);

  assertFalse(values.advanceExact(0));
  assertFalse(values.advanceExact(1));

  ir.close();
  dir.close();
}
 

@Override
public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
  Map<Object, Object> context = new IdentityHashMap<>();
  ScoreAndDoc scorer = new ScoreAndDoc();
  context.put("scorer", scorer);
  final FunctionValues fv = in.getValues(context, ctx);
  return new LongValues() {

    @Override
    public long longValue() throws IOException {
      return fv.longVal(scorer.current);
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      scorer.current = doc;
      if (scores != null && scores.advanceExact(doc))
        scorer.score = (float) scores.doubleValue();
      else
        scorer.score = 0;
      return fv.exists(doc);
    }
  };
}
 

@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {

  final ShapeValues shapeValues = shapeValueSource.getValues(readerContext);

  return DoubleValues.withDefault(new DoubleValues() {
    @Override
    public double doubleValue() throws IOException {
      return distCalc.distance(queryPoint, shapeValues.value().getCenter()) * multiplier;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return shapeValues.advanceExact(doc);
    }
  }, nullValue);
}
 

/**
 * Returns the FunctionValues used by the function query.
 */
@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {

  final DoubleValues ptX = DoubleValuesSource.fromDoubleField(strategy.getFieldNameX()).getValues(readerContext, null);
  final DoubleValues ptY = DoubleValuesSource.fromDoubleField(strategy.getFieldNameY()).getValues(readerContext, null);
  final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();

  return DoubleValues.withDefault(new DoubleValues() {

    @Override
    public double doubleValue() throws IOException {
      return calculator.distance(from, ptX.doubleValue(), ptY.doubleValue()) * multiplier;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return ptX.advanceExact(doc) && ptY.advanceExact(doc);
    }
  }, nullValue);

}
 

@Override
public ShapeValues getValues(LeafReaderContext readerContext) throws IOException {

  final DoubleValues minX = DoubleValuesSource.fromDoubleField(strategy.field_minX).getValues(readerContext, null);
  final DoubleValues minY = DoubleValuesSource.fromDoubleField(strategy.field_minY).getValues(readerContext, null);
  final DoubleValues maxX = DoubleValuesSource.fromDoubleField(strategy.field_maxX).getValues(readerContext, null);
  final DoubleValues maxY = DoubleValuesSource.fromDoubleField(strategy.field_maxY).getValues(readerContext, null);

  //reused
  final Rectangle rect = strategy.getSpatialContext().makeRectangle(0,0,0,0);

  return new ShapeValues() {

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return minX.advanceExact(doc) && maxX.advanceExact(doc) && minY.advanceExact(doc) && maxY.advanceExact(doc);
    }

    @Override
    public Shape value() throws IOException {
      rect.reset(minX.doubleValue(), maxX.doubleValue(), minY.doubleValue(), maxY.doubleValue());
      return rect;
    }

  };
}
 

@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
  return new DoubleValues() {

    @SuppressWarnings("unchecked")
    final FieldComparator<Double> comparator =
        (FieldComparator<Double>) getSortField(false).getComparator(1, 1);
    final LeafFieldComparator leafComparator = comparator.getLeafComparator(ctx);
    final double mult = multiplier; // so it's a local field

    double value = Double.POSITIVE_INFINITY;

    @Override
    public double doubleValue() throws IOException {
      return value;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      leafComparator.copy(0, doc);
      value = comparator.value(0) * mult;
      return true;
    }
  };
}
 

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
    Bindings bindings = new Bindings(){
        @Override
        public DoubleValuesSource getDoubleValuesSource(String name) {
            Double queryParamValue  = queryParamValues.get(name);
            if (queryParamValue != null) {
                return DoubleValuesSource.constant(queryParamValue);
            }
            return new FVDoubleValuesSource(vectorSupplier, features.featureOrdinal(name));
        }
    };

    DocIdSetIterator iterator = DocIdSetIterator.all(context.reader().maxDoc());
    DoubleValuesSource src = expression.getDoubleValuesSource(bindings);
    DoubleValues values = src.getValues(context, null);

    return new DValScorer(this, iterator, values);
}
 

private static DoubleValues zeroWhenUnpositioned(DoubleValues in) {
  return new DoubleValues() {

    boolean positioned = false;

    @Override
    public double doubleValue() throws IOException {
      return positioned ? in.doubleValue() : 0;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return positioned = in.advanceExact(doc);
    }
  };
}
 

@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {

  final ShapeValues shapeValues = bboxValueSource.getValues(readerContext);
  return DoubleValues.withDefault(new DoubleValues() {
    @Override
    public double doubleValue() throws IOException {
      return score((Rectangle) shapeValues.value(), null);
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return shapeValues.advanceExact(doc);
    }
  }, 0);

}
 

ExpressionFunctionValues(Expression expression, DoubleValues[] functionValues) {
  if (expression == null) {
    throw new NullPointerException();
  }
  if (functionValues == null) {
    throw new NullPointerException();
  }
  this.expression = expression;
  this.functionValues = functionValues;
}
 

@Override
public boolean advanceExact(int doc) throws IOException {
  for (DoubleValues v : functionValues) {
    v.advanceExact(doc);
  }
  return true;
}
 

@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
    return new DoubleValues() {
        @Override
        public double doubleValue() throws IOException {
            assert vectorSupplier.get() != null;
            return vectorSupplier.get().getFeatureScore(ordinal);
        }

        @Override
        public boolean advanceExact(int doc) throws IOException {
            return true;
        }
    };
}
 

@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
  Terms terms = ctx.reader().terms(field);
  if (terms == null) {
    return DoubleValues.EMPTY;
  } else {
    TermsEnum termsEnum = terms.iterator();
    if (termsEnum.seekExact(featureName) == false) {
      return DoubleValues.EMPTY;
    } else {
      PostingsEnum currentReaderPostingsValues = termsEnum.postings(null, PostingsEnum.FREQS);
      return new FeatureDoubleValues(currentReaderPostingsValues);
    }
  }
}
 

public void testFeature() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  doc.add(new FeatureField("field", "name", 30F));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 1F));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 4F));
  writer.addDocument(doc);
  writer.forceMerge(1);
  IndexReader ir = writer.getReader();
  writer.close();

  assertEquals(1, ir.leaves().size());
  LeafReaderContext context = ir.leaves().get(0);
  DoubleValuesSource valuesSource = FeatureField.newDoubleValues("field", "name");
  DoubleValues values = valuesSource.getValues(context, null);

  assertTrue(values.advanceExact(0));
  assertEquals(30, values.doubleValue(), 0f);
  assertTrue(values.advanceExact(1));
  assertEquals(1, values.doubleValue(), 0f);
  assertTrue(values.advanceExact(2));
  assertEquals(4, values.doubleValue(), 0f);
  assertFalse(values.advanceExact(3));

  ir.close();
  dir.close();
}
 

public void testFeatureMissing() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 1F));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 4F));
  writer.addDocument(doc);
  writer.forceMerge(1);
  IndexReader ir = writer.getReader();
  writer.close();

  assertEquals(1, ir.leaves().size());
  LeafReaderContext context = ir.leaves().get(0);
  DoubleValuesSource valuesSource = FeatureField.newDoubleValues("field", "name");
  DoubleValues values = valuesSource.getValues(context, null);

  assertFalse(values.advanceExact(0));
  assertTrue(values.advanceExact(1));
  assertEquals(1, values.doubleValue(), 0f);
  assertTrue(values.advanceExact(2));
  assertEquals(4, values.doubleValue(), 0f);
  assertFalse(values.advanceExact(3));

  ir.close();
  dir.close();
}
 

@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
    Bindings bindings = new Bindings(){
        @Override
        public DoubleValuesSource getDoubleValuesSource(String name) {
            return new FVDoubleValuesSource(vectorSupplier, features.featureOrdinal(name));
        }
    };

    DoubleValuesSource src = expression.getDoubleValuesSource(bindings);
    DoubleValues values = src.getValues(context, null);
    values.advanceExact(doc);
    return Explanation.match((float) values.doubleValue(), "Evaluation of derived expression: " + expression.sourceText);
}
 

@Override
public void indexingDone() {
	try {
		spellChecker = new DirectSpellChecker();
		spellChecker.setMaxEdits(2);
		spellChecker.setAccuracy(0.1f);
		spellChecker.setMinPrefix(0);
		reader = DirectoryReader.open(writer);

		fuzzySuggester = new FuzzySuggester(directory, "", writer.getAnalyzer());
		Dictionary dict = new DocumentValueSourceDictionary(reader, WORD_FIELD, new LongValuesSource() {
			
			@Override
			public boolean needsScores() {
				return false;
			}
			
			@Override
			public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
				return null;
			}
		});
		fuzzySuggester.build(dict);
		
		writer.close();
		searcher = new IndexSearcher(DirectoryReader.open(directory));
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

@Override
public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
  return new LongValues() {
    @Override
    public long longValue() throws IOException {
      return value;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return true;
    }
  };
}
 

@Override
public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
  DoubleValues dv = getValues(ctx, DoubleValuesSource.constant(scoreExplanation.getValue().doubleValue()).getValues(ctx, null));
  if (dv.advanceExact(docId)) {
    AtomicReference<Explanation> explanation = new AtomicReference<>();
    final ShapeValues shapeValues = bboxValueSource.getValues(ctx);
    if (shapeValues.advanceExact(docId)) {
      score((Rectangle) shapeValues.value(), explanation);
      return explanation.get();
    }
  }
  return Explanation.noMatch(this.toString());
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  Weight w = inner.createWeight(searcher, scoreMode, 1f);
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      Scorer in = w.scorer(context);
      if (in == null)
        return null;
      DoubleValues v = distanceSource.getValues(context, DoubleValuesSource.fromScorer(in));
      DocIdSetIterator approximation = in.iterator();
      TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
        @Override
        public boolean matches() throws IOException {
          return v.advanceExact(approximation.docID()) && v.doubleValue() <= limit;
        }

        @Override
        public float matchCost() {
          return 100;   // distance calculation can be heavy!
        }
      };
      return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return distanceSource.isCacheable(ctx);
    }

  };
}
 

/**
 * Returns the FunctionValues used by the function query.
 */
@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
  LeafReader reader = readerContext.reader();

  final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
  final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());

  return DoubleValues.withDefault(new DoubleValues() {

    private final Point from = DistanceValueSource.this.from;
    private final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();

    @Override
    public double doubleValue() throws IOException {
      double x = Double.longBitsToDouble(ptX.longValue());
      double y = Double.longBitsToDouble(ptY.longValue());
      return calculator.distance(from, x, y) * multiplier;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return ptX.advanceExact(doc) && ptY.advanceExact(doc);
    }

  }, nullValue);
}
 

@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
  DoubleValues in = input.getValues(ctx, scores);
  return new DoubleValues() {
    @Override
    public double doubleValue() throws IOException {
      return recip(in.doubleValue());
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return in.advanceExact(doc);
    }
  };
}
 

@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {

  final double nullValue = (ctx.isGeo() ? 180 * multiplier : Double.MAX_VALUE);

  return DoubleValues.withDefault(new DoubleValues() {
    private final ShapeFieldCache<Point> cache =
        provider.getCache(readerContext.reader());
    private final Point from = ShapeFieldCacheDistanceValueSource.this.from;
    private final DistanceCalculator calculator = ctx.getDistCalc();

    private List<Point> currentVals;

    @Override
    public double doubleValue() throws IOException {
      double v = calculator.distance(from, currentVals.get(0));
      for (int i = 1; i < currentVals.size(); i++) {
        v = Math.min(v, calculator.distance(from, currentVals.get(i)));
      }
      return v * multiplier;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      currentVals = cache.getShapes(doc);
      return currentVals != null;
    }
  }, nullValue);
}