Java Code Examples for org.apache.lucene.index.LeafReader#close()

The following examples show how to use org.apache.lucene.index.LeafReader#close() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KNearestNeighborClassifierTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBasicUsage() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    checkCorrectClassification(new KNearestNeighborClassifier(leafReader, null, analyzer, null, 1, 0, 0, categoryFieldName, textFieldName), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
    checkCorrectClassification(new KNearestNeighborClassifier(leafReader, new LMDirichletSimilarity(), analyzer, null, 1, 0, 0, categoryFieldName, textFieldName), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
    ClassificationResult<BytesRef> resultDS =  checkCorrectClassification(new KNearestNeighborClassifier(leafReader, new BM25Similarity(), analyzer, null, 3, 2, 1, categoryFieldName, textFieldName), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
    ClassificationResult<BytesRef> resultLMS =  checkCorrectClassification(new KNearestNeighborClassifier(leafReader, new LMDirichletSimilarity(), analyzer, null, 3, 2, 1, categoryFieldName, textFieldName), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
    assertTrue(resultDS.getScore() != resultLMS.getScore());
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 2
Source File: TestCompressingTermVectorsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
public void testNoOrds() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setStoreTermVectors(true);
  doc.add(new Field("foo", "this is a test", ft));
  iw.addDocument(doc);
  LeafReader ir = getOnlyLeafReader(iw.getReader());
  Terms terms = ir.getTermVector(0, "foo");
  assertNotNull(terms);
  TermsEnum termsEnum = terms.iterator();
  assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef("this")));

  expectThrows(UnsupportedOperationException.class, termsEnum::ord);
  expectThrows(UnsupportedOperationException.class, () -> termsEnum.seekExact(0));

  ir.close();
  iw.close();
  dir.close();
}
Example 3
Source File: ConfusionMatrixGeneratorTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetConfusionMatrixWithBM25NB() throws Exception {
  LeafReader reader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    reader = getSampleIndex(analyzer);
    Classifier<BytesRef> classifier = new BM25NBClassifier(reader, analyzer, null, categoryFieldName, textFieldName);
    ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader,
        classifier, categoryFieldName, textFieldName, -1);
    checkCM(confusionMatrix);
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
Example 4
Source File: BooleanPerceptronClassifierTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBasicUsageWithQuery() throws Exception {
  TermQuery query = new TermQuery(new Term(textFieldName, "of"));
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    BooleanPerceptronClassifier classifier = new BooleanPerceptronClassifier(leafReader, analyzer, query, 1, null, booleanFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, false);
    checkCorrectClassification(classifier, POLITICS_INPUT, true);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 5
Source File: SimpleNaiveBayesClassifierTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBasicUsageWithQuery() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    TermQuery query = new TermQuery(new Term(textFieldName, "a"));
    SimpleNaiveBayesClassifier classifier = new SimpleNaiveBayesClassifier(leafReader, analyzer, query, categoryFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
    checkCorrectClassification(classifier, POLITICS_INPUT, POLITICS_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 6
Source File: KNearestNeighborClassifierTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
/**
 * This test is for the scenario where in the first topK results from the MLT query, we have less results
 * for the expected class than the results for the bad class.
 * But the results for the expected class have a better score in comparison with the results of the second class.
 * So we would expect a greater score for the best ranked class.
 *
 * @throws Exception if any error happens
 */
@Test
public void testUnbalancedClasses() throws Exception {
  LeafReader leafReader = null;
  try {
    Analyzer analyzer = new EnglishAnalyzer();
    leafReader = getSampleIndex(analyzer);
    KNearestNeighborClassifier knnClassifier = new KNearestNeighborClassifier(leafReader, null,analyzer, null, 3, 1, 1, categoryFieldName, textFieldName);
    List<ClassificationResult<BytesRef>> classes = knnClassifier.getClasses(SUPER_STRONG_TECHNOLOGY_INPUT);
    assertTrue(classes.get(0).getScore() > classes.get(1).getScore());
    checkCorrectClassification(knnClassifier, SUPER_STRONG_TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 7
Source File: KNearestNeighborClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBasicUsageWithQuery() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    TermQuery query = new TermQuery(new Term(textFieldName, "it"));
    checkCorrectClassification(new KNearestNeighborClassifier(leafReader, null, analyzer, query, 1, 0, 0, categoryFieldName, textFieldName), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 8
Source File: BM25NBClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNGramUsage() throws Exception {
  LeafReader leafReader = null;
  try {
    Analyzer analyzer = new NGramAnalyzer();
    leafReader = getSampleIndex(analyzer);
    BM25NBClassifier classifier = new BM25NBClassifier(leafReader, analyzer, null, categoryFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 9
Source File: BM25NBClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBasicUsageWithQuery() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    TermQuery query = new TermQuery(new Term(textFieldName, "not"));
    BM25NBClassifier classifier = new BM25NBClassifier(leafReader, analyzer, query, categoryFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 10
Source File: SimpleNaiveBayesClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNGramUsage() throws Exception {
  LeafReader leafReader = null;
  try {
    Analyzer analyzer = new NGramAnalyzer();
    leafReader = getSampleIndex(analyzer);
    SimpleNaiveBayesClassifier classifier = new SimpleNaiveBayesClassifier(leafReader, analyzer, null, categoryFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 11
Source File: ConfusionMatrixGeneratorTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetConfusionMatrixWithBP() throws Exception {
  LeafReader reader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    reader = getSampleIndex(analyzer);
    Classifier<Boolean> classifier = new BooleanPerceptronClassifier(reader, analyzer, null, 1, null, booleanFieldName, textFieldName);
    ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(reader,
        classifier, booleanFieldName, textFieldName, -1);
    checkCM(confusionMatrix);
    assertTrue(confusionMatrix.getPrecision("true") >= 0d);
    assertTrue(confusionMatrix.getPrecision("true") <= 1d);
    assertTrue(confusionMatrix.getPrecision("false") >= 0d);
    assertTrue(confusionMatrix.getPrecision("false") <= 1d);
    assertTrue(confusionMatrix.getRecall("true") >= 0d);
    assertTrue(confusionMatrix.getRecall("true") <= 1d);
    assertTrue(confusionMatrix.getRecall("false") >= 0d);
    assertTrue(confusionMatrix.getRecall("false") <= 1d);
    assertTrue(confusionMatrix.getF1Measure("true") >= 0d);
    assertTrue(confusionMatrix.getF1Measure("true") <= 1d);
    assertTrue(confusionMatrix.getF1Measure("false") >= 0d);
    assertTrue(confusionMatrix.getF1Measure("false") <= 1d);
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
Example 12
Source File: KNearestFuzzyClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBasicUsage() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    Classifier<BytesRef> classifier = new KNearestFuzzyClassifier(leafReader, null, analyzer, null, 3, categoryFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
    checkCorrectClassification(classifier, POLITICS_INPUT, POLITICS_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 13
Source File: BooleanPerceptronClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBasicUsage() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    BooleanPerceptronClassifier classifier = new BooleanPerceptronClassifier(leafReader, analyzer, null, 1, null, booleanFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, false);
    checkCorrectClassification(classifier, POLITICS_INPUT, true);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 14
Source File: BooleanPerceptronClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testExplicitThreshold() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    BooleanPerceptronClassifier classifier = new BooleanPerceptronClassifier(leafReader, analyzer, null, 1, 50d, booleanFieldName, textFieldName);
    checkCorrectClassification(classifier, TECHNOLOGY_INPUT, false);
    checkCorrectClassification(classifier, POLITICS_INPUT, true);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 15
Source File: CachingNaiveBayesClassifierTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBasicUsageWithQuery() throws Exception {
  LeafReader leafReader = null;
  try {
    MockAnalyzer analyzer = new MockAnalyzer(random());
    leafReader = getSampleIndex(analyzer);
    TermQuery query = new TermQuery(new Term(textFieldName, "it"));
    checkCorrectClassification(new CachingNaiveBayesClassifier(leafReader, analyzer, query, categoryFieldName, textFieldName), TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
  } finally {
    if (leafReader != null) {
      leafReader.close();
    }
  }
}
Example 16
Source File: SimpleNaiveBayesClassifierTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPerformance() throws Exception {
  MockAnalyzer analyzer = new MockAnalyzer(random());
  int numDocs = atLeast(10);
  LeafReader leafReader = getRandomIndex(analyzer, numDocs);
  try {
    SimpleNaiveBayesClassifier simpleNaiveBayesClassifier = new SimpleNaiveBayesClassifier(leafReader,
        analyzer, null, categoryFieldName, textFieldName);

    ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader,
        simpleNaiveBayesClassifier, categoryFieldName, textFieldName, -1);
    assertNotNull(confusionMatrix);

    double avgClassificationTime = confusionMatrix.getAvgClassificationTime();
    assertTrue(avgClassificationTime >= 0);

    double f1 = confusionMatrix.getF1Measure();
    assertTrue(f1 >= 0d);
    assertTrue(f1 <= 1d);

    double accuracy = confusionMatrix.getAccuracy();
    assertTrue(accuracy >= 0d);
    assertTrue(accuracy <= 1d);

    double recall = confusionMatrix.getRecall();
    assertTrue(recall >= 0d);
    assertTrue(recall <= 1d);

    double precision = confusionMatrix.getPrecision();
    assertTrue(precision >= 0d);
    assertTrue(precision <= 1d);

    Terms terms = MultiTerms.getTerms(leafReader, categoryFieldName);
    TermsEnum iterator = terms.iterator();
    BytesRef term;
    while ((term = iterator.next()) != null) {
      String s = term.utf8ToString();
      recall = confusionMatrix.getRecall(s);
      assertTrue(recall >= 0d);
      assertTrue(recall <= 1d);
      precision = confusionMatrix.getPrecision(s);
      assertTrue(precision >= 0d);
      assertTrue(precision <= 1d);
      double f1Measure = confusionMatrix.getF1Measure(s);
      assertTrue(f1Measure >= 0d);
      assertTrue(f1Measure <= 1d);
    }

  } finally {
    leafReader.close();
  }

}
Example 17
Source File: KNearestNeighborClassifierTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPerformance() throws Exception {
  MockAnalyzer analyzer = new MockAnalyzer(random());
  int numDocs = atLeast(10);
  LeafReader leafReader = getRandomIndex(analyzer,  numDocs);
  try {
    KNearestNeighborClassifier kNearestNeighborClassifier = new KNearestNeighborClassifier(leafReader, null,
        analyzer, null, 1, 1, 1, categoryFieldName, textFieldName);

    ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader,
        kNearestNeighborClassifier, categoryFieldName, textFieldName, -1);
    assertNotNull(confusionMatrix);

    double avgClassificationTime = confusionMatrix.getAvgClassificationTime();
    assertTrue(avgClassificationTime >= 0);

    double accuracy = confusionMatrix.getAccuracy();
    assertTrue(accuracy >= 0d);
    assertTrue(accuracy <= 1d);

    double recall = confusionMatrix.getRecall();
    assertTrue(recall >= 0d);
    assertTrue(recall <= 1d);

    double precision = confusionMatrix.getPrecision();
    assertTrue(precision >= 0d);
    assertTrue(precision <= 1d);

    Terms terms = MultiTerms.getTerms(leafReader, categoryFieldName);
    TermsEnum iterator = terms.iterator();
    BytesRef term;
    while ((term = iterator.next()) != null) {
      String s = term.utf8ToString();
      recall = confusionMatrix.getRecall(s);
      assertTrue(recall >= 0d);
      assertTrue(recall <= 1d);
      precision = confusionMatrix.getPrecision(s);
      assertTrue(precision >= 0d);
      assertTrue(precision <= 1d);
      double f1Measure = confusionMatrix.getF1Measure(s);
      assertTrue(f1Measure >= 0d);
      assertTrue(f1Measure <= 1d);
    }
  } finally {
    leafReader.close();
  }
}
Example 18
Source File: KNearestFuzzyClassifierTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPerformance() throws Exception {
  MockAnalyzer analyzer = new MockAnalyzer(random());
  int numDocs = atLeast(10);
  LeafReader leafReader = getRandomIndex(analyzer, numDocs);
  try {
    Classifier<BytesRef> classifier = new KNearestFuzzyClassifier(leafReader, null, analyzer, null, 3, categoryFieldName, textFieldName);

    ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader,
        classifier, categoryFieldName, textFieldName, -1);
    assertNotNull(confusionMatrix);

    double avgClassificationTime = confusionMatrix.getAvgClassificationTime();
    assertTrue(avgClassificationTime >= 0);

    double accuracy = confusionMatrix.getAccuracy();
    assertTrue(accuracy >= 0d);
    assertTrue(accuracy <= 1d);

    double recall = confusionMatrix.getRecall();
    assertTrue(recall >= 0d);
    assertTrue(recall <= 1d);

    double precision = confusionMatrix.getPrecision();
    assertTrue(precision >= 0d);
    assertTrue(precision <= 1d);

    Terms terms = MultiTerms.getTerms(leafReader, categoryFieldName);
    TermsEnum iterator = terms.iterator();
    BytesRef term;
    while ((term = iterator.next()) != null) {
      String s = term.utf8ToString();
      recall = confusionMatrix.getRecall(s);
      assertTrue(recall >= 0d);
      assertTrue(recall <= 1d);
      precision = confusionMatrix.getPrecision(s);
      assertTrue(precision >= 0d);
      assertTrue(precision <= 1d);
      double f1Measure = confusionMatrix.getF1Measure(s);
      assertTrue(f1Measure >= 0d);
      assertTrue(f1Measure <= 1d);
    }
  } finally {
    leafReader.close();
  }
}
Example 19
Source File: BM25NBClassifierTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up 
@Test @Slow
public void testPerformance() throws Exception {
  MockAnalyzer analyzer = new MockAnalyzer(random());
  int numDocs = atLeast(10);
  LeafReader leafReader = getRandomIndex(analyzer, numDocs);
  try {
    BM25NBClassifier classifier = new BM25NBClassifier(leafReader,
        analyzer, null, categoryFieldName, textFieldName);

    ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader,
        classifier, categoryFieldName, textFieldName, -1);
    assertNotNull(confusionMatrix);

    double avgClassificationTime = confusionMatrix.getAvgClassificationTime();
    assertTrue(avgClassificationTime >= 0);

    double f1 = confusionMatrix.getF1Measure();
    assertTrue(f1 >= 0d);
    assertTrue(f1 <= 1d);

    double accuracy = confusionMatrix.getAccuracy();
    assertTrue(accuracy >= 0d);
    assertTrue(accuracy <= 1d);

    double recall = confusionMatrix.getRecall();
    assertTrue(recall >= 0d);
    assertTrue(recall <= 1d);

    double precision = confusionMatrix.getPrecision();
    assertTrue(precision >= 0d);
    assertTrue(precision <= 1d);

    Terms terms = MultiTerms.getTerms(leafReader, categoryFieldName);
    TermsEnum iterator = terms.iterator();
    BytesRef term;
    while ((term = iterator.next()) != null) {
      String s = term.utf8ToString();
      recall = confusionMatrix.getRecall(s);
      assertTrue(recall >= 0d);
      assertTrue(recall <= 1d);
      precision = confusionMatrix.getPrecision(s);
      assertTrue(precision >= 0d);
      assertTrue(precision <= 1d);
      double f1Measure = confusionMatrix.getF1Measure(s);
      assertTrue(f1Measure >= 0d);
      assertTrue(f1Measure <= 1d);
    }

  } finally {
    leafReader.close();
  }

}
Example 20
Source File: BooleanPerceptronClassifierTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPerformance() throws Exception {
  MockAnalyzer analyzer = new MockAnalyzer(random());
  int numDocs = atLeast(10);
  LeafReader leafReader = getRandomIndex(analyzer, numDocs);
  try {
    BooleanPerceptronClassifier classifier = new BooleanPerceptronClassifier(leafReader, analyzer, null, 1, null, booleanFieldName, textFieldName);

    ConfusionMatrixGenerator.ConfusionMatrix confusionMatrix = ConfusionMatrixGenerator.getConfusionMatrix(leafReader,
        classifier, booleanFieldName, textFieldName, -1);
    assertNotNull(confusionMatrix);

    double avgClassificationTime = confusionMatrix.getAvgClassificationTime();
    assertTrue(avgClassificationTime >= 0);

    double f1 = confusionMatrix.getF1Measure();
    assertTrue(f1 >= 0d);
    assertTrue(f1 <= 1d);

    double accuracy = confusionMatrix.getAccuracy();
    assertTrue(accuracy >= 0d);
    assertTrue(accuracy <= 1d);

    double recall = confusionMatrix.getRecall();
    assertTrue(recall >= 0d);
    assertTrue(recall <= 1d);

    double precision = confusionMatrix.getPrecision();
    assertTrue(precision >= 0d);
    assertTrue(precision <= 1d);

    Terms terms = MultiTerms.getTerms(leafReader, booleanFieldName);
    TermsEnum iterator = terms.iterator();
    BytesRef term;
    while ((term = iterator.next()) != null) {
      String s = term.utf8ToString();
      recall = confusionMatrix.getRecall(s);
      assertTrue(recall >= 0d);
      assertTrue(recall <= 1d);
      precision = confusionMatrix.getPrecision(s);
      assertTrue(precision >= 0d);
      assertTrue(precision <= 1d);
      double f1Measure = confusionMatrix.getF1Measure(s);
      assertTrue(f1Measure >= 0d);
      assertTrue(f1Measure <= 1d);
    }
  } finally {
    leafReader.close();
  }
}