org.apache.lucene.util.TestUtil Java Examples

public void testNoParent() throws IOException {
  final Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  final int numDocs = TestUtil.nextInt(random(), 1, 3);
  for (int i = 0; i < numDocs; ++i) {
    w.addDocument(new Document());
  }
  final IndexReader reader = w.getReader();
  w.close();
  BitSetProducer parentsFilter = new QueryBitSetProducer(new MatchNoDocsQuery());
  try {
    expectThrows(IllegalStateException.class, () -> CheckJoinIndex.check(reader, parentsFilter));
  } finally {
    reader.close();
    dir.close();
  }
}
 

public void testTermDocs() throws IOException {
  //After adding the document, we should be able to read it back in
  SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
  assertTrue(reader != null);

  TermsEnum terms = reader.terms(DocHelper.TEXT_FIELD_2_KEY).iterator();
  terms.seekCeil(new BytesRef("field"));
  PostingsEnum termDocs = TestUtil.docs(random(), terms, null, PostingsEnum.FREQS);
  if (termDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)    {
    int docId = termDocs.docID();
    assertTrue(docId == 0);
    int freq = termDocs.freq();
    assertTrue(freq == 3);  
  }
  reader.close();
}
 

public void testTermInfoComparisonConsistency() {
  TermInfo a = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 0, 1 );
  TermInfo b = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 1, 1 );
  TermInfo c = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 2, 1 );
  TermInfo d = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 0, 1 );

  assertConsistentEquals( a, a );
  assertConsistentEquals( b, b );
  assertConsistentEquals( c, c );
  assertConsistentEquals( d, d );
  assertConsistentEquals( a, d );
  assertConsistentLessThan( a, b );
  assertConsistentLessThan( b, c );
  assertConsistentLessThan( a, c );
  assertConsistentLessThan( d, b );
  assertConsistentLessThan( d, c );
}
 

public void testNCommon() throws Exception {
  final Random r = random();
  final int N = TestUtil.nextInt(r, 2, 15);
  final long[] commonValues = new long[N];
  for (int j = 0; j < N; ++j) {
    commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
  }
  final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N);
  final long[] otherValues = new long[numOtherValues];
  for (int j = 0; j < numOtherValues; ++j) {
    otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
  }
  doTestNormsVersusDocValues(1, new LongSupplier() {
    @Override
    public long getAsLong() {
      return r.nextInt(100) == 0 ? otherValues[r.nextInt(numOtherValues - 1)] : commonValues[r.nextInt(N - 1)];
    }
  });
}
 

public void testConcurrentIndexing() throws Exception {
  final CloudSolrClient cloudClient = cluster.getSolrClient();
  final String collectionName = createAndSetNewDefaultCollection();

  final int numDocs = atLeast(50);
  final JettySolrRunner nodeToUpdate = cluster.getRandomJetty(random());
  try (ConcurrentUpdateSolrClient indexClient
       = getConcurrentUpdateSolrClient(nodeToUpdate.getProxyBaseUrl() + "/" + collectionName, 10, 2)) {
    
    for (int i = 0; i < numDocs; i++) {
      indexClient.add(sdoc("id", i, "text_t",
                           TestUtil.randomRealisticUnicodeString(random(), 200)));
    }
    indexClient.blockUntilFinished();
    
    assertEquals(0, indexClient.commit().getStatus());
    assertEquals(numDocs, cloudClient.query(params("q","*:*")).getResults().getNumFound());

    checkShardConsistency(params("q","*:*", "rows", ""+(1 + numDocs),"_trace","addAll"));
  }
}
 

public void testFullEviction() throws IOException {
    int pass = TestUtil.nextInt(random(), 10, 100);
    CachedFeatureStore cachedFeatureStore = new CachedFeatureStore(memStore, caches);
    while (pass-- > 0) {
        StoredFeature feat = LtrTestUtils.randomFeature();
        memStore.add(feat);
        cachedFeatureStore.load(feat.name());

        StoredFeatureSet set = LtrTestUtils.randomFeatureSet();
        memStore.add(set);
        cachedFeatureStore.loadSet(set.name());

        CompiledLtrModel model = LtrTestUtils.buildRandomModel();
        memStore.add(model);
        cachedFeatureStore.loadModel(model.name());
    }
    caches.evict(memStore.getStoreName());
    assertEquals(0, cachedFeatureStore.totalWeight());
    assertTrue(caches.getCachedStoreNames().isEmpty());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).modelRam());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).totalCount());
}
 

public void testLetterHtmlish() throws Exception {
  Random random = random();
  Analyzer left = new MockAnalyzer(random, jvmLetter, false);
  Analyzer right = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
      Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
      return new TokenStreamComponents(tokenizer, tokenizer);
    }
  };
  for (int i = 0; i < 200; i++) {
    String s = TestUtil.randomHtmlishString(random, 20);
    assertEquals(s, left.tokenStream("foo", newStringReader(s)), 
                 right.tokenStream("foo", newStringReader(s)));
  }
  IOUtils.close(left, right);
}
 

public void testRandomAdvanceNumeric() throws IOException {
  final long longRange;
  if (random().nextBoolean()) {
    longRange = TestUtil.nextInt(random(), 1, 1024);
  } else {
    longRange = TestUtil.nextLong(random(), 1, Long.MAX_VALUE);
  }
  doTestRandomAdvance(new FieldCreator() {
      @Override
      public Field next() {
        return new NumericDocValuesField("field", TestUtil.nextLong(random(), 0, longRange));
      }

      @Override
      public DocIdSetIterator iterator(IndexReader r) throws IOException {
        return MultiDocValues.getNumericValues(r, "field");
      }
    });
}
 

@Test
public void testRegisterMetricsWithReplacements() {
  Random random = random();

  Map<String, Counter> registered = new HashMap<>();
  String scope = SolrMetricTestUtils.getRandomScope(random, true);
  SolrInfoBean.Category category = SolrMetricTestUtils.getRandomCategory(random, true);

  int iterations = TestUtil.nextInt(random, 0, MAX_ITERATIONS);
  for (int i = 0; i < iterations; ++i) {
    Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetricsWithReplacements(random, registered);
    if (metrics.isEmpty()) {
      continue;
    }
    SolrMetricProducer producer = SolrMetricTestUtils.getProducerOf(metricManager, category, scope, metrics);
    coreMetricManager.registerMetricProducer(scope, producer);
    registered.putAll(metrics);
    assertRegistered(scope, registered, coreMetricManager);
  }
}
 

public void testMoreThanOneDocPerIDOneSegment() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
  Document doc = new Document();
  doc.add(makeIDField("id", 17));
  w.addDocument(doc);

  Document duplicate = new Document();
  duplicate.add(makeIDField("id", 17));
  expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(duplicate);
    w.commit(false);
  });

  w.close();
  dir.close();
}
 

public void testPackedLongValuesOnZeros() {
  // Make sure that when all values are the same, they use 0 bits per value
  final int pageSize = 1 << TestUtil.nextInt(random(), 6, 20);
  final float acceptableOverheadRatio = random().nextFloat();

  assertEquals(
      PackedLongValues.packedBuilder(pageSize, acceptableOverheadRatio).add(0).build().ramBytesUsed(),
      PackedLongValues.packedBuilder(pageSize, acceptableOverheadRatio).add(0).add(0).build().ramBytesUsed());

  final long l = random().nextLong();
  assertEquals(
    PackedLongValues.deltaPackedBuilder(pageSize, acceptableOverheadRatio).add(l).build().ramBytesUsed(),
    PackedLongValues.deltaPackedBuilder(pageSize, acceptableOverheadRatio).add(l).add(l).build().ramBytesUsed());

  final long avg = random().nextInt(100);
  assertEquals(
    PackedLongValues.monotonicBuilder(pageSize, acceptableOverheadRatio).add(l).add(l + avg).build().ramBytesUsed(),
    PackedLongValues.monotonicBuilder(pageSize, acceptableOverheadRatio).add(l).add(l + avg).add(l + 2 * avg).build().ramBytesUsed());
}
 

public void testInvalidVersions2() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
  Document doc = new Document();
  // Long.MAX_VALUE:
  doc.add(new StringAndPayloadField("id", "id", new BytesRef(new byte[] {(byte)0x7f, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff})));
  expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
    w.commit(false);
  });
  expectThrows(AlreadyClosedException.class, () -> {
    w.addDocument(doc);
  });

  dir.close();
}
 

public void testLetterUnicode() throws Exception {
  Random random = random();
  Analyzer left = new MockAnalyzer(random(), jvmLetter, false);
  Analyzer right = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
      Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
      return new TokenStreamComponents(tokenizer, tokenizer);
    }
  };
  for (int i = 0; i < 200; i++) {
    String s = TestUtil.randomUnicodeString(random);
    assertEquals(s, left.tokenStream("foo", newStringReader(s)), 
                 right.tokenStream("foo", newStringReader(s)));
  }
  IOUtils.close(left, right);
}
 

public void testAddIndexesWithCloseNoWait() throws Throwable {

    final int NUM_COPY = 50;
    CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
    c.launchThreads(-1);

    Thread.sleep(TestUtil.nextInt(random(), 10, 500));

    // Close w/o first stopping/joining the threads
    if (VERBOSE) {
      System.out.println("TEST: now close(false)");
    }
    c.close(false);

    c.joinThreads();

    if (VERBOSE) {
      System.out.println("TEST: done join threads");
    }
    c.closeDir();

    assertTrue(c.failures.size() == 0);
  }
 

public void testBulkMergeWithDeletes() throws IOException {
  final int numDocs = atLeast(200);
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE));
  for (int i = 0; i < numDocs; ++i) {
    Document doc = new Document();
    doc.add(new StringField("id", Integer.toString(i), Store.YES));
    doc.add(new StoredField("f", TestUtil.randomSimpleString(random())));
    w.addDocument(doc);
  }
  final int deleteCount = TestUtil.nextInt(random(), 5, numDocs);
  for (int i = 0; i < deleteCount; ++i) {
    final int id = random().nextInt(numDocs);
    w.deleteDocuments(new Term("id", Integer.toString(id)));
  }
  w.commit();
  w.close();
  w = new RandomIndexWriter(random(), dir);
  w.forceMerge(TestUtil.nextInt(random(), 1, 3));
  w.commit();
  w.close();
  TestUtil.checkIndex(dir);
  dir.close();
}
 

private void createLargeIndex() throws Exception {
  int BATCH_SIZE = 1000;
  int NUM_BATCHES = 100;
  SolrInputDocument[] docs = new SolrInputDocument[BATCH_SIZE];
  for (int i = 0; i < NUM_BATCHES; i++) {
    for (int j = 0; j < BATCH_SIZE; j++) {
      docs[j] = new SolrInputDocument(
          "id", String.valueOf(i * BATCH_SIZE + j),
          "batch_i_p", String.valueOf(i),
          "random_i_p", String.valueOf(random().nextInt(BATCH_SIZE)),
          "sortabledv", TestUtil.randomSimpleString(random(), 2, 3),
          "sortabledv_udvas", String.valueOf(random().nextInt(100)),
          "small_i_p", String.valueOf((i + j) % 7)
          );
    }
    updateJ(jsonAdd(docs), null);
  }
  assertU(commit());
}
 

public void testInvalidVersions() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
  RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc, false);
  Document doc = new Document();
  // -1
  doc.add(new StringAndPayloadField("id", "id", new BytesRef(new byte[] {(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff})));
  expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
    w.commit(false);
  });
  expectThrows(AlreadyClosedException.class, () -> {
    w.addDocument(doc);
  });
  dir.close();
}
 

@Test
public void testRegisterAll() throws Exception {
  Random r = random();

  SolrMetricManager metricManager = new SolrMetricManager();

  Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetrics(r, true);
  MetricRegistry mr = new MetricRegistry();
  for (Map.Entry<String, Counter> entry : metrics.entrySet()) {
    mr.register(entry.getKey(), entry.getValue());
  }

  String registryName = TestUtil.randomSimpleString(r, 1, 10);
  assertEquals(0, metricManager.registry(registryName).getMetrics().size());
  // There is nothing registered so we should be error-free on the first pass
  metricManager.registerAll(registryName, mr, SolrMetricManager.ResolutionStrategy.ERROR);
  // this should simply skip existing names
  metricManager.registerAll(registryName, mr, SolrMetricManager.ResolutionStrategy.IGNORE);
  // this should re-register everything, and no errors
  metricManager.registerAll(registryName, mr, SolrMetricManager.ResolutionStrategy.REPLACE);
  // this should produce error
  expectThrows(IllegalArgumentException.class, () -> metricManager.registerAll(registryName, mr, SolrMetricManager.ResolutionStrategy.ERROR));
}
 

public void testRandomLastByteTwoValues() throws IOException {
  int values = random().nextInt(15000) + 1;
  Directory dir = getDirectory(values);
  int partitionPoint = random().nextInt(values);
  int sortedOnHeap = random().nextInt(5000);
  int indexDimensions =  TestUtil.nextInt(random(), 1, 8);
  int dataDimensions =  TestUtil.nextInt(random(), indexDimensions, 8);
  int bytesPerDimensions = TestUtil.nextInt(random(), 2, 30);
  int packedLength = dataDimensions * bytesPerDimensions;
  PointWriter points = getRandomPointWriter(dir, values, packedLength);
  byte[] value = new byte[packedLength];
  random().nextBytes(value);
  for (int i =0; i < values; i++) {
    if (random().nextBoolean()) {
      points.append(value, 1);
    } else {
      points.append(value, 2);
    }
  }
  points.close();
  verify(dir, points, dataDimensions, indexDimensions, 0, values, partitionPoint, packedLength, bytesPerDimensions, sortedOnHeap);
  dir.close();
}
 

private void indexOneDoc(long seed, Directory dir, Document doc, boolean useCFS) throws IOException {
  Random random = new Random(seed);
  IndexWriterConfig conf = newIndexWriterConfig(random, new MockAnalyzer(random));
  conf.setCodec(TestUtil.getDefaultCodec());

  if (useCFS == false) {
    conf.setUseCompoundFile(false);
    conf.getMergePolicy().setNoCFSRatio(0.0);
  } else {
    conf.setUseCompoundFile(true);
    conf.getMergePolicy().setNoCFSRatio(1.0);
  }

  RandomIndexWriter w = new RandomIndexWriter(random, dir, conf);
  w.addDocument(doc);
  w.close();
}
 

/**
 * test faceting with deep paging
 */
public void testFacetingWithRandomSorts() throws Exception {
  final int numDocs = TestUtil.nextInt(random(), 1000, 3000);
  String[] fieldsToFacetOn = { "int", "long", "str" };
  String[] facetMethods = { "enum", "fc", "fcs" };

  for (int i = 1; i <= numDocs; i++) {
    SolrInputDocument doc = buildRandomDocument(i);
    assertU(adoc(doc));
  }
  assertU(commit());

  Collection<String> allFieldNames = getAllSortFieldNames();
  String[] fieldNames = new String[allFieldNames.size()];
  allFieldNames.toArray(fieldNames);
  String f = fieldNames[TestUtil.nextInt(random(), 0, fieldNames.length - 1)];
  String order = 0 == TestUtil.nextInt(random(), 0, 1) ? " asc" : " desc";
  String sort = f + order + (f.equals("id") ? "" : ", id" + order);
  String rows = "" + TestUtil.nextInt(random(), 13, 50);
  String facetField = fieldsToFacetOn
      [TestUtil.nextInt(random(), 0, fieldsToFacetOn.length - 1)];
  String facetMethod = facetMethods
      [TestUtil.nextInt(random(), 0, facetMethods.length - 1)];
  SentinelIntSet ids = assertFullWalkNoDupsWithFacets
      (numDocs, params("q", "*:*",
          "fl", "id," + facetField,
          "facet", "true",
          "facet.field", facetField,
          "facet.method", facetMethod,
          "facet.missing", "true",
          "facet.limit", "-1", // unlimited
          "rows", rows,
          "sort", sort));
  assertEquals(numDocs, ids.size());
}
 

private IndexSearcher parallel(Random random) throws IOException {
  dir1 = getDir1(random);
  dir2 = getDir2(random);
  ParallelLeafReader pr = new ParallelLeafReader(
      getOnlyLeafReader(DirectoryReader.open(dir1)),
      getOnlyLeafReader(DirectoryReader.open(dir2)));
  TestUtil.checkReader(pr);
  return newSearcher(pr);
}
 

/**
 * half of the time, returns a random term from TEST_TERMS.
 * the other half of the time, returns a random unicode string.
 */
private String randomTerm() {
  if (random().nextBoolean()) {
    // return a random TEST_TERM
    return TEST_TERMS[random().nextInt(TEST_TERMS.length)];
  } else {
    // return a random unicode term
    return TestUtil.randomUnicodeString(random());
  }
}
 

public void testDocValuesUpdates() throws Exception {
  Path oldIndexDir = createTempDir("dvupdates");
  TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
  Directory dir = newFSDirectory(oldIndexDir);
  verifyUsesDefaultCodec(dir, dvUpdatesIndex);
  
  verifyDocValues(dir);
  
  // update fields and verify index
  IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  updateNumeric(writer, "1", "ndv1", "ndv1_c", 300L);
  updateNumeric(writer, "1", "ndv2", "ndv2_c", 300L);
  updateBinary(writer, "1", "bdv1", "bdv1_c", 300L);
  updateBinary(writer, "1", "bdv2", "bdv2_c", 300L);

  writer.commit();
  verifyDocValues(dir);
  
  // merge all segments
  writer.forceMerge(1);
  writer.commit();
  verifyDocValues(dir);
  
  writer.close();
  dir.close();
}
 

public void testTotalBytesSize() throws Exception {
  Directory d = newDirectory();
  if (d instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)d).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
  }
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  iwc.setMaxBufferedDocs(5);
  CountDownLatch atLeastOneMerge = new CountDownLatch(1);
  iwc.setMergeScheduler(new TrackingCMS(atLeastOneMerge));
  if (TestUtil.getPostingsFormat("id").equals("SimpleText")) {
    // no
    iwc.setCodec(TestUtil.alwaysPostingsFormat(TestUtil.getDefaultPostingsFormat()));
  }
  IndexWriter w = new IndexWriter(d, iwc);
  for(int i=0;i<1000;i++) {
    Document doc = new Document();
    doc.add(new StringField("id", ""+i, Field.Store.NO));
    w.addDocument(doc);

    if (random().nextBoolean()) {
      w.deleteDocuments(new Term("id", ""+random().nextInt(i+1)));
    }
  }
  atLeastOneMerge.await();
  assertTrue(((TrackingCMS) w.getConfig().getMergeScheduler()).totMergedBytes != 0);
  w.close();
  d.close();
}
 

public void testSparseLongRange() throws Exception {
  assumeTrue("Requires sparse norms support", codecSupportsSparsity());
  int iterations = atLeast(1);
  final Random r = random();
  for (int i = 0; i < iterations; i++) {
    doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() {
      @Override
      public long getAsLong() {
        return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE);
      }
    });
  }
}
 

private static String randomText() {
  int numWords = TestUtil.nextInt(random(), 1, 4);
    
  StringBuilder b = new StringBuilder();
  for(int i=0;i<numWords;i++) {
    if (i > 0) {
      b.append(' ');
    }
    b.append(TestUtil.randomSimpleString(random(), 1, 10));
  }

  return b.toString();
}
 

@Nightly
public void testTermsEnumRandomMany() throws Exception {
  int numIterations = atLeast(1);
  for (int i = 0; i < numIterations; i++) {
    doTestTermsEnumRandom(TestUtil.nextInt(random(), 1025, 8121), () -> TestUtil.randomSimpleString(random(), 1, 500));
  }
}
 

public void testSortedNumericsMultipleValuesVsStoredFields() throws Exception {
  int numIterations = atLeast(1);
  for (int i = 0; i < numIterations; i++) {
    doTestSortedNumericsVsStoredFields(
        () -> TestUtil.nextLong(random(), 0, 50),
        random()::nextLong
    );
  }
}
 

public void testFewTokenChars() throws IOException {
  final char[] chrs = new char[TestUtil.nextInt(random(), 4000, 5000)];
  Arrays.fill(chrs, ' ');
  for (int i = 0; i < chrs.length; ++i) {
    if (random().nextFloat() < 0.1) {
      chrs[i] = 'a';
    }
  }
  final int minGram = TestUtil.nextInt(random(), 1, 2);
  final int maxGram = TestUtil.nextInt(random(), minGram, 2);
  testNGrams(minGram, maxGram, new String(chrs), " ");
}