Java Code Examples for com.carrotsearch.randomizedtesting.generators.RandomPicks#randomFrom()

void checkPointEncoding(final double latitude, final double longitude) {
  PlanetModel planetModel = RandomPicks.randomFrom(random(), new PlanetModel[] {PlanetModel.WGS84, PlanetModel.CLARKE_1866});
  final GeoPoint point = new GeoPoint(planetModel, Geo3DUtil.fromDegrees(latitude), Geo3DUtil.fromDegrees(longitude));
  long pointValue = planetModel.getDocValueEncoder().encodePoint(point);
  final double x = planetModel.getDocValueEncoder().decodeXValue(pointValue);
  final double y = planetModel.getDocValueEncoder().decodeYValue(pointValue);
  final double z = planetModel.getDocValueEncoder().decodeZValue(pointValue);
  final GeoPoint pointR = new GeoPoint(x,y,z);
  // Check whether stable
  pointValue = planetModel.getDocValueEncoder().encodePoint(x, y, z);
  assertEquals(x, planetModel.getDocValueEncoder().decodeXValue(pointValue), 0.0);
  assertEquals(y, planetModel.getDocValueEncoder().decodeYValue(pointValue), 0.0);
  assertEquals(z, planetModel.getDocValueEncoder().decodeZValue(pointValue), 0.0);
  // Check whether has some relationship with original point
  assertEquals(0.0, point.arcDistance(pointR), 0.02);
}
 

static IndexWriterConfig iwcWithSuggestField(Analyzer analyzer, final Set<String> suggestFields) {
  IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  Codec filterCodec = new Lucene86Codec() {
    CompletionPostingsFormat.FSTLoadMode fstLoadMode =
        RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());
    PostingsFormat postingsFormat = new Completion84PostingsFormat(fstLoadMode);

    @Override
    public PostingsFormat getPostingsFormatForField(String field) {
      if (suggestFields.contains(field)) {
        return postingsFormat;
      }
      return super.getPostingsFormatForField(field);
    }
  };
  iwc.setCodec(filterCodec);
  return iwc;
}
 

protected RandomDocument(int fieldCount, int maxTermCount, Options options, String[] fieldNames, String[] sampleTerms, BytesRef[] sampleTermBytes) {
  if (fieldCount > fieldNames.length) {
    throw new IllegalArgumentException();
  }
  this.fieldNames = new String[fieldCount];
  fieldTypes = new FieldType[fieldCount];
  tokenStreams = new RandomTokenStream[fieldCount];
  Arrays.fill(fieldTypes, fieldType(options));
  final Set<String> usedFileNames = new HashSet<>();
  for (int i = 0; i < fieldCount; ++i) {
    do {
      this.fieldNames[i] = RandomPicks.randomFrom(random(), fieldNames);
    } while (usedFileNames.contains(this.fieldNames[i]));
    usedFileNames.add(this.fieldNames[i]);
    tokenStreams[i] = new RandomTokenStream(TestUtil.nextInt(random(), 1, maxTermCount), sampleTerms, sampleTermBytes);
  }
}
 

public void testDisjunctionCost() throws IOException {
  Map<Occur, Collection<ScorerSupplier>> subs = new EnumMap<>(Occur.class);
  for (Occur occur : Occur.values()) {
    subs.put(occur, new ArrayList<>());
  }

  subs.get(Occur.SHOULD).add(new FakeScorerSupplier(42));
  ScorerSupplier s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
  assertEquals(42, s.cost());
  assertEquals(42, s.get(random().nextInt(100)).iterator().cost());

  subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12));
  s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
  assertEquals(42 + 12, s.cost());
  assertEquals(42 + 12, s.get(random().nextInt(100)).iterator().cost());

  subs.get(Occur.SHOULD).add(new FakeScorerSupplier(20));
  s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
  assertEquals(42 + 12 + 20, s.cost());
  assertEquals(42 + 12 + 20, s.get(random().nextInt(100)).iterator().cost());
}
 

public void testBoxQueryEqualsAndHashcode() {
  Rectangle rectangle = GeoTestUtil.nextBox();
  QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
  String fieldName = "foo";
  Query q1 = newRectQuery(fieldName, queryRelation, rectangle.minLon, rectangle.maxLon, rectangle.minLat, rectangle.maxLat);
  Query q2 = newRectQuery(fieldName, queryRelation, rectangle.minLon, rectangle.maxLon, rectangle.minLat, rectangle.maxLat);
  QueryUtils.checkEqual(q1, q2);
  //different field name
  Query q3 = newRectQuery("bar", queryRelation, rectangle.minLon, rectangle.maxLon, rectangle.minLat, rectangle.maxLat);
  QueryUtils.checkUnequal(q1, q3);
  //different query relation
  QueryRelation newQueryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
  Query q4 = newRectQuery(fieldName, newQueryRelation, rectangle.minLon, rectangle.maxLon, rectangle.minLat, rectangle.maxLat);
  if (queryRelation == newQueryRelation) {
    QueryUtils.checkEqual(q1, q4);
  } else {
    QueryUtils.checkUnequal(q1, q4);
  }
  //different shape
  Rectangle newRectangle = GeoTestUtil.nextBox();
  Query q5 = newRectQuery(fieldName, queryRelation, newRectangle.minLon, newRectangle.maxLon, newRectangle.minLat, newRectangle.maxLat);
  if (rectangle.equals(newRectangle)) {
    QueryUtils.checkEqual(q1, q5);
  } else {
    QueryUtils.checkUnequal(q1, q5);
  }
}
 

DocValuesUpdate.BinaryDocValuesUpdate getRandomBinaryUpdate() {
  String termField = RandomPicks.randomFrom(random(), Arrays.asList("id", "_id", "some_other_field"));
  String docId = "" + random().nextInt(10);
  DocValuesUpdate.BinaryDocValuesUpdate value = new DocValuesUpdate.BinaryDocValuesUpdate(new Term(termField, docId), "binary",
      rarely() ? null : new BytesRef(TestUtil.randomRealisticUnicodeString(random())));
  return rarely() ? value.prepareForApply(randomDocUpTo()) : value;
}
 

/** test random generated polygons */
protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryPolygon = randomQueryPolygon();
    Component2D queryPoly2D = toPolygon2D(queryPolygon);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newPolygonQuery(FIELD_NAME, queryRelation, queryPolygon);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryPolygon);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

private static char randomSeparator() {
  return RandomPicks.randomFrom(random(), SEPARATORS);
}
 

public void testTonsOfUpdates() throws Exception {
  // LUCENE-5248: make sure that when there are many updates, we don't use too much RAM
  Directory dir = newDirectory();
  final Random random = random();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random));
  conf.setRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
  conf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); // don't flush by doc
  IndexWriter writer = new IndexWriter(dir, conf);
  
  // test data: lots of documents (few 10Ks) and lots of update terms (few hundreds)
  final int numDocs = TEST_NIGHTLY ? atLeast(20000) : atLeast(200);
  final int numNumericFields = atLeast(5);
  final int numTerms = TestUtil.nextInt(random, 10, 100); // terms should affect many docs
  Set<String> updateTerms = new HashSet<>();
  while (updateTerms.size() < numTerms) {
    updateTerms.add(TestUtil.randomSimpleString(random));
  }

  // build a large index with many NDV fields and update terms
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    int numUpdateTerms = TestUtil.nextInt(random, 1, numTerms / 10);
    for (int j = 0; j < numUpdateTerms; j++) {
      doc.add(new StringField("upd", RandomPicks.randomFrom(random, updateTerms), Store.NO));
    }
    for (int j = 0; j < numNumericFields; j++) {
      long val = random.nextInt();
      doc.add(new NumericDocValuesField("f" + j, val));
      doc.add(new NumericDocValuesField("cf" + j, val * 2));
    }
    writer.addDocument(doc);
  }
  
  writer.commit(); // commit so there's something to apply to
  
  // set to flush every 2048 bytes (approximately every 12 updates), so we get
  // many flushes during numeric updates
  writer.getConfig().setRAMBufferSizeMB(2048.0 / 1024 / 1024);
  final int numUpdates = atLeast(100);
  for (int i = 0; i < numUpdates; i++) {
    int field = random.nextInt(numNumericFields);
    Term updateTerm = new Term("upd", RandomPicks.randomFrom(random, updateTerms));
    long value = random.nextInt();
    writer.updateDocValues(updateTerm, new NumericDocValuesField("f"+field, value), new NumericDocValuesField("cf"+field, value*2));
  }

  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  for (LeafReaderContext context : reader.leaves()) {
    for (int i = 0; i < numNumericFields; i++) {
      LeafReader r = context.reader();
      NumericDocValues f = r.getNumericDocValues("f" + i);
      NumericDocValues cf = r.getNumericDocValues("cf" + i);
      for (int j = 0; j < r.maxDoc(); j++) {
        assertEquals(j, f.nextDoc());
        assertEquals(j, cf.nextDoc());
        assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.longValue(), f.longValue() * 2);
      }
    }
  }
  reader.close();
  
  dir.close();
}
 

/** Pick a random object from the given collection. */
public static <T> T randomFrom(Random random, Collection<T> collection) {
    return RandomPicks.randomFrom(random, collection);
}
 

@Nightly
  public void testTonsOfUpdates() throws Exception {
    // LUCENE-5248: make sure that when there are many updates, we don't use too much RAM
    Directory dir = newDirectory();
    final Random random = random();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random));
    conf.setRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
    conf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); // don't flush by doc
    IndexWriter writer = new IndexWriter(dir, conf);
    
    // test data: lots of documents (few 10Ks) and lots of update terms (few hundreds)
    final int numDocs = atLeast(20000);
    final int numBinaryFields = atLeast(5);
    final int numTerms = TestUtil.nextInt(random, 10, 100); // terms should affect many docs
    Set<String> updateTerms = new HashSet<>();
    while (updateTerms.size() < numTerms) {
      updateTerms.add(TestUtil.randomSimpleString(random));
    }

//    System.out.println("numDocs=" + numDocs + " numBinaryFields=" + numBinaryFields + " numTerms=" + numTerms);
    
    // build a large index with many BDV fields and update terms
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      int numUpdateTerms = TestUtil.nextInt(random, 1, numTerms / 10);
      for (int j = 0; j < numUpdateTerms; j++) {
        doc.add(new StringField("upd", RandomPicks.randomFrom(random, updateTerms), Store.NO));
      }
      for (int j = 0; j < numBinaryFields; j++) {
        long val = random.nextInt();
        doc.add(new BinaryDocValuesField("f" + j, toBytes(val)));
        doc.add(new BinaryDocValuesField("cf" + j, toBytes(val * 2)));
      }
      writer.addDocument(doc);
    }
    
    writer.commit(); // commit so there's something to apply to
    
    // set to flush every 2048 bytes (approximately every 12 updates), so we get
    // many flushes during binary updates
    writer.getConfig().setRAMBufferSizeMB(2048.0 / 1024 / 1024);
    final int numUpdates = atLeast(100);
//    System.out.println("numUpdates=" + numUpdates);
    for (int i = 0; i < numUpdates; i++) {
      int field = random.nextInt(numBinaryFields);
      Term updateTerm = new Term("upd", RandomPicks.randomFrom(random, updateTerms));
      long value = random.nextInt();
      writer.updateDocValues(updateTerm, new BinaryDocValuesField("f" + field, toBytes(value)), 
          new BinaryDocValuesField("cf" + field, toBytes(value * 2)));
    }

    writer.close();
    
    DirectoryReader reader = DirectoryReader.open(dir);
    for (LeafReaderContext context : reader.leaves()) {
      for (int i = 0; i < numBinaryFields; i++) {
        LeafReader r = context.reader();
        BinaryDocValues f = r.getBinaryDocValues("f" + i);
        BinaryDocValues cf = r.getBinaryDocValues("cf" + i);
        for (int j = 0; j < r.maxDoc(); j++) {
          assertEquals(j, f.nextDoc());
          assertEquals(j, cf.nextDoc());
          assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, getValue(cf), getValue(f) * 2);
        }
      }
    }
    reader.close();
    
    dir.close();
  }
 

@Nightly
  public void testTonsOfUpdates() throws Exception {
    // LUCENE-5248: make sure that when there are many updates, we don't use too much RAM
    Directory dir = newDirectory();
    final Random random = random();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random));
    conf.setRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
    conf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); // don't flush by doc
    IndexWriter writer = new IndexWriter(dir, conf);
    
    // test data: lots of documents (few 10Ks) and lots of update terms (few hundreds)
    final int numDocs = atLeast(20000);
    final int numBinaryFields = atLeast(5);
    final int numTerms = TestUtil.nextInt(random, 10, 100); // terms should affect many docs
    Set<String> updateTerms = new HashSet<>();
    while (updateTerms.size() < numTerms) {
      updateTerms.add(TestUtil.randomSimpleString(random));
    }

//    System.out.println("numDocs=" + numDocs + " numBinaryFields=" + numBinaryFields + " numTerms=" + numTerms);
    
    // build a large index with many BDV fields and update terms
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      int numUpdateTerms = TestUtil.nextInt(random, 1, numTerms / 10);
      for (int j = 0; j < numUpdateTerms; j++) {
        doc.add(new StringField("upd", RandomPicks.randomFrom(random, updateTerms), Store.NO));
      }
      for (int j = 0; j < numBinaryFields; j++) {
        long val = random.nextInt();
        doc.add(new BinaryDocValuesField("f" + j, TestBinaryDocValuesUpdates.toBytes(val)));
        doc.add(new NumericDocValuesField("cf" + j, val * 2));
      }
      writer.addDocument(doc);
    }
    
    writer.commit(); // commit so there's something to apply to
    
    // set to flush every 2048 bytes (approximately every 12 updates), so we get
    // many flushes during binary updates
    writer.getConfig().setRAMBufferSizeMB(2048.0 / 1024 / 1024);
    final int numUpdates = atLeast(100);
//    System.out.println("numUpdates=" + numUpdates);
    for (int i = 0; i < numUpdates; i++) {
      int field = random.nextInt(numBinaryFields);
      Term updateTerm = new Term("upd", RandomPicks.randomFrom(random, updateTerms));
      long value = random.nextInt();
      writer.updateDocValues(updateTerm, new BinaryDocValuesField("f"+field, TestBinaryDocValuesUpdates.toBytes(value)),
          new NumericDocValuesField("cf"+field, value*2));
    }

    writer.close();
    
    DirectoryReader reader = DirectoryReader.open(dir);
    for (LeafReaderContext context : reader.leaves()) {
      for (int i = 0; i < numBinaryFields; i++) {
        LeafReader r = context.reader();
        BinaryDocValues f = r.getBinaryDocValues("f" + i);
        NumericDocValues cf = r.getNumericDocValues("cf" + i);
        for (int j = 0; j < r.maxDoc(); j++) {
          assertEquals(j, cf.nextDoc());
          assertEquals(j, f.nextDoc());
          assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.longValue(), TestBinaryDocValuesUpdates.getValue(f) * 2);
        }
      }
    }
    reader.close();
    
    dir.close();
  }
 

/** test random generated lines */
protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // line
    Object queryLine = randomQueryLine(shapes);
    Component2D queryLine2D = toLine2D(queryLine);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), POINT_LINE_RELATIONS);
    Query query = newLineQuery(FIELD_NAME, queryRelation, queryLine);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryLine2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryLine);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/**
 * Corrupts a random file at a random position
 */
public static void corruptFile(Random random, Path... files) throws IOException {
    assertTrue("files must be non-empty", files.length > 0);
    final Path fileToCorrupt = RandomPicks.randomFrom(random, files);
    assertTrue(fileToCorrupt + " is not a file", Files.isRegularFile(fileToCorrupt));
    try (Directory dir = FSDirectory.open(fileToCorrupt.toAbsolutePath().getParent())) {
        long checksumBeforeCorruption;
        try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
        }
        try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
            long maxPosition = raf.size();

            if (fileToCorrupt.getFileName().toString().endsWith(".cfs") && maxPosition > 4) {
                // TODO: it is known that Lucene does not check the checksum of CFS file (CompoundFileS, like an archive)
                // see note at https://github.com/elastic/elasticsearch/pull/33911
                // so far, don't corrupt crc32 part of checksum (last 4 bytes) of cfs file
                // checksum is 8 bytes: first 4 bytes have to be zeros, while crc32 value is not verified
                maxPosition -= 4;
            }
            final int position = random.nextInt((int) Math.min(Integer.MAX_VALUE, maxPosition));
            corruptAt(fileToCorrupt, raf, position);
        }

        long checksumAfterCorruption;
        long actualChecksumAfterCorruption;
        try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            assertThat(input.getFilePointer(), is(0L));
            input.seek(input.length() - 8); // one long is the checksum... 8 bytes
            checksumAfterCorruption = input.getChecksum();
            actualChecksumAfterCorruption = input.readLong();
        }
        // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions
        // in the checksum which is ok though....
        StringBuilder msg = new StringBuilder();
        msg.append("before: [").append(checksumBeforeCorruption).append("] ");
        msg.append("after: [").append(checksumAfterCorruption).append("] ");
        msg.append("checksum value after corruption: ").append(actualChecksumAfterCorruption).append("] ");
        msg.append("file: ").append(fileToCorrupt.getFileName()).append(" length: ");
        msg.append(dir.fileLength(fileToCorrupt.getFileName().toString()));
        logger.info("Checksum {}", msg);
        assumeTrue("Checksum collision - " + msg.toString(),
                checksumAfterCorruption != checksumBeforeCorruption // collision
                        || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted
        assertThat("no file corrupted", fileToCorrupt, notNullValue());
    }
}
 

private void createRandomIndex(boolean singleSortedSegment) throws IOException {
  dir = newDirectory();
  numDocs = atLeast(150);
  final int numTerms = TestUtil.nextInt(random(), 1, numDocs / 5);
  Set<String> randomTerms = new HashSet<>();
  while (randomTerms.size() < numTerms) {
    randomTerms.add(TestUtil.randomSimpleString(random()));
  }
  terms = new ArrayList<>(randomTerms);
  final long seed = random().nextLong();
  final IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(new Random(seed)));
  if (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {
    // MockRandomMP randomly wraps the leaf readers which makes merging angry
    iwc.setMergePolicy(newTieredMergePolicy());
  }
  iwc.setMergeScheduler(new SerialMergeScheduler()); // for reproducible tests
  iwc.setIndexSort(sort);
  iw = new RandomIndexWriter(new Random(seed), dir, iwc);
  iw.setDoRandomForceMerge(false); // don't do this, it may happen anyway with MockRandomMP
  for (int i = 0; i < numDocs; ++i) {
    final Document doc = randomDocument();
    iw.addDocument(doc);
    if (i == numDocs / 2 || (i != numDocs - 1 && random().nextInt(8) == 0)) {
      iw.commit();
    }
    if (random().nextInt(15) == 0) {
      final String term = RandomPicks.randomFrom(random(), terms);
      iw.deleteDocuments(new Term("s", term));
    }
  }
  if (singleSortedSegment) {
    iw.forceMerge(1);
  }
  else if (random().nextBoolean()) {
    iw.forceMerge(FORCE_MERGE_MAX_SEGMENT_COUNT);
  }
  reader = iw.getReader();
  if (reader.numDocs() == 0) {
    iw.addDocument(new Document());
    reader.close();
    reader = iw.getReader();
  }
}
 

private static Query buildRandomQuery(int level) {
  if (level == 10) {
    // at most 10 levels
    return new MatchAllDocsQuery();
  }
  switch (random().nextInt(6)) {
    case 0:
      return new TermQuery(randomTerm());
    case 1:
      BooleanQuery.Builder bq = new BooleanQuery.Builder();
      final int numClauses = TestUtil.nextInt(random(), 1, 3);
      int numShould = 0;
      for (int i = 0; i < numClauses; ++i) {
        final Occur occur = RandomPicks.randomFrom(random(), Occur.values());
        bq.add(buildRandomQuery(level + 1), occur);
        if (occur == Occur.SHOULD) {
          numShould++;
        }
      }
      bq.setMinimumNumberShouldMatch(TestUtil.nextInt(random(), 0, numShould));
      return bq.build();
    case 2:
      Term t1 = randomTerm();
      Term t2 = randomTerm();
      PhraseQuery pq = new PhraseQuery(random().nextInt(2), t1.field(), t1.bytes(), t2.bytes());
      return pq;
    case 3:
      return new MatchAllDocsQuery();
    case 4:
      return new ConstantScoreQuery(buildRandomQuery(level + 1));
    case 5:
      List<Query> disjuncts = new ArrayList<>();
      final int numQueries = TestUtil.nextInt(random(), 1, 3);
      for (int i = 0; i < numQueries; ++i) {
        disjuncts.add(buildRandomQuery(level + 1));
      }
      return new DisjunctionMaxQuery(disjuncts, random().nextFloat());
    default:
      throw new AssertionError();
  }
}
 

private static Term randomTerm() {
  final String term = RandomPicks.randomFrom(random(), Arrays.asList("foo", "bar", "baz"));
  return new Term("foo", term);
}
 

@Nightly
public void testBigDocuments() throws IOException {
  assumeWorkingMMapOnWindows();
  
  // "big" as "much bigger than the chunk size"
  // for this test we force a FS dir
  // we can't just use newFSDirectory, because this test doesn't really index anything.
  // so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
  Directory dir = new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("testBigDocuments")));
  IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
  iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);

  if (dir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper) dir).setThrottling(Throttling.NEVER);
  }

  final Document emptyDoc = new Document(); // emptyDoc
  final Document bigDoc1 = new Document(); // lot of small fields
  final Document bigDoc2 = new Document(); // 1 very big field

  final Field idField = new StringField("id", "", Store.NO);
  emptyDoc.add(idField);
  bigDoc1.add(idField);
  bigDoc2.add(idField);

  final FieldType onlyStored = new FieldType(StringField.TYPE_STORED);
  onlyStored.setIndexOptions(IndexOptions.NONE);

  final Field smallField = new Field("fld", randomByteArray(random().nextInt(10), 256), onlyStored);
  final int numFields = RandomNumbers.randomIntBetween(random(), 500000, 1000000);
  for (int i = 0; i < numFields; ++i) {
    bigDoc1.add(smallField);
  }

  final Field bigField = new Field("fld", randomByteArray(RandomNumbers.randomIntBetween(random(), 1000000, 5000000), 2), onlyStored);
  bigDoc2.add(bigField);

  final int numDocs = atLeast(5);
  final Document[] docs = new Document[numDocs];
  for (int i = 0; i < numDocs; ++i) {
    docs[i] = RandomPicks.randomFrom(random(), Arrays.asList(emptyDoc, bigDoc1, bigDoc2));
  }
  for (int i = 0; i < numDocs; ++i) {
    idField.setStringValue("" + i);
    iw.addDocument(docs[i]);
    if (random().nextInt(numDocs) == 0) {
      iw.commit();
    }
  }
  iw.commit();
  iw.forceMerge(1); // look at what happens when big docs are merged
  final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
  final IndexSearcher searcher = new IndexSearcher(rd);
  for (int i = 0; i < numDocs; ++i) {
    final Query query = new TermQuery(new Term("id", "" + i));
    final TopDocs topDocs = searcher.search(query, 1);
    assertEquals("" + i, 1, topDocs.totalHits.value);
    final Document doc = rd.document(topDocs.scoreDocs[0].doc);
    assertNotNull(doc);
    final IndexableField[] fieldValues = doc.getFields("fld");
    assertEquals(docs[i].getFields("fld").length, fieldValues.length);
    if (fieldValues.length > 0) {
      assertEquals(docs[i].getFields("fld")[0].binaryValue(), fieldValues[0].binaryValue());
    }
  }
  rd.close();
  iw.close();
  dir.close();
}
 

/** Pick a random object from the given array. The array must not be empty. */
public static <T> T randomFrom(Random random, T... array) {
    return RandomPicks.randomFrom(random, array);
}
 

protected PlanetModel randomPlanetModel() {
  return RandomPicks.randomFrom(random(), new PlanetModel[] {PlanetModel.WGS84, PlanetModel.CLARKE_1866, PlanetModel.SPHERE});
}