org.apache.lucene.store.IndexInput Java Examples

public static void readRandomData(IndexInput baseInput, IndexInput testInput, Random random, int sampleSize,
    int maxBufSize, int maxOffset) throws IOException {
  assertEquals(baseInput.length(), testInput.length());
  int fileLength = (int) baseInput.length();
  for (int i = 0; i < sampleSize; i++) {
    int position = random.nextInt(fileLength - maxBufSize);
    int bufSize = random.nextInt(maxBufSize - maxOffset) + 1;
    byte[] buf1 = new byte[bufSize];
    byte[] buf2 = new byte[bufSize];

    int offset = random.nextInt(Math.min(maxOffset, bufSize));
    int len = Math.min(random.nextInt(bufSize - offset), fileLength - position);

    baseInput.seek(position);
    baseInput.readBytes(buf1, offset, len);
    testInput.seek(position);
    testInput.readBytes(buf2, offset, len);
    assertArrayEquals("Read [" + i + "] The position is [" + position + "] and bufSize [" + bufSize + "]", buf1, buf2);
  }
}
 

/** simple encode/decode */
public void testSimple() throws Exception {
  Directory dir = newDirectory();
  int bitsPerValue = DirectWriter.bitsRequired(2);
  IndexOutput output = dir.createOutput("foo", IOContext.DEFAULT);
  DirectWriter writer = DirectWriter.getInstance(output, 5, bitsPerValue);
  writer.add(1);
  writer.add(0);
  writer.add(2);
  writer.add(1);
  writer.add(2);
  writer.finish();
  output.close();
  IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
  LongValues reader = DirectReader.getInstance(input.randomAccessSlice(0, input.length()), bitsPerValue, 0);
  assertEquals(1, reader.get(0));
  assertEquals(0, reader.get(1));
  assertEquals(2, reader.get(2));
  assertEquals(1, reader.get(3));
  assertEquals(2, reader.get(4));
  input.close();
  dir.close();
}
 

private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
  CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
  for (Impact impact : impacts) {
    acc.add(impact.freq, impact.norm);
  }
  try(Directory dir = newDirectory()) {
    try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
      Lucene84SkipWriter.writeImpacts(acc, out);
    }
    try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
      byte[] b = new byte[Math.toIntExact(in.length())];
      in.readBytes(b, 0, b.length);
      List<Impact> impacts2 = Lucene84ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new MutableImpactList());
      assertEquals(impacts, impacts2);
    }
  }
}
 

private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
  switch(b) {
  case 0:
    return DocValuesType.NONE;
  case 1:
    return DocValuesType.NUMERIC;
  case 2:
    return DocValuesType.BINARY;
  case 3:
    return DocValuesType.SORTED;
  case 4:
    return DocValuesType.SORTED_SET;
  case 5:
    return DocValuesType.SORTED_NUMERIC;
  default:
    throw new CorruptIndexException("invalid docvalues byte: " + b, input);
  }
}
 

/**
 * Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} on or off-heap
 * depending on the provided <code>fstLoadMode</code>
 */
public static NRTSuggester load(IndexInput input, FSTLoadMode fstLoadMode) throws IOException {
  final FST<Pair<Long, BytesRef>> fst;
  if (shouldLoadFSTOffHeap(input, fstLoadMode)) {
    OffHeapFSTStore store = new OffHeapFSTStore();
    IndexInput clone = input.clone();
    clone.seek(input.getFilePointer());
    fst = new FST<>(clone, clone, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()), store);
    input.seek(clone.getFilePointer() + store.size());
  } else {
    fst = new FST<>(input, input, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  }

  /* read some meta info */
  int maxAnalyzedPathsPerOutput = input.readVInt();
  /*
   * Label used to denote the end of an input in the FST and
   * the beginning of dedup bytes
   */
  int endByte = input.readVInt();
  int payloadSep = input.readVInt();
  return new NRTSuggester(fst, maxAnalyzedPathsPerOutput, payloadSep);
}
 

private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
  final int cardinality = set.cardinality();
  final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
  long length;
  int jumpTableentryCount;
  try (IndexOutput out = dir.createOutput("bar", IOContext.DEFAULT)) {
    jumpTableentryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
  }

  try (IndexInput in = dir.openInput("bar", IOContext.DEFAULT)) {
    BitSetIterator disi2 = new BitSetIterator(set, cardinality);
    int doc = disi2.docID();
    int index = 0;
    while (doc < cardinality) {
      doc = disi2.nextDoc();
      index++;
    }

    IndexedDISI disi = new IndexedDISI(in, 0L, in.length(), jumpTableentryCount, denseRankPower, cardinality);
    // Advance 1 docID beyond end
    assertFalse("There should be no set bit beyond the valid docID range", disi.advanceExact(set.length()));
    disi.advance(doc); // Should be the special docID signifyin NO_MORE_DOCS from the BitSetIterator
    assertEquals("The index when advancing beyond the last defined docID should be correct",
        index, disi.index()+1); // disi.index()+1 as the while-loop also counts the NO_MORE_DOCS
  }
}
 

/**
 * Adds the index input to list.
 *
 * @param name the name
 * @param in the in
 * @param postingsFormatName the postings format name
 * @return the string
 * @throws IOException Signals that an I/O exception has occurred.
 */
private String addIndexInputToList(String name, IndexInput in,
    String postingsFormatName) throws IOException {
  if (indexInputList.get(name) != null) {
    indexInputList.get(name).close();
  }
  if (in != null) {
    String localPostingsFormatName = postingsFormatName;
    if (localPostingsFormatName == null) {
      localPostingsFormatName = in.readString();
    } else if (!in.readString().equals(localPostingsFormatName)) {
      throw new IOException("delegate codec " + name + " doesn't equal "
          + localPostingsFormatName);
    }
    indexInputList.put(name, in);
    indexInputOffsetList.put(name, in.getFilePointer());
    return localPostingsFormatName;
  } else {
    log.debug("no " + name + " registered");
    return null;
  }
}
 

@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
  int delta = skipStream.readVInt();
  docPointer[level] += skipStream.readVLong();

  if (posPointer != null) {
    posPointer[level] += skipStream.readVLong();
    posBufferUpto[level] = skipStream.readVInt();

    if (payloadByteUpto != null) {
      payloadByteUpto[level] = skipStream.readVInt();
    }

    if (payPointer != null) {
      payPointer[level] += skipStream.readVLong();
    }
  }
  readImpacts(level, skipStream);
  return delta;
}
 

public void testReadBogusCRC() throws Exception {
  ByteBuffersDataOutput out = new ByteBuffersDataOutput();
  IndexOutput output = new ByteBuffersIndexOutput(out, "temp", "temp");
  output.writeLong(-1L); // bad
  output.writeLong(1L << 32); // bad
  output.writeLong(-(1L << 32)); // bad
  output.writeLong((1L << 32) - 1); // ok
  output.close();
  IndexInput input = new BufferedChecksumIndexInput(new ByteBuffersIndexInput(out.toDataInput(), "temp"));
  // read 3 bogus values
  for (int i = 0; i < 3; i++) {
    expectThrows(CorruptIndexException.class, () -> {
      CodecUtil.readCRC(input);
    });
  }
  // good value
  CodecUtil.readCRC(input);
}
 

/** Read this many bytes from in */
public void copy(IndexInput in, long byteCount) throws IOException {
  while (byteCount > 0) {
    int left = blockSize - upto;
    if (left == 0) {
      if (currentBlock != null) {
        addBlock(currentBlock);
      }
      currentBlock = new byte[blockSize];
      upto = 0;
      left = blockSize;
    }
    if (left < byteCount) {
      in.readBytes(currentBlock, upto, left, false);
      upto = blockSize;
      byteCount -= left;
    } else {
      in.readBytes(currentBlock, upto, (int) byteCount, false);
      upto += byteCount;
      break;
    }
  }
}
 

/** 
 * This test creates compound file based on a single file.
 * Files of different sizes are tested: 0, 1, 10, 100 bytes.
 */
public void testSingleFile() throws IOException {
  int data[] = new int[] { 0, 1, 10, 100 };
  for (int i=0; i<data.length; i++) {
    String testfile = "_" + i + ".test";
    Directory dir = newDirectory();
    SegmentInfo si = newSegmentInfo(dir, "_" + i);
    createSequenceFile(dir, testfile, (byte) 0, data[i], si.getId(), "suffix");
    
    si.setFiles(Collections.singleton(testfile));
    si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
    Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
    
    IndexInput expected = dir.openInput(testfile, newIOContext(random()));
    IndexInput actual = cfs.openInput(testfile, newIOContext(random()));
    assertSameStreams(testfile, expected, actual);
    assertSameSeekBehavior(testfile, expected, actual);
    expected.close();
    actual.close();
    cfs.close();
    dir.close();
  }
}
 

public CompressedIndexInput(IndexInput in) throws IOException {
    super("compressed(" + in.toString() + ")");
    this.in = in;
    readHeader(in);
    this.version = in.readInt();
    long metaDataPosition = in.readLong();
    long headerLength = in.getFilePointer();
    in.seek(metaDataPosition);
    this.totalUncompressedLength = in.readVLong();
    int size = in.readVInt();
    offsets = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size);
    for (int i = 0; i < size; i++) {
        offsets.set(i, in.readVLong());
    }
    this.currentOffsetIdx = -1;
    this.currentUncompressedChunkPointer = 0;
    in.seek(headerLength);
}
 

public void testRename() throws IOException {
  String[] listAll = directory.listAll();
  for (String file : listAll) {
    directory.deleteFile(file);
  }
  
  IndexOutput output = directory.createOutput("testing.test", new IOContext());
  output.writeInt(12345);
  output.close();
  directory.rename("testing.test", "testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test"));
  assertTrue(slowFileExists(directory, "testing.test.renamed"));
  IndexInput input = directory.openInput("testing.test.renamed", new IOContext());
  assertEquals(12345, input.readInt());
  assertEquals(input.getFilePointer(), input.length());
  input.close();
  directory.deleteFile("testing.test.renamed");
  assertFalse(slowFileExists(directory, "testing.test.renamed"));
}
 

private static Directory copyFilesLocally(Configuration configuration, Directory dir, String table, Path shardDir,
    Path localCachePath, Collection<String> files, String segmentName) throws IOException {
  LOG.info("Copying files need to local cache for faster reads [{0}].", shardDir);
  Path localShardPath = new Path(new Path(new Path(localCachePath, table), shardDir.getName()), segmentName);
  HdfsDirectory localDir = new HdfsDirectory(configuration, localShardPath, null);
  for (String name : files) {
    if (!isValidFileToCache(name)) {
      continue;
    }
    LOG.info("Valid file for local copy [{0}].", name);
    if (!isValid(localDir, dir, name)) {
      LastModified lastModified = (LastModified) dir;
      long fileModified = lastModified.getFileModified(name);

      IndexInput input = dir.openInput(name, IOContext.READONCE);
      IndexOutput output = localDir.createOutput(name, IOContext.READONCE);
      output.copyBytes(input, input.length());
      output.close();
      IndexOutput lastMod = localDir.createOutput(name + LASTMOD, IOContext.DEFAULT);
      lastMod.writeLong(fileModified);
      lastMod.close();
    }
  }
  return localDir;
}
 

/**
 * Read values that have been written using variable-length encoding instead of bit-packing.
 */
static void readVIntBlock(IndexInput docIn, int[] docBuffer,
    int[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
  if (indexHasFreq) {
    for(int i=0;i<num;i++) {
      final int code = docIn.readVInt();
      docBuffer[i] = code >>> 1;
      if ((code & 1) != 0) {
        freqBuffer[i] = 1;
      } else {
        freqBuffer[i] = docIn.readVInt();
      }
    }
  } else {
    for(int i=0;i<num;i++) {
      docBuffer[i] = docIn.readVInt();
    }
  }
}
 

/**
 * Snapshot individual file
 * <p>
 * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
 * added to the {@code failures} list
 *
 * @param fileInfo file to be snapshotted
 */
private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException {
    final String file = fileInfo.physicalName();
    try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
        for (int i = 0; i < fileInfo.numberOfParts(); i++) {
            final long partBytes = fileInfo.partBytes(i);

            final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes);
            InputStream inputStream = snapshotRateLimiter == null ? inputStreamIndexInput : new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter, snapshotThrottleListener);
            inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName());
            blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes);
        }
        Store.verify(indexInput);
        snapshotStatus.addProcessedFile(fileInfo.length());
    } catch (Throwable t) {
        failStoreIfCorrupted(t);
        snapshotStatus.addProcessedFile(0);
        throw t;
    }
}
 

private static void readInts24(IndexInput in, int count, int[] docIDs) throws IOException {
  int i;
  for (i = 0; i < count - 7; i += 8) {
    long l1 = in.readLong();
    long l2 = in.readLong();
    long l3 = in.readLong();
    docIDs[i] =  (int) (l1 >>> 40);
    docIDs[i+1] = (int) (l1 >>> 16) & 0xffffff;
    docIDs[i+2] = (int) (((l1 & 0xffff) << 8) | (l2 >>> 56));
    docIDs[i+3] = (int) (l2 >>> 32) & 0xffffff;
    docIDs[i+4] = (int) (l2 >>> 8) & 0xffffff;
    docIDs[i+5] = (int) (((l2 & 0xff) << 16) | (l3 >>> 48));
    docIDs[i+6] = (int) (l3 >>> 24) & 0xffffff;
    docIDs[i+7] = (int) l3 & 0xffffff;
  }
  for (; i < count; ++i) {
    docIDs[i] = (Short.toUnsignedInt(in.readShort()) << 8) | Byte.toUnsignedInt(in.readByte());
  }
}
 

@Test
public void test1() throws IOException {
  RAMDirectory directory = new RAMDirectory();

  String name = "test";

  IndexOutput output = directory.createOutput(name, IOContext.DEFAULT);
  byte[] bs = "hello world".getBytes();
  output.writeBytes(bs, bs.length);
  output.close();

  IndexInput input = directory.openInput(name, IOContext.DEFAULT);
  Cache cache = getCache();
  CacheIndexInput cacheInput = new CacheIndexInput(null, name, input, cache);
  byte[] buf = new byte[bs.length];
  cacheInput.readBytes(buf, 0, buf.length);
  cacheInput.close();

  assertArrayEquals(bs, buf);
  directory.close();
}
 

/** {@inheritDoc} */
@Override public IndexInput openInput(final String name, final IOContext context) throws IOException {
    ensureOpen();

    GridLuceneFile file = fileMap.get(name);

    if (file == null)
        throw new FileNotFoundException(name);

    // Lock for using in stream. Will be unlocked on stream closing.
    file.lockRef();

    if (!fileMap.containsKey(name)) {
        // Unblock for deferred delete.
        file.releaseRef();

        throw new FileNotFoundException(name);
    }

    return new GridLuceneInputStream(name, file);
}
 

@Override
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
  int length = skipStream.readVInt();
  if (impactData[level].length < length) {
    impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
  }
  skipStream.readBytes(impactData[level], 0, length);
  impactDataLength[level] = length;
}
 

private static BytesRef readBytesRef(IndexInput in) throws IOException {
  BytesRef bytes = new BytesRef();
  bytes.length = in.readVInt();
  bytes.bytes = new byte[bytes.length];
  in.readBytes(bytes.bytes, 0, bytes.length);
  return bytes;
}
 

private void executeReads(HdfsDirectory dir, String name) throws IOException, InterruptedException {
  IndexInput input = dir.openInput(name, IOContext.READ);
  assertResourceCount(1);
  input.readLong();
  input.seek(0L);
  for (int i = 0; i < 2; i++) {
    readSeq(input.clone(), READ_SIZE);
    assertResourceCount(1 + i + 1);
  }
  input.close();
}
 

protected static void assertSameStreams(String msg, IndexInput expected, IndexInput actual, long seekTo) throws IOException {
  if (seekTo >= 0 && seekTo < expected.length()) {
    expected.seek(seekTo);
    actual.seek(seekTo);
    assertSameStreams(msg + ", seek(mid)", expected, actual);
  }
}
 

private String getIndexPropertyFromPropFile(Directory dir) throws IOException {
  IndexInput input;
  try {
    input = dir.openInput(IndexFetcher.INDEX_PROPERTIES, IOContext.DEFAULT);
  } catch (FileNotFoundException | NoSuchFileException e) {
    // Swallow this error, dataDir/index is the right thing to return
    // if there is no index.properties file
    // All other exceptions are will propagate to caller.
    return dataDir + "index/";
  }
  final InputStream is = new PropertiesInputStream(input); // c'tor just assigns a variable here, no exception thrown.
  try {
    Properties p = new Properties();
    p.load(new InputStreamReader(is, StandardCharsets.UTF_8));

    String s = p.getProperty("index");
    if (s != null && s.trim().length() > 0) {
      return dataDir + s.trim();
    }

    // We'll return dataDir/index/ if the properties file has an "index" property with
    // no associated value or does not have an index property at all.
    return dataDir + "index/";
  } finally {
    IOUtils.closeQuietly(is);
  }
}
 

/**
 * This constructor allows to pass the slice and jumpTable directly in case it helps reuse.
 * see eg. Lucene80 norms producer's merge instance.
 * @param blockSlice data blocks, normally created by {@link #createBlockSlice}.
 * @param jumpTable table holding jump-data for block-skips, normally created by {@link #createJumpTable}.
 * @param jumpTableEntryCount the number of blocks covered by the jump-table.
 *                            This must match the number returned by {@link #writeBitSet(DocIdSetIterator, IndexOutput, byte)}.
 * @param denseRankPower the number of docIDs covered by each rank entry in DENSE blocks, expressed as {@code 2^denseRankPower}.
 *                       This must match the power given in {@link #writeBitSet(DocIdSetIterator, IndexOutput, byte)}
 * @param cost normally the number of logical docIDs.
 */
IndexedDISI(IndexInput blockSlice, RandomAccessInput jumpTable, int jumpTableEntryCount, byte denseRankPower, long cost) throws IOException {
  if ((denseRankPower < 7 || denseRankPower > 15) && denseRankPower != -1) {
    throw new IllegalArgumentException("Acceptable values for denseRankPower are 7-15 (every 128-32768 docIDs). " +
        "The provided power was " + denseRankPower + " (every " + (int)Math.pow(2, denseRankPower) + " docIDs). ");
  }

  this.slice = blockSlice;
  this.jumpTable = jumpTable;
  this.jumpTableEntryCount = jumpTableEntryCount;
  this.denseRankPower = denseRankPower;
  final int rankIndexShift = denseRankPower-7;
  this.denseRankTable = denseRankPower == -1 ? null : new byte[DENSE_BLOCK_LONGS >> rankIndexShift];
  this.cost = cost;
}
 

private static BytesRef readBytesRef(IndexInput in) throws IOException {
  BytesRef bytes = new BytesRef();
  bytes.length = in.readVInt();
  bytes.bytes = new byte[bytes.length];
  in.readBytes(bytes.bytes, 0, bytes.length);
  return bytes;
}
 

/**
 * Gets the term.
 *
 * @param inTerm the in term
 * @param ref the ref
 * @return the term
 * @throws IOException Signals that an I/O exception has occurred.
 */
public static String getTerm(IndexInput inTerm, Long ref) throws IOException {
  try {
    inTerm.seek(ref);
    return inTerm.readString();
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

/** 
 * Returns (but does not validate) the checksum previously written by {@link #checkFooter}.
 * @return actual checksum value
 * @throws IOException if the footer is invalid
 */
public static long retrieveChecksum(IndexInput in, long expectedLength) throws IOException {
  if (expectedLength < footerLength()) {
    throw new IllegalArgumentException("expectedLength cannot be less than the footer length");
  }
  if (in.length() < expectedLength) {
    throw new CorruptIndexException("truncated file: length=" + in.length() + " but expectedLength==" + expectedLength, in);
  } else if (in.length() > expectedLength) {
    throw new CorruptIndexException("file too long: length=" + in.length() + " but expectedLength==" + expectedLength, in);
  }

  return retrieveChecksum(in);
}
 

@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
  if (sleepMillis != -1) {
    return new SlowIndexInput(super.openInput(name, context));
  } 
  return super.openInput(name, context);
}
 

private int readCompressedDim(IndexInput in) throws IOException {
  int compressedDim = in.readByte();
  if (compressedDim < -2 || compressedDim >= numDataDims || (version < BKDWriter.VERSION_LOW_CARDINALITY_LEAVES && compressedDim == -2)) {
    throw new CorruptIndexException("Got compressedDim="+compressedDim, in);
  }
  return compressedDim;
}