Java Code Examples for org.apache.lucene.store.IndexInput#length()

public static void readRandomData(IndexInput baseInput, IndexInput testInput, Random random, int sampleSize,
    int maxBufSize, int maxOffset) throws IOException {
  assertEquals(baseInput.length(), testInput.length());
  int fileLength = (int) baseInput.length();
  for (int i = 0; i < sampleSize; i++) {
    int position = random.nextInt(fileLength - maxBufSize);
    int bufSize = random.nextInt(maxBufSize - maxOffset) + 1;
    byte[] buf1 = new byte[bufSize];
    byte[] buf2 = new byte[bufSize];

    int offset = random.nextInt(Math.min(maxOffset, bufSize));
    int len = Math.min(random.nextInt(bufSize - offset), fileLength - position);

    baseInput.seek(position);
    baseInput.readBytes(buf1, offset, len);
    testInput.seek(position);
    testInput.readBytes(buf2, offset, len);
    assertArrayEquals("Read [" + i + "] The position is [" + position + "] and bufSize [" + bufSize + "]", buf1, buf2);
  }
}
 

public CacheIndexInput(CacheDirectory directory, String fileName, IndexInput indexInput, Cache cache)
    throws IOException {
  super("CacheIndexInput(" + indexInput.toString() + ")");
  _directory = directory;
  _fileName = fileName;
  _indexInput = indexInput;
  _fileLength = indexInput.length();
  _cache = cache;

  _fileId = _cache.getFileId(_directory, _fileName);
  _cacheBlockSize = _cache.getCacheBlockSize(_directory, _fileName);
  _indexInputCache = _cache.createIndexInputCache(_directory, _fileName, _fileLength);
  _bufferSize = _cache.getFileBufferSize(_directory, _fileName);
  _quiet = _cache.shouldBeQuiet(_directory, _fileName);
  _key.setFileId(_fileId);
  _isClosed = false;
  _store = BufferStore.instance(_bufferSize);
}
 

private static void validateFooter(IndexInput in) throws IOException {
  long remaining = in.length() - in.getFilePointer();
  long expected = footerLength();
  if (remaining < expected) {
    throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
  } else if (remaining > expected) {
    throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
  }
  
  final int magic = in.readInt();
  if (magic != FOOTER_MAGIC) {
    throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in);
  }
  
  final int algorithmID = in.readInt();
  if (algorithmID != 0) {
    throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in);
  }
}
 

protected static void assertSameStreams(String msg, IndexInput expected, IndexInput test) throws IOException {
  assertNotNull(msg + " null expected", expected);
  assertNotNull(msg + " null test", test);
  assertEquals(msg + " length", expected.length(), test.length());
  assertEquals(msg + " position", expected.getFilePointer(), test.getFilePointer());
  
  byte expectedBuffer[] = new byte[512];
  byte testBuffer[] = new byte[expectedBuffer.length];
  
  long remainder = expected.length() - expected.getFilePointer();
  while (remainder > 0) {
    int readLen = (int) Math.min(remainder, expectedBuffer.length);
    expected.readBytes(expectedBuffer, 0, readLen);
    test.readBytes(testBuffer, 0, readLen);
    assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
    remainder -= readLen;
  }
}
 

@Test
public void testOpen() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev = new IndexRevision(writer);
    @SuppressWarnings("unchecked")
    Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    String source = sourceFiles.keySet().iterator().next();
    for (RevisionFile file : sourceFiles.values().iterator().next()) {
      IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
      InputStream in = rev.open(source, file.fileName);
      assertEquals(src.length(), in.available());
      byte[] srcBytes = new byte[(int) src.length()];
      byte[] inBytes = new byte[(int) src.length()];
      int offset = 0;
      if (random().nextBoolean()) {
        int skip = random().nextInt(10);
        if (skip >= src.length()) {
          skip = 0;
        }
        in.skip(skip);
        src.seek(skip);
        offset = skip;
      }
      src.readBytes(srcBytes, offset, srcBytes.length - offset);
      in.read(inBytes, offset, inBytes.length - offset);
      assertArrayEquals(srcBytes, inBytes);
      IOUtils.close(src, in);
    }
    writer.close();
  } finally {
    IOUtils.close(dir);
  }
}
 

@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
  IndexInput in = super.openInput(name, context);
  final FixedBitSet set = readBytes.computeIfAbsent(name, n -> new FixedBitSet(Math.toIntExact(in.length())));
  if (set.length() != in.length()) {
    throw new IllegalStateException();
  }
  return new ReadBytesIndexInputWrapper(in, set::set);
}
 

protected static void assertSameStreams(String msg, IndexInput expected, IndexInput actual, long seekTo) throws IOException {
  if (seekTo >= 0 && seekTo < expected.length()) {
    expected.seek(seekTo);
    actual.seek(seekTo);
    assertSameStreams(msg + ", seek(mid)", expected, actual);
  }
}
 

private void read(IndexInput input) throws IOException {
  byte[] buf = new byte[10000];
  long length = input.length();
  while (length > 0) {
    int len = (int) Math.min(length, buf.length);
    input.readBytes(buf, 0, len);
    length -= len;
  }
}
 

public static void readRandomDataInt(IndexInput baseInput, IndexInput testInput, Random random, int sampleSize)
    throws IOException {
  assertEquals(baseInput.length(), testInput.length());
  int fileLength = (int) baseInput.length();
  for (int i = 0; i < sampleSize; i++) {
    int position = random.nextInt(fileLength - 4);
    baseInput.seek(position);
    int i1 = baseInput.readInt();
    testInput.seek(position);
    int i2 = testInput.readInt();
    assertEquals("Read [" + i + "] The position is [" + position + "]", i1, i2);
  }
}
 

public static void readRandomDataLong(IndexInput baseInput, IndexInput testInput, Random random, int sampleSize)
    throws IOException {
  assertEquals(baseInput.length(), testInput.length());
  int fileLength = (int) baseInput.length();
  for (int i = 0; i < sampleSize; i++) {
    int position = random.nextInt(fileLength - 8);
    baseInput.seek(position);
    long i1 = baseInput.readLong();
    testInput.seek(position);
    long i2 = testInput.readLong();
    assertEquals("Read [" + i + "] The position is [" + position + "]", i1, i2);
  }
}
 

/** 
 * Returns (but does not validate) the checksum previously written by {@link #checkFooter}.
 * @return actual checksum value
 * @throws IOException if the footer is invalid
 */
public static long retrieveChecksum(IndexInput in) throws IOException {
  if (in.length() < footerLength()) {
    throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in);
  }
  in.seek(in.length() - footerLength());
  validateFooter(in);
  return readCRC(in);
}
 

/** 
 * Returns (but does not validate) the checksum previously written by {@link #checkFooter}.
 * @return actual checksum value
 * @throws IOException if the footer is invalid
 */
public static long retrieveChecksum(IndexInput in, long expectedLength) throws IOException {
  if (expectedLength < footerLength()) {
    throw new IllegalArgumentException("expectedLength cannot be less than the footer length");
  }
  if (in.length() < expectedLength) {
    throw new CorruptIndexException("truncated file: length=" + in.length() + " but expectedLength==" + expectedLength, in);
  } else if (in.length() > expectedLength) {
    throw new CorruptIndexException("file too long: length=" + in.length() + " but expectedLength==" + expectedLength, in);
  }

  return retrieveChecksum(in);
}
 

public void copyFile(Directory dir, String src, String dest) throws IOException {
  IndexInput in = dir.openInput(src, newIOContext(random()));
  IndexOutput out = dir.createOutput(dest, newIOContext(random()));
  byte[] b = new byte[1024];
  long remainder = in.length();
  while(remainder > 0) {
    int len = (int) Math.min(b.length, remainder);
    in.readBytes(b, 0, len);
    out.writeBytes(b, len);
    remainder -= len;
  }
  in.close();
  out.close();
}
 

public CachedIndexInput(IndexInput source, int blockSize, String name,
    String cacheName, Cache cache, int bufferSize) {
  super(name, bufferSize);
  this.source = source;
  this.blockSize = blockSize;
  fileLength = source.length();
  this.cacheName = cacheName;
  this.cache = cache;
  store = BufferStore.instance(blockSize);
}
 

@Override
public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segment, "", Blur022SegmentInfoFormat.SI_EXTENSION);
  final IndexInput input = dir.openInput(fileName, context);
  boolean success = false;
  try {
    CodecUtil.checkHeader(input, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_START,
        Blur022SegmentInfoFormat.VERSION_CURRENT);
    final String version = input.readString();
    final int docCount = input.readInt();
    if (docCount < 0) {
      throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")");
    }
    final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
    final Map<String, String> diagnostics = input.readStringStringMap();
    final Map<String, String> attributes = input.readStringStringMap();
    final Set<String> files = input.readStringSet();

    if (input.getFilePointer() != input.length()) {
      throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read "
          + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
    }

    final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics,
        Collections.unmodifiableMap(attributes));
    si.setFiles(files);

    success = true;

    return si;

  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(input);
    } else {
      input.close();
    }
  }
}
 

public static void readRandomDataShort(IndexInput baseInput, IndexInput testInput, Random random, int sampleSize)
    throws IOException {
  assertEquals(baseInput.length(), testInput.length());
  int fileLength = (int) baseInput.length();
  for (int i = 0; i < sampleSize; i++) {
    int position = random.nextInt(fileLength - 2);
    baseInput.seek(position);
    short i1 = baseInput.readShort();
    testInput.seek(position);
    short i2 = testInput.readShort();
    assertEquals("Read [" + i + "] The position is [" + position + "]", i1, i2);
  }
}
 

public InputStreamIndexInput(IndexInput indexInput, long limit) {
    this.indexInput = indexInput;
    this.limit = limit;
    if ((indexInput.length() - indexInput.getFilePointer()) > limit) {
        actualSizeToRead = limit;
    } else {
        actualSizeToRead = indexInput.length() - indexInput.getFilePointer();
    }
}
 

ProgressableIndexInput(String name, IndexInput indexInput, int buffer, Progressable progressable) {
  super("ProgressableIndexInput(" + indexInput.toString() + ")", buffer);
  _indexInput = indexInput;
  _length = indexInput.length();
  _progressable = progressable;
}
 

public IndexInputInputStream(IndexInput in) {
  this.in = in;
  remaining = in.length();
}
 

VerifyingIndexInput(IndexInput input, Checksum digest) {
    super("VerifyingIndexInput(" + input + ")");
    this.input = input;
    this.digest = digest;
    checksumPosition = input.length() - 8;
}