Java Code Examples for org.apache.lucene.store.IndexInput#readBytes()

private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, BKDReaderDocIDSetIterator scratchIterator, int count, IntersectVisitor visitor, int compressedDim) throws IOException {
  // the byte at `compressedByteOffset` is compressed using run-length compression,
  // other suffix bytes are stored verbatim
  final int compressedByteOffset = compressedDim * bytesPerDim + commonPrefixLengths[compressedDim];
  commonPrefixLengths[compressedDim]++;
  int i;
  for (i = 0; i < count; ) {
    scratchPackedValue[compressedByteOffset] = in.readByte();
    final int runLen = Byte.toUnsignedInt(in.readByte());
    for (int j = 0; j < runLen; ++j) {
      for(int dim = 0; dim < numDataDims; dim++) {
        int prefix = commonPrefixLengths[dim];
        in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
      }
      visitor.visit(scratchIterator.docIDs[i+j], scratchPackedValue);
    }
    i += runLen;
  }
  if (i != count) {
    throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in);
  }
}
 

private void visitSparseRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, BKDReaderDocIDSetIterator scratchIterator, int count, IntersectVisitor visitor) throws IOException {
  int i;
  for (i = 0; i < count;) {
    int length = in.readVInt();
    for(int dim = 0; dim < numDataDims; dim++) {
      int prefix = commonPrefixLengths[dim];
      in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
    }
    scratchIterator.reset(i, length);
    visitor.visit(scratchIterator, scratchPackedValue);
    i += length;
  }
  if (i != count) {
    throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in);
  }
}
 

/** Read this many bytes from in */
public void copy(IndexInput in, long byteCount) throws IOException {
  while (byteCount > 0) {
    int left = blockSize - upto;
    if (left == 0) {
      if (currentBlock != null) {
        addBlock(currentBlock);
      }
      currentBlock = new byte[blockSize];
      upto = 0;
      left = blockSize;
    }
    if (left < byteCount) {
      in.readBytes(currentBlock, upto, left, false);
      upto = blockSize;
      byteCount -= left;
    } else {
      in.readBytes(currentBlock, upto, (int) byteCount, false);
      upto += byteCount;
      break;
    }
  }
}
 

@Test
public void test1() throws IOException {
  Random random = new Random(seed);
  RAMDirectory directory = new RAMDirectory();

  Cache cache = CacheIndexInputTest.getCache();
  CacheIndexOutput indexOutput = new CacheIndexOutput(null, "test", cache, directory, IOContext.DEFAULT);
  indexOutput.writeByte((byte) 1);
  indexOutput.writeByte((byte) 2);
  byte[] b = new byte[16000];
  random.nextBytes(b);
  indexOutput.writeBytes(b, 16000);
  indexOutput.close();

  IndexInput input = directory.openInput("test", IOContext.DEFAULT);
  assertEquals(16002, input.length());
  assertEquals(1, input.readByte());
  assertEquals(2, input.readByte());

  byte[] buf = new byte[16000];
  input.readBytes(buf, 0, 16000);
  input.close();
  assertArrayEquals(b, buf);
  directory.close();
}
 

/** Retrieves the full index header from the provided {@link IndexInput}.
 *  This throws {@link CorruptIndexException} if this file does
 * not appear to be an index file. */
public static byte[] readIndexHeader(IndexInput in) throws IOException {
  in.seek(0);
  final int actualHeader = in.readInt();
  if (actualHeader != CODEC_MAGIC) {
    throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC, in);
  }
  String codec = in.readString();
  in.readInt();
  in.seek(in.getFilePointer() + StringHelper.ID_LENGTH);
  int suffixLength = in.readByte() & 0xFF;
  byte[] bytes = new byte[headerLength(codec) + StringHelper.ID_LENGTH + 1 + suffixLength];
  in.seek(0);
  in.readBytes(bytes, 0, bytes.length);
  return bytes;
}
 

protected static void assertSameStreams(String msg, IndexInput expected, IndexInput test) throws IOException {
  assertNotNull(msg + " null expected", expected);
  assertNotNull(msg + " null test", test);
  assertEquals(msg + " length", expected.length(), test.length());
  assertEquals(msg + " position", expected.getFilePointer(), test.getFilePointer());
  
  byte expectedBuffer[] = new byte[512];
  byte testBuffer[] = new byte[expectedBuffer.length];
  
  long remainder = expected.length() - expected.getFilePointer();
  while (remainder > 0) {
    int readLen = (int) Math.min(remainder, expectedBuffer.length);
    expected.readBytes(expectedBuffer, 0, readLen);
    test.readBytes(testBuffer, 0, readLen);
    assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
    remainder -= readLen;
  }
}
 

private void readFile(Directory dir, String name) throws Exception {
  IndexInput input = dir.openInput(name, newIOContext(random()));
  try {
    long size = dir.fileLength(name);
    long bytesLeft = size;
    while (bytesLeft > 0) {
      final int numToRead;
      if (bytesLeft < buffer.length)
        numToRead = (int) bytesLeft;
      else
        numToRead = buffer.length;
      input.readBytes(buffer, 0, numToRead, false);
      bytesLeft -= numToRead;
    }
    // Don't do this in your real backups!  This is just
    // to force a backup to take a somewhat long time, to
    // make sure we are exercising the fact that the
    // IndexWriter should not delete this file even when I
    // take my time reading it.
    Thread.sleep(1);
  } finally {
    input.close();
  }
}
 

private static BytesRef readBytesRef(IndexInput in) throws IOException {
  int numBytes = in.readVInt();
  if (numBytes < 0) {
    throw new CorruptIndexException("invalid bytes length: " + numBytes, in);
  }
  
  BytesRef bytes = new BytesRef();
  bytes.length = numBytes;
  bytes.bytes = new byte[numBytes];
  in.readBytes(bytes.bytes, 0, numBytes);

  return bytes;
}
 

private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException {
  for(int dim=0;dim<numDataDims;dim++) {
    int prefix = in.readVInt();
    commonPrefixLengths[dim] = prefix;
    if (prefix > 0) {
      in.readBytes(scratchPackedValue, dim*bytesPerDim, prefix);
    }
    //System.out.println("R: " + dim + " of " + numDims + " prefix=" + prefix);
  }
}
 

private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[] maxPackedValue, IndexInput in) throws IOException {
  for (int dim = 0; dim < numIndexDims; dim++) {
    int prefix = commonPrefixLengths[dim];
    in.readBytes(minPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
    in.readBytes(maxPackedValue, dim * bytesPerDim + prefix, bytesPerDim - prefix);
  }
}
 

/**
 * Expert: verifies the incoming {@link IndexInput} has an index header
 * and that its segment ID matches the expected one, and then copies
 * that index header into the provided {@link DataOutput}.  This is
 * useful when building compound files.
 *
 * @param in Input stream, positioned at the point where the
 *        index header was previously written. Typically this is located
 *        at the beginning of the file.
 * @param out Output stream, where the header will be copied to.
 * @param expectedID Expected segment ID
 * @throws CorruptIndexException If the first four bytes are not
 *         {@link #CODEC_MAGIC}, or if the <code>expectedID</code>
 *         does not match.
 * @throws IOException If there is an I/O error reading from the underlying medium.
 *
 * @lucene.internal 
 */
public static void verifyAndCopyIndexHeader(IndexInput in, DataOutput out, byte[] expectedID) throws IOException {
  // make sure it's large enough to have a header and footer
  if (in.length() < footerLength() + headerLength("")) {
    throw new CorruptIndexException("compound sub-files must have a valid codec header and footer: file is too small (" + in.length() + " bytes)", in);
  }

  int actualHeader = in.readInt();
  if (actualHeader != CODEC_MAGIC) {
    throw new CorruptIndexException("compound sub-files must have a valid codec header and footer: codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CodecUtil.CODEC_MAGIC, in);
  }

  // we can't verify these, so we pass-through:
  String codec = in.readString();
  int version = in.readInt();

  // verify id:
  checkIndexHeaderID(in, expectedID);

  // we can't verify extension either, so we pass-through:
  int suffixLength = in.readByte() & 0xFF;
  byte[] suffixBytes = new byte[suffixLength];
  in.readBytes(suffixBytes, 0, suffixLength);

  // now write the header we just verified
  out.writeInt(CodecUtil.CODEC_MAGIC);
  out.writeString(codec);
  out.writeInt(version);
  out.writeBytes(expectedID, 0, expectedID.length);
  out.writeByte((byte) suffixLength);
  out.writeBytes(suffixBytes, 0, suffixLength);
}
 

@Override
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
  int length = skipStream.readVInt();
  if (impactData[level].length < length) {
    impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
  }
  skipStream.readBytes(impactData[level], 0, length);
  impactDataLength[level] = length;
}
 

private void read(IndexInput input) throws IOException {
  byte[] buf = new byte[10000];
  long length = input.length();
  while (length > 0) {
    int len = (int) Math.min(length, buf.length);
    input.readBytes(buf, 0, len);
    length -= len;
  }
}
 

private static BytesRef readBytesRef(IndexInput in) throws IOException {
  BytesRef bytes = new BytesRef();
  bytes.length = in.readVInt();
  bytes.bytes = new byte[bytes.length];
  in.readBytes(bytes.bytes, 0, bytes.length);
  return bytes;
}
 

private void readCommonPrefixes(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in) throws IOException {
  for(int dim = 0; dim< numDims; dim++) {
    int prefix = in.readVInt();
    commonPrefixLengths[dim] = prefix;
    if (prefix > 0) {
      in.readBytes(scratchPackedValue, dim*bytesPerDim, prefix);
    }
    //System.out.println("R: " + dim + " of " + numDims + " prefix=" + prefix);
  }
}
 

private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) {
  final IndexInput data = this.data.clone();

  return new LongBinaryDocValues() {

    private final ThreadValue<IndexInput> in = new ThreadValue<IndexInput>() {
      @Override
      protected IndexInput initialValue() {
        return data.clone();
      }
    };

    @Override
    public void get(long id, BytesRef result) {
      long address = bytes.offset + id * bytes.maxLength;
      try {
        IndexInput indexInput = in.get();
        indexInput.seek(address);
        // NOTE: we could have one buffer, but various consumers (e.g.
        // FieldComparatorSource)
        // assume "they" own the bytes after calling this!
        final byte[] buffer = new byte[bytes.maxLength];
        indexInput.readBytes(buffer, 0, buffer.length);
        result.bytes = buffer;
        result.offset = 0;
        result.length = buffer.length;
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  };
}
 

public void copyFile(Directory dir, String src, String dest) throws IOException {
  IndexInput in = dir.openInput(src, newIOContext(random()));
  IndexOutput out = dir.createOutput(dest, newIOContext(random()));
  byte[] b = new byte[1024];
  long remainder = in.length();
  while(remainder > 0) {
    int len = (int) Math.min(b.length, remainder);
    in.readBytes(b, 0, len);
    out.writeBytes(b, len);
    remainder -= len;
  }
  in.close();
  out.close();
}
 

@Test
public void testOpen() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev = new IndexRevision(writer);
    @SuppressWarnings("unchecked")
    Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    String source = sourceFiles.keySet().iterator().next();
    for (RevisionFile file : sourceFiles.values().iterator().next()) {
      IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
      InputStream in = rev.open(source, file.fileName);
      assertEquals(src.length(), in.available());
      byte[] srcBytes = new byte[(int) src.length()];
      byte[] inBytes = new byte[(int) src.length()];
      int offset = 0;
      if (random().nextBoolean()) {
        int skip = random().nextInt(10);
        if (skip >= src.length()) {
          skip = 0;
        }
        in.skip(skip);
        src.seek(skip);
        offset = skip;
      }
      src.readBytes(srcBytes, offset, srcBytes.length - offset);
      in.read(inBytes, offset, inBytes.length - offset);
      assertArrayEquals(srcBytes, inBytes);
      IOUtils.close(src, in);
    }
    writer.close();
  } finally {
    IOUtils.close(dir);
  }
}
 

public void write(OutputStream out) throws IOException {
  createOutputStream(out);

  IndexInput in = null;
  try {
    initWrite();

    Directory dir = core.withSearcher(searcher -> searcher.getIndexReader().directory());
    in = dir.openInput(fileName, IOContext.READONCE);
    // if offset is mentioned move the pointer to that point
    if (offset != -1) in.seek(offset);

    long filelen = dir.fileLength(fileName);
    long maxBytesBeforePause = 0;

    while (true) {
      offset = offset == -1 ? 0 : offset;
      int read = (int) Math.min(buf.length, filelen - offset);
      in.readBytes(buf, 0, read);

      fos.writeInt(read);
      if (useChecksum) {
        checksum.reset();
        checksum.update(buf, 0, read);
        fos.writeLong(checksum.getValue());
      }
      fos.write(buf, 0, read);
      fos.flush();
      log.debug("Wrote {} bytes for file {}", offset + read, fileName); // logOK

      //Pause if necessary
      maxBytesBeforePause += read;
      if (maxBytesBeforePause >= rateLimiter.getMinPauseCheckBytes()) {
        rateLimiter.pause(maxBytesBeforePause);
        maxBytesBeforePause = 0;
      }
      if (read != buf.length) {
        writeNothingAndFlush();
        fos.close(); // we close because DeflaterOutputStream requires a close call, but but the request outputstream is protected
        break;
      }
      offset += read;
      in.seek(offset);
    }
  } catch (IOException e) {
    log.warn("Exception while writing response for params: {}", params, e);
  } finally {
    if (in != null) {
      in.close();
    }
    extendReserveAndReleaseCommitPoint();
  }
}
 

@Test
public void testOpen() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  try {
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    for (Entry<String,List<RevisionFile>> e : rev.getSourceFiles().entrySet()) {
      String source = e.getKey();
      @SuppressWarnings("resource") // silly, both directories are closed in the end
      Directory dir = source.equals(IndexAndTaxonomyRevision.INDEX_SOURCE) ? indexDir : taxoDir;
      for (RevisionFile file : e.getValue()) {
        IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
        InputStream in = rev.open(source, file.fileName);
        assertEquals(src.length(), in.available());
        byte[] srcBytes = new byte[(int) src.length()];
        byte[] inBytes = new byte[(int) src.length()];
        int offset = 0;
        if (random().nextBoolean()) {
          int skip = random().nextInt(10);
          if (skip >= src.length()) {
            skip = 0;
          }
          in.skip(skip);
          src.seek(skip);
          offset = skip;
        }
        src.readBytes(srcBytes, offset, srcBytes.length - offset);
        in.read(inBytes, offset, inBytes.length - offset);
        assertArrayEquals(srcBytes, inBytes);
        IOUtils.close(src, in);
      }
    }
    indexWriter.close();
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
  }
}