Java Code Examples for org.apache.lucene.store.IndexInput#readVInt()

public CompressedIndexInput(IndexInput in) throws IOException {
    super("compressed(" + in.toString() + ")");
    this.in = in;
    readHeader(in);
    this.version = in.readInt();
    long metaDataPosition = in.readLong();
    long headerLength = in.getFilePointer();
    in.seek(metaDataPosition);
    this.totalUncompressedLength = in.readVLong();
    int size = in.readVInt();
    offsets = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size);
    for (int i = 0; i < size; i++) {
        offsets.set(i, in.readVLong());
    }
    this.currentOffsetIdx = -1;
    this.currentUncompressedChunkPointer = 0;
    in.seek(headerLength);
}
 

/**
 * Instantiates a new index doc.
 *
 * @param ref
 *          the ref
 * @throws IOException
 *           Signals that an I/O exception has occurred.
 */
public IndexDoc(Long ref) throws IOException {
  try {
    IndexInput inIndexDoc = indexInputList.get("doc");
    if (ref != null) {
      inIndexDoc.seek(ref);
    }
    docId = inIndexDoc.readVInt(); // docId
    fpIndexObjectId = inIndexDoc.readVLong(); // ref indexObjectId
    fpIndexObjectPosition = inIndexDoc.readVLong(); // ref
                                                    // indexObjectPosition
    fpIndexObjectParent = inIndexDoc.readVLong(); // ref indexObjectParent
    smallestObjectFilepointer = inIndexDoc.readVLong(); // offset
    objectRefApproxQuotient = inIndexDoc.readVInt(); // slope
    objectRefApproxOffset = inIndexDoc.readZLong(); // offset
    storageFlags = inIndexDoc.readByte(); // flag
    size = inIndexDoc.readVInt(); // number of objects
    minPosition = inIndexDoc.readVInt(); // minimum position
    maxPosition = inIndexDoc.readVInt(); // maximum position
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

private void visitSparseRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, BKDReaderDocIDSetIterator scratchIterator, int count, IntersectVisitor visitor) throws IOException {
  int i;
  for (i = 0; i < count;) {
    int length = in.readVInt();
    for(int dim = 0; dim < numDataDims; dim++) {
      int prefix = commonPrefixLengths[dim];
      in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
    }
    scratchIterator.reset(i, length);
    visitor.visit(scratchIterator, scratchPackedValue);
    i += length;
  }
  if (i != count) {
    throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in);
  }
}
 

/**
 * Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} on or off-heap
 * depending on the provided <code>fstLoadMode</code>
 */
public static NRTSuggester load(IndexInput input, FSTLoadMode fstLoadMode) throws IOException {
  final FST<Pair<Long, BytesRef>> fst;
  if (shouldLoadFSTOffHeap(input, fstLoadMode)) {
    OffHeapFSTStore store = new OffHeapFSTStore();
    IndexInput clone = input.clone();
    clone.seek(input.getFilePointer());
    fst = new FST<>(clone, clone, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()), store);
    input.seek(clone.getFilePointer() + store.size());
  } else {
    fst = new FST<>(input, input, new PairOutputs<>(
        PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  }

  /* read some meta info */
  int maxAnalyzedPathsPerOutput = input.readVInt();
  /*
   * Label used to denote the end of an input in the FST and
   * the beginning of dedup bytes
   */
  int endByte = input.readVInt();
  int payloadSep = input.readVInt();
  return new NRTSuggester(fst, maxAnalyzedPathsPerOutput, payloadSep);
}
 

@Override
public FieldsProducer fieldsProducer(SegmentReadState readState)
  throws IOException {

  // Load our ID:
  final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION);
  IndexInput in = readState.directory.openInput(idFileName, readState.context);
  boolean success = false;
  final int id;
  try {
    CodecUtil.checkHeader(in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST);
    id = in.readVInt();
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(in);
    } else {
      IOUtils.close(in);
    }
  }
  
  synchronized(state) {
    return state.get(id);
  }
}
 

/**
 * Read values that have been written using variable-length encoding instead of bit-packing.
 */
static void readVIntBlock(IndexInput docIn, int[] docBuffer,
    int[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
  if (indexHasFreq) {
    for(int i=0;i<num;i++) {
      final int code = docIn.readVInt();
      docBuffer[i] = code >>> 1;
      if ((code & 1) != 0) {
        freqBuffer[i] = 1;
      } else {
        freqBuffer[i] = docIn.readVInt();
      }
    }
  } else {
    for(int i=0;i<num;i++) {
      docBuffer[i] = docIn.readVInt();
    }
  }
}
 

/**
 * Read values that have been written using variable-length encoding instead of bit-packing.
 */
static void readVIntBlock(IndexInput docIn, long[] docBuffer,
    long[] freqBuffer, int num, boolean indexHasFreq) throws IOException {
  if (indexHasFreq) {
    for(int i=0;i<num;i++) {
      final int code = docIn.readVInt();
      docBuffer[i] = code >>> 1;
      if ((code & 1) != 0) {
        freqBuffer[i] = 1;
      } else {
        freqBuffer[i] = docIn.readVInt();
      }
    }
  } else {
    for(int i=0;i<num;i++) {
      docBuffer[i] = docIn.readVInt();
    }
  }
}
 

@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
  int delta = skipStream.readVInt();
  docPointer[level] += skipStream.readVLong();

  if (posPointer != null) {
    posPointer[level] += skipStream.readVLong();
    posBufferUpto[level] = skipStream.readVInt();

    if (payloadByteUpto != null) {
      payloadByteUpto[level] = skipStream.readVInt();
    }

    if (payPointer != null) {
      payPointer[level] += skipStream.readVLong();
    }
  }
  readImpacts(level, skipStream);
  return delta;
}
 

@Override
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
  int length = skipStream.readVInt();
  if (impactData[level].length < length) {
    impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
  }
  skipStream.readBytes(impactData[level], 0, length);
  impactDataLength[level] = length;
}
 

private static void readDeltaVInts(IndexInput in, int count, IntersectVisitor visitor) throws IOException {
  int doc = 0;
  for (int i = 0; i < count; i++) {
    doc += in.readVInt();
    visitor.visit(doc);
  }
}
 

private static void readDeltaVInts(IndexInput in, int count, int[] docIDs) throws IOException {
  int doc = 0;
  for (int i = 0; i < count; i++) {
    doc += in.readVInt();
    docIDs[i] = doc;
  }
}
 

@Override
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
  int length = skipStream.readVInt();
  if (impactData[level].length < length) {
    impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
  }
  skipStream.readBytes(impactData[level], 0, length);
  impactDataLength[level] = length;
}
 

private MonotonicBlockPackedReader(IndexInput in, int packedIntsVersion, int blockSize, long valueCount, boolean direct) throws IOException {
  this.valueCount = valueCount;
  blockShift = checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
  blockMask = blockSize - 1;
  final int numBlocks = numBlocks(valueCount, blockSize);
  minValues = new long[numBlocks];
  averages = new float[numBlocks];
  subReaders = new PackedInts.Reader[numBlocks];
  long sumBPV = 0;
  for (int i = 0; i < numBlocks; ++i) {
    minValues[i] = in.readZLong();
    averages[i] = Float.intBitsToFloat(in.readInt());
    final int bitsPerValue = in.readVInt();
    sumBPV += bitsPerValue;
    if (bitsPerValue > 64) {
      throw new IOException("Corrupted");
    }
    if (bitsPerValue == 0) {
      subReaders[i] = new PackedInts.NullReader(blockSize);
    } else {
      final int size = (int) Math.min(blockSize, valueCount - (long) i * blockSize);
      if (direct) {
        final long pointer = in.getFilePointer();
        subReaders[i] = PackedInts.getDirectReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
        in.seek(pointer + PackedInts.Format.PACKED.byteCount(packedIntsVersion, size, bitsPerValue));
      } else {
        subReaders[i] = PackedInts.getReaderNoHeader(in, PackedInts.Format.PACKED, packedIntsVersion, size, bitsPerValue);
      }
    }
  }
  this.sumBPV = sumBPV;
}
 

static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
  NumericEntry entry = new NumericEntry();
  entry.packedIntsVersion = meta.readVInt();
  entry.offset = meta.readLong();
  entry.count = meta.readVLong();
  entry.blockSize = meta.readVInt();
  return entry;
}
 

private static BytesRef readBytesRef(IndexInput in) throws IOException {
  BytesRef bytes = new BytesRef();
  bytes.length = in.readVInt();
  bytes.bytes = new byte[bytes.length];
  in.readBytes(bytes.bytes, 0, bytes.length);
  return bytes;
}
 

/**
 * @param indexInput {@link IndexInput} must be positioned to the fields metadata
 *                   details by calling {@link #seekFieldsMetadata(IndexInput)} before this call.
 * @param blockDecoder Optional block decoder, may be null if none.
 */
protected Collection<FieldMetadata> readFieldsMetadata(IndexInput indexInput, BlockDecoder blockDecoder, FieldInfos fieldInfos,
                                                              FieldMetadata.Serializer fieldMetadataReader, int maxNumDocs) throws IOException {
  int numFields = indexInput.readVInt();
  if (numFields < 0) {
    throw new CorruptIndexException("Illegal number of fields= " + numFields, indexInput);
  }
  return (blockDecoder != null && version >= VERSION_ENCODABLE_FIELDS_METADATA) ?
      readEncodedFieldsMetadata(numFields, indexInput, blockDecoder, fieldInfos, fieldMetadataReader, maxNumDocs)
      : readUnencodedFieldsMetadata(numFields, indexInput, fieldInfos, fieldMetadataReader, maxNumDocs);
}
 

static BinaryEntry readBinaryEntry(IndexInput meta) throws IOException {
  BinaryEntry entry = new BinaryEntry();
  entry.minLength = meta.readVInt();
  entry.maxLength = meta.readVInt();
  entry.count = meta.readVLong();
  entry.offset = meta.readLong();
  if (entry.minLength != entry.maxLength) {
    entry.addressesOffset = meta.readLong();
    entry.packedIntsVersion = meta.readVInt();
    entry.blockSize = meta.readVInt();
  }
  return entry;
}
 

private static BytesRef readBytesRef(IndexInput in) throws IOException {
  BytesRef bytes = new BytesRef();
  bytes.length = in.readVInt();
  bytes.bytes = new byte[bytes.length];
  in.readBytes(bytes.bytes, 0, bytes.length);
  return bytes;
}
 

public VariableGapTermsIndexReader(SegmentReadState state) throws IOException {
  String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, 
                                                   state.segmentSuffix, 
                                                   VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
  final IndexInput in = state.directory.openInput(fileName, new IOContext(state.context, true));
  boolean success = false;

  try {
    
    CodecUtil.checkIndexHeader(in, VariableGapTermsIndexWriter.CODEC_NAME,
                                     VariableGapTermsIndexWriter.VERSION_START,
                                     VariableGapTermsIndexWriter.VERSION_CURRENT,
                                     state.segmentInfo.getId(), state.segmentSuffix);
    
    CodecUtil.checksumEntireFile(in);

    seekDir(in);

    // Read directory
    final int numFields = in.readVInt();
    if (numFields < 0) {
      throw new CorruptIndexException("invalid numFields: " + numFields, in);
    }

    for(int i=0;i<numFields;i++) {
      final int field = in.readVInt();
      final long indexStart = in.readVLong();
      final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
      FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, fieldInfo, indexStart));
      if (previous != null) {
        throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
      }
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(in);
    } else {
      IOUtils.closeWhileHandlingException(in);
    }
  }
}
 

/**
 * Gets the mtas tree item.
 *
 * @param ref the ref
 * @param isSinglePoint the is single point
 * @param isStoreAdditionalIdAndRef the is store additional id and ref
 * @param nodeRefApproxOffset the node ref approx offset
 * @param in the in
 * @param objectRefApproxOffset the object ref approx offset
 * @return the mtas tree item
 * @throws IOException Signals that an I/O exception has occurred.
 */
private static MtasTreeItem getMtasTreeItem(Long ref,
    AtomicBoolean isSinglePoint, AtomicBoolean isStoreAdditionalIdAndRef,
    AtomicLong nodeRefApproxOffset, IndexInput in, long objectRefApproxOffset)
    throws IOException {
  try {
    Boolean isRoot = false;
    if (nodeRefApproxOffset.get() < 0) {
      isRoot = true;
    }
    in.seek(ref);
    if (isRoot) {
      nodeRefApproxOffset.set(in.readVLong());
      Byte flag = in.readByte();
      if ((flag
          & MtasTree.SINGLE_POSITION_TREE) == MtasTree.SINGLE_POSITION_TREE) {
        isSinglePoint.set(true);
      }
      if ((flag
          & MtasTree.STORE_ADDITIONAL_ID) == MtasTree.STORE_ADDITIONAL_ID) {
        isStoreAdditionalIdAndRef.set(true);
      }
    }
    int left = in.readVInt();
    int right = in.readVInt();
    int max = in.readVInt();
    Long leftChild = in.readVLong() + nodeRefApproxOffset.get();
    Long rightChild = in.readVLong() + nodeRefApproxOffset.get();
    int size = 1;
    if (!isSinglePoint.get()) {
      size = in.readVInt();
    }
    // initialize
    long[] objectRefs = new long[size];
    int[] objectAdditionalIds = null;
    long[] objectAdditionalRefs = null;
    // get first
    long objectRef = in.readVLong();
    long objectRefPrevious = objectRef + objectRefApproxOffset;
    objectRefs[0] = objectRefPrevious;
    if (isStoreAdditionalIdAndRef.get()) {
      objectAdditionalIds = new int[size];
      objectAdditionalRefs = new long[size];
      objectAdditionalIds[0] = in.readVInt();
      objectAdditionalRefs[0] = in.readVLong();
    }
    // get others
    for (int t = 1; t < size; t++) {
      objectRef = objectRefPrevious + in.readVLong();
      objectRefs[t] = objectRef;
      objectRefPrevious = objectRef;
      if (isStoreAdditionalIdAndRef.get()) {
        objectAdditionalIds[t] = in.readVInt();
        objectAdditionalRefs[t] = in.readVLong();
      }
    }
    return new MtasTreeItem(left, right, max, objectRefs, objectAdditionalIds,
        objectAdditionalRefs, ref, leftChild, rightChild);
  } catch (Exception e) {
    throw new IOException(e.getMessage());
  }
}