org.apache.lucene.store.DataInput Java Examples

private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 22);
  shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);
  for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {
    long l0 = (tmp[tmpIdx+0] & MASK16_5) << 6;
    l0 |= (tmp[tmpIdx+1] & MASK16_5) << 1;
    l0 |= (tmp[tmpIdx+2] >>> 4) & MASK16_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+2] & MASK16_4) << 7;
    l1 |= (tmp[tmpIdx+3] & MASK16_5) << 2;
    l1 |= (tmp[tmpIdx+4] >>> 3) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+4] & MASK16_3) << 8;
    l2 |= (tmp[tmpIdx+5] & MASK16_5) << 3;
    l2 |= (tmp[tmpIdx+6] >>> 2) & MASK16_3;
    longs[longsIdx+2] = l2;
    long l3 = (tmp[tmpIdx+6] & MASK16_2) << 9;
    l3 |= (tmp[tmpIdx+7] & MASK16_5) << 4;
    l3 |= (tmp[tmpIdx+8] >>> 1) & MASK16_4;
    longs[longsIdx+3] = l3;
    long l4 = (tmp[tmpIdx+8] & MASK16_1) << 10;
    l4 |= (tmp[tmpIdx+9] & MASK16_5) << 5;
    l4 |= (tmp[tmpIdx+10] & MASK16_5) << 0;
    longs[longsIdx+4] = l4;
  }
}
 

public static FuzzySet deserialize(DataInput in) throws IOException
{
  int version=in.readInt();
  if (version == VERSION_SPI) {
    in.readString();
  }
  final HashFunction hashFunction = hashFunctionForVersion(version);
  int bloomSize=in.readInt();
  int numLongs=in.readInt();
  long[]longs=new long[numLongs];
  for (int i = 0; i < numLongs; i++) {
    longs[i]=in.readLong();
  }
  FixedBitSet bits = new FixedBitSet(longs,bloomSize+1);
  return new FuzzySet(bits,bloomSize,hashFunction);
}
 

private static void countFSTArcs(String fstFilePath) throws IOException {
  byte[] buf = Files.readAllBytes(Paths.get(fstFilePath));
  DataInput in = new ByteArrayDataInput(buf);
  FST<BytesRef> fst = new FST<>(in, in, ByteSequenceOutputs.getSingleton());
  BytesRefFSTEnum<BytesRef> fstEnum = new BytesRefFSTEnum<>(fst);
  int binarySearchArcCount = 0, directAddressingArcCount = 0, listArcCount = 0;
  while(fstEnum.next() != null) {
    if (fstEnum.arcs[fstEnum.upto].bytesPerArc() == 0) {
      listArcCount ++;
    } else if (fstEnum.arcs[fstEnum.upto].nodeFlags() == FST.ARCS_FOR_DIRECT_ADDRESSING) {
      directAddressingArcCount ++;
    } else {
      binarySearchArcCount ++;
    }
  }
  System.out.println("direct addressing arcs = " + directAddressingArcCount
      + ", binary search arcs = " + binarySearchArcCount
      + " list arcs = " + listArcCount);
}
 

/**
 * Reads a double in a variable-length format.  Reads between one and
 * nine bytes. Small integral values typically take fewer bytes.
 */
static double readZDouble(DataInput in) throws IOException {
  int b = in.readByte() & 0xFF;
  if (b == 0xFF) {
    // negative value
    return Double.longBitsToDouble(in.readLong());
  } else if (b == 0xFE) {
    // float
    return Float.intBitsToFloat(in.readInt());
  } else if ((b & 0x80) != 0) {
    // small integer [-1..124]
    return (b & 0x7f) - 1;
  } else {
    // positive double
    long bits = ((long) b) << 56 | ((in.readInt() & 0xFFFFFFFFL) << 24) | ((in.readShort() & 0xFFFFL) << 8) | (in.readByte() & 0xFFL);
    return Double.longBitsToDouble(bits);
  }
}
 

/**
 * Creates an array with content retrieved from the given DataInput.
 * @param in       a DataInput, positioned at the start of Packed64-content.
 * @param valueCount  the number of elements.
 * @param bitsPerValue the number of bits available for any given value.
 * @throws java.io.IOException if the values for the backing array could not
 *                             be retrieved.
 */
public Packed64(int packedIntsVersion, DataInput in, int valueCount, int bitsPerValue)
                                                          throws IOException {
  super(valueCount, bitsPerValue);
  final PackedInts.Format format = PackedInts.Format.PACKED;
  final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read
  final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array
  blocks = new long[longCount];
  // read as many longs as we can
  for (int i = 0; i < byteCount / 8; ++i) {
    blocks[i] = in.readLong();
  }
  final int remaining = (int) (byteCount % 8);
  if (remaining != 0) {
    // read the last bytes
    long lastLong = 0;
    for (int i = 0; i < remaining; ++i) {
      lastLong |= (in.readByte() & 0xFFL) << (56 - i * 8);
    }
    blocks[blocks.length - 1] = lastLong;
  }
  maskRight = ~0L << (BLOCK_SIZE-bitsPerValue) >>> (BLOCK_SIZE-bitsPerValue);
  bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE;
}
 

private void handleAddDocument(DataInput in, DataOutput out) throws IOException {
  int fieldCount = in.readVInt();
  Document doc = new Document();
  for(int i=0;i<fieldCount;i++) {
    String name = in.readString();
    String value = in.readString();
    // NOTE: clearly NOT general!
    if (name.equals("docid") || name.equals("marker")) {
      doc.add(new StringField(name, value, Field.Store.YES));
    } else if (name.equals("title")) {
      doc.add(new StringField("title", value, Field.Store.YES));
      doc.add(new Field("titleTokenized", value, tokenizedWithTermVectors));
    } else if (name.equals("body")) {
      doc.add(new Field("body", value, tokenizedWithTermVectors));
    } else {
      throw new IllegalArgumentException("unhandled field name " + name);
    }
  }
  writer.addDocument(doc);
}
 

/** Pulls bytes from the provided IndexInput.  */
public BytesStore(DataInput in, long numBytes, int maxBlockSize) throws IOException {
  int blockSize = 2;
  int blockBits = 1;
  while(blockSize < numBytes && blockSize < maxBlockSize) {
    blockSize *= 2;
    blockBits++;
  }
  this.blockBits = blockBits;
  this.blockSize = blockSize;
  this.blockMask = blockSize-1;
  long left = numBytes;
  while(left > 0) {
    final int chunk = (int) Math.min(blockSize, left);
    byte[] block = new byte[chunk];
    in.readBytes(block, 0, block.length);
    blocks.add(block);
    left -= chunk;
  }

  // So .getPosition still works
  nextWrite = blocks.get(blocks.size()-1).length;
}
 

@Override
public Object read(DataInput in) throws IOException {
  final long code = in.readVLong();
  if ((code & 1) == 0) {
    // single long
    final long v = code >>> 1;
    if (v == 0) {
      return NO_OUTPUT;
    } else {
      return Long.valueOf(v);
    }
  } else {
    // two longs
    final long first = code >>> 1;
    final long second = in.readVLong();
    return new TwoLongs(first, second);
  }
}
 

private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 40);
  shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);
  for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK32_12) << 8;
    l0 |= (tmp[tmpIdx+1] >>> 4) & MASK32_8;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK32_4) << 16;
    l1 |= (tmp[tmpIdx+2] & MASK32_12) << 4;
    l1 |= (tmp[tmpIdx+3] >>> 8) & MASK32_4;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK32_8) << 12;
    l2 |= (tmp[tmpIdx+4] & MASK32_12) << 0;
    longs[longsIdx+2] = l2;
  }
}
 

private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 10);
  shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);
  for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK8_3) << 2;
    l0 |= (tmp[tmpIdx+1] >>> 1) & MASK8_2;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK8_1) << 4;
    l1 |= (tmp[tmpIdx+2] & MASK8_3) << 1;
    l1 |= (tmp[tmpIdx+3] >>> 2) & MASK8_1;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK8_2) << 3;
    l2 |= (tmp[tmpIdx+4] & MASK8_3) << 0;
    longs[longsIdx+2] = l2;
  }
}
 

public BlockHeader read(DataInput input, BlockHeader reuse) throws IOException {
  int linesCount = input.readVInt();
  if (linesCount <= 0 || linesCount > UniformSplitTermsWriter.MAX_NUM_BLOCK_LINES) {
    throw new CorruptIndexException("Illegal number of lines in block: " + linesCount, input);
  }

  long baseDocsFP = input.readVLong();
  long basePositionsFP = input.readVLong();
  long basePayloadsFP = input.readVLong();

  int termStatesBaseOffset = input.readVInt();
  if (termStatesBaseOffset < 0) {
    throw new CorruptIndexException("Illegal termStatesBaseOffset= " + termStatesBaseOffset, input);
  }
  int middleTermOffset = input.readVInt();
  if (middleTermOffset < 0) {
    throw new CorruptIndexException("Illegal middleTermOffset= " + middleTermOffset, input);
  }

  BlockHeader blockHeader = reuse == null ? new BlockHeader() : reuse;
  return blockHeader.reset(linesCount, baseDocsFP, basePositionsFP, basePayloadsFP, termStatesBaseOffset, middleTermOffset);
}
 

private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 20);
  shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);
  for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK16_6) << 4;
    l0 |= (tmp[tmpIdx+1] >>> 2) & MASK16_4;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK16_2) << 8;
    l1 |= (tmp[tmpIdx+2] & MASK16_6) << 2;
    l1 |= (tmp[tmpIdx+3] >>> 4) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+3] & MASK16_4) << 6;
    l2 |= (tmp[tmpIdx+4] & MASK16_6) << 0;
    longs[longsIdx+2] = l2;
  }
}
 

private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 26);
  shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);
  for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {
    long l0 = (tmp[tmpIdx+0] & MASK16_3) << 10;
    l0 |= (tmp[tmpIdx+1] & MASK16_3) << 7;
    l0 |= (tmp[tmpIdx+2] & MASK16_3) << 4;
    l0 |= (tmp[tmpIdx+3] & MASK16_3) << 1;
    l0 |= (tmp[tmpIdx+4] >>> 2) & MASK16_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+4] & MASK16_2) << 11;
    l1 |= (tmp[tmpIdx+5] & MASK16_3) << 8;
    l1 |= (tmp[tmpIdx+6] & MASK16_3) << 5;
    l1 |= (tmp[tmpIdx+7] & MASK16_3) << 2;
    l1 |= (tmp[tmpIdx+8] >>> 1) & MASK16_2;
    longs[longsIdx+1] = l1;
    long l2 = (tmp[tmpIdx+8] & MASK16_1) << 12;
    l2 |= (tmp[tmpIdx+9] & MASK16_3) << 9;
    l2 |= (tmp[tmpIdx+10] & MASK16_3) << 6;
    l2 |= (tmp[tmpIdx+11] & MASK16_3) << 3;
    l2 |= (tmp[tmpIdx+12] & MASK16_3) << 0;
    longs[longsIdx+2] = l2;
  }
}
 

private void replayAddDocument(Connection c, NodeProcess primary, DataInput in) throws IOException {
  String id = in.readString();

  Document doc = new Document();
  doc.add(new StringField("docid", id, Field.Store.YES));

  String title = readNullableString(in);
  if (title != null) {
    doc.add(new StringField("title", title, Field.Store.NO));
    doc.add(new TextField("titleTokenized", title, Field.Store.NO));
  }
  String body = readNullableString(in);
  if (body != null) {
    doc.add(new TextField("body", body, Field.Store.NO));
  }
  String marker = readNullableString(in);
  if (marker != null) {
    //TestStressNRTReplication.message("xlog: replay marker=" + id);
    doc.add(new StringField("marker", marker, Field.Store.YES));
  }

  // For both add and update originally, we use updateDocument to replay,
  // because the doc could in fact already be in the index:
  // nocomit what if this fails?
  primary.addOrUpdateDocument(c, doc, false);
}
 

/**
 * Restore a {@link ForUtil} from a {@link DataInput}.
 */
ForUtil(DataInput in) throws IOException {
  int packedIntsVersion = in.readVInt();
  PackedInts.checkVersion(packedIntsVersion);
  encodedSizes = new int[33];
  encoders = new PackedInts.Encoder[33];
  decoders = new PackedInts.Decoder[33];
  iterations = new int[33];

  for (int bpv = 1; bpv <= 32; ++bpv) {
    final int code = in.readVInt();
    final int formatId = code >>> 5;
    final int bitsPerValue = (code & 31) + 1;

    final PackedInts.Format format = PackedInts.Format.byId(formatId);
    assert format.isSupported(bitsPerValue);
    encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue);
    encoders[bpv] = PackedInts.getEncoder(
        format, packedIntsVersion, bitsPerValue);
    decoders[bpv] = PackedInts.getDecoder(
        format, packedIntsVersion, bitsPerValue);
    iterations[bpv] = computeIterations(decoders[bpv]);
  }
}
 

/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  this.fst = new TokenInfoFST(fst);
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  final int len = in.readVInt();
  for(int idx=0;idx<len;idx++) {
    in.readVInt();
  }
}
 

private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 2);
  shiftLongs(tmp, 2, longs, 0, 7, MASK8_1);
  shiftLongs(tmp, 2, longs, 2, 6, MASK8_1);
  shiftLongs(tmp, 2, longs, 4, 5, MASK8_1);
  shiftLongs(tmp, 2, longs, 6, 4, MASK8_1);
  shiftLongs(tmp, 2, longs, 8, 3, MASK8_1);
  shiftLongs(tmp, 2, longs, 10, 2, MASK8_1);
  shiftLongs(tmp, 2, longs, 12, 1, MASK8_1);
  shiftLongs(tmp, 2, longs, 14, 0, MASK8_1);
}
 

private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 12);
  shiftLongs(tmp, 12, longs, 0, 2, MASK8_6);
  for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) {
    long l0 = (tmp[tmpIdx+0] & MASK8_2) << 4;
    l0 |= (tmp[tmpIdx+1] & MASK8_2) << 2;
    l0 |= (tmp[tmpIdx+2] & MASK8_2) << 0;
    longs[longsIdx+0] = l0;
  }
}
 

private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 6);
  shiftLongs(tmp, 6, longs, 0, 5, MASK8_3);
  shiftLongs(tmp, 6, longs, 6, 2, MASK8_3);
  for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) {
    long l0 = (tmp[tmpIdx+0] & MASK8_2) << 1;
    l0 |= (tmp[tmpIdx+1] >>> 1) & MASK8_1;
    longs[longsIdx+0] = l0;
    long l1 = (tmp[tmpIdx+1] & MASK8_1) << 2;
    l1 |= (tmp[tmpIdx+2] & MASK8_2) << 0;
    longs[longsIdx+1] = l1;
  }
}
 

private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 4);
  shiftLongs(tmp, 4, longs, 0, 6, MASK8_2);
  shiftLongs(tmp, 4, longs, 4, 4, MASK8_2);
  shiftLongs(tmp, 4, longs, 8, 2, MASK8_2);
  shiftLongs(tmp, 4, longs, 12, 0, MASK8_2);
}
 

private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 14);
  shiftLongs(tmp, 14, longs, 0, 1, MASK8_7);
  for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) {
    long l0 = (tmp[tmpIdx+0] & MASK8_1) << 6;
    l0 |= (tmp[tmpIdx+1] & MASK8_1) << 5;
    l0 |= (tmp[tmpIdx+2] & MASK8_1) << 4;
    l0 |= (tmp[tmpIdx+3] & MASK8_1) << 3;
    l0 |= (tmp[tmpIdx+4] & MASK8_1) << 2;
    l0 |= (tmp[tmpIdx+5] & MASK8_1) << 1;
    l0 |= (tmp[tmpIdx+6] & MASK8_1) << 0;
    longs[longsIdx+0] = l0;
  }
}
 

protected static Type readType(DataInput in) throws IOException {
  String type = in.readString();
  try {
    return Type.valueOf(type);
  }
  catch (IllegalArgumentException e) {
    throw new IllegalArgumentException("Can't deserialize SortField - unknown type " + type);
  }
}
 

public static byte[] encode(DataInput bytesInput, int length) throws IOException {
  byte[] encodedBytes = new byte[length + ENCODING_OFFSET];
  for (int i = 0; i < length; i++) {
    encodedBytes[i + ENCODING_OFFSET] = (byte)(bytesInput.readByte() + ENCODING_ROTATION);
  }
  return encodedBytes;
}
 

@Override
public IntsRef read(DataInput in) throws IOException {
  final int len = in.readVInt();
  if (len == 0) {
    return NO_OUTPUT;
  } else {
    final IntsRef output = new IntsRef(len);
    for(int idx=0;idx<len;idx++) {
      output.ints[idx] = in.readVInt();
    }
    output.length = len;
    return output;
  }
}
 

@Override
public void decompress(final DataInput in, final int originalLength, final int offset, final int length,
    final BytesRef bytes) throws IOException {
  if (in instanceof IndexInput) {
    IndexInput indexInput = (IndexInput) in;
    String name = indexInput.toString();
    long filePointer = indexInput.getFilePointer();

    Entry entry = _entry.get();
    if (!entry.isValid(indexInput, name, filePointer)) {
      entry.setup(indexInput, name, filePointer);
      entry._cache.grow(originalLength + 7);
      _decompressor.decompress(indexInput, originalLength, 0, originalLength, entry._cache);
      entry._cache.length = originalLength;
      entry._cache.offset = 0;
      _entry.set(entry);
    }
    if (bytes.bytes.length < originalLength + 7) {
      bytes.bytes = new byte[ArrayUtil.oversize(originalLength + 7, 1)];
    }
    System.arraycopy(entry._cache.bytes, entry._cache.offset, bytes.bytes, 0, length + offset);
    bytes.offset = offset;
    bytes.length = length;
  } else {
    _decompressor.decompress(in, originalLength, offset, length, bytes);
  }
}
 

/**
 * Retrieve PackedInts as a {@link ReaderIterator}
 * @param in positioned at the beginning of a stored packed int structure.
 * @param mem how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration)
 * @return an iterator to access the values
 * @throws IOException if the structure could not be retrieved.
 * @lucene.internal
 */
public static ReaderIterator getReaderIterator(DataInput in, int mem) throws IOException {
  final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
  final int bitsPerValue = in.readVInt();
  assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
  final int valueCount = in.readVInt();
  final Format format = Format.byId(in.readVInt());
  return getReaderIteratorNoHeader(in, format, version, valueCount, bitsPerValue, mem);
}
 

/**
 * Decode deltas, compute the prefix sum and add {@code base} to all decoded longs.
 */
void decodeAndPrefixSum(DataInput in, long base, long[] longs) throws IOException {
  final int bitsPerValue = Byte.toUnsignedInt(in.readByte());
  if (bitsPerValue == 0) {
    prefixSumOfOnes(longs, base);
  } else {
    forUtil.decodeAndPrefixSum(bitsPerValue, in, base, longs);
  }
}
 

@Override
public synchronized boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  root = new TernaryTreeNode();
  readRecursively(input, root);
  return true;
}
 

private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException {
  in.readLELongs(tmp, 0, 48);
  shiftLongs(tmp, 48, longs, 0, 8, MASK32_24);
  for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) {
    long l0 = (tmp[tmpIdx+0] & MASK32_8) << 16;
    l0 |= (tmp[tmpIdx+1] & MASK32_8) << 8;
    l0 |= (tmp[tmpIdx+2] & MASK32_8) << 0;
    longs[longsIdx+0] = l0;
  }
}