Java Code Examples for org.apache.lucene.store.DataInput#readByte()

/**
 * Creates an array with content retrieved from the given DataInput.
 * @param in       a DataInput, positioned at the start of Packed64-content.
 * @param valueCount  the number of elements.
 * @param bitsPerValue the number of bits available for any given value.
 * @throws java.io.IOException if the values for the backing array could not
 *                             be retrieved.
 */
public Packed64(int packedIntsVersion, DataInput in, int valueCount, int bitsPerValue)
                                                          throws IOException {
  super(valueCount, bitsPerValue);
  final PackedInts.Format format = PackedInts.Format.PACKED;
  final long byteCount = format.byteCount(packedIntsVersion, valueCount, bitsPerValue); // to know how much to read
  final int longCount = format.longCount(PackedInts.VERSION_CURRENT, valueCount, bitsPerValue); // to size the array
  blocks = new long[longCount];
  // read as many longs as we can
  for (int i = 0; i < byteCount / 8; ++i) {
    blocks[i] = in.readLong();
  }
  final int remaining = (int) (byteCount % 8);
  if (remaining != 0) {
    // read the last bytes
    long lastLong = 0;
    for (int i = 0; i < remaining; ++i) {
      lastLong |= (in.readByte() & 0xFFL) << (56 - i * 8);
    }
    blocks[blocks.length - 1] = lastLong;
  }
  maskRight = ~0L << (BLOCK_SIZE-bitsPerValue) >>> (BLOCK_SIZE-bitsPerValue);
  bpvMinusBlockSize = bitsPerValue - BLOCK_SIZE;
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  int bits = in.readByte() & 0xff;
  int bit0 = bits & 1;
  int bit1 = bits & 2;
  int bytesSize = (bits >>> 2);
  if (bit0 > 0 && bytesSize == 0) {  // determine extra length
    bytesSize = in.readVInt();
  }
  if (bit0 > 0) {  // bytes exists
    in.skipBytes(bytesSize);
  }
  if (bit1 > 0) {  // stats exist
    int code = in.readVInt();
    if (hasPos && (code & 1) == 0) {
      in.readVLong();
    }
  }
}
 

public static void readLine(DataInput in, BytesRefBuilder scratch) throws IOException {
  int upto = 0;
  while(true) {
    byte b = in.readByte();
    scratch.grow(1+upto);
    if (b == ESCAPE) {
      scratch.setByteAt(upto++, in.readByte());
    } else {
      if (b == NEWLINE) {
        break;
      } else {
        scratch.setByteAt(upto++, b);
      }
    }
  }
  scratch.setLength(upto);
}
 

@Override
public boolean load(DataInput input) throws IOException {
  CodecUtil.checkHeader(input, CODEC_NAME, VERSION_START, VERSION_START);
  count = input.readVLong();
  byte separatorOrig = input.readByte();
  if (separatorOrig != separator) {
    throw new IllegalStateException("separator=" + separator + " is incorrect: original model was built with separator=" + separatorOrig);
  }
  int gramsOrig = input.readVInt();
  if (gramsOrig != grams) {
    throw new IllegalStateException("grams=" + grams + " is incorrect: original model was built with grams=" + gramsOrig);
  }
  totTokens = input.readVLong();

  fst = new FST<>(input, input, PositiveIntOutputs.getSingleton());

  return true;
}
 

/**
 * Reads a double in a variable-length format.  Reads between one and
 * nine bytes. Small integral values typically take fewer bytes.
 */
static double readZDouble(DataInput in) throws IOException {
  int b = in.readByte() & 0xFF;
  if (b == 0xFF) {
    // negative value
    return Double.longBitsToDouble(in.readLong());
  } else if (b == 0xFE) {
    // float
    return Float.intBitsToFloat(in.readInt());
  } else if ((b & 0x80) != 0) {
    // small integer [-1..124]
    return (b & 0x7f) - 1;
  } else {
    // positive double
    long bits = ((long) b) << 56 | ((in.readInt() & 0xFFFFFFFFL) << 24) | ((in.readShort() & 0xFFFFL) << 8) | (in.readByte() & 0xFFL);
    return Double.longBitsToDouble(bits);
  }
}
 

private void readRecursively(DataInput in, TernaryTreeNode node) throws IOException {
  node.splitchar = in.readString().charAt(0);
  byte mask = in.readByte();
  if ((mask & HAS_TOKEN) != 0) {
    node.token = in.readString();
  }
  if ((mask & HAS_VALUE) != 0) {
    node.val = Long.valueOf(in.readLong());
  }
  if ((mask & LO_KID) != 0) {
    node.loKid = new TernaryTreeNode();
    readRecursively(in, node.loKid);
  }
  if ((mask & EQ_KID) != 0) {
    node.eqKid = new TernaryTreeNode();
    readRecursively(in, node.eqKid);
  }
  if ((mask & HI_KID) != 0) {
    node.hiKid = new TernaryTreeNode();
    readRecursively(in, node.hiKid);
  }
}
 

static long readVLong(DataInput in) throws IOException {
  byte b = in.readByte();
  if (b >= 0) return b;
  long i = b & 0x7FL;
  b = in.readByte();
  i |= (b & 0x7FL) << 7;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 14;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 21;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 28;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 35;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 42;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0x7FL) << 49;
  if (b >= 0) return i;
  b = in.readByte();
  i |= (b & 0xFFL) << 56;
  return i;
}
 

@Override
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
    maxAnalyzedPathsForOneInput = dataIn.readVInt();
    hasPayloads = dataIn.readByte() == 1;
  } finally {
    IOUtils.close(input);
  }
  return true;
}
 

/** Reads one BYTE1/2/4 label from the provided {@link DataInput}. */
public int readLabel(DataInput in) throws IOException {
  final int v;
  if (inputType == INPUT_TYPE.BYTE1) {
    // Unsigned byte:
    v = in.readByte() & 0xFF;
  } else if (inputType == INPUT_TYPE.BYTE2) {
    // Unsigned short:
    v = in.readShort() & 0xFFFF;
  } else { 
    v = in.readVInt();
  }
  return v;
}
 

/** Expert: just reads and verifies the suffix of an index header */
public static String checkIndexHeaderSuffix(DataInput in, String expectedSuffix) throws IOException {
  int suffixLength = in.readByte() & 0xFF;
  byte suffixBytes[] = new byte[suffixLength];
  in.readBytes(suffixBytes, 0, suffixBytes.length);
  String suffix = new String(suffixBytes, 0, suffixBytes.length, StandardCharsets.UTF_8);
  if (!suffix.equals(expectedSuffix)) {
    throw new CorruptIndexException("file mismatch, expected suffix=" + expectedSuffix
                                                           + ", got=" + suffix, in);
  }
  return suffix;
}
 

/**
 * Reads a long in a variable-length format.  Reads between one andCorePropLo
 * nine bytes. Small values typically take fewer bytes.
 */
static long readTLong(DataInput in) throws IOException {
  int header = in.readByte() & 0xFF;

  long bits = header & 0x1F;
  if ((header & 0x20) != 0) {
    // continuation bit
    bits |= in.readVLong() << 5;
  }

  long l = BitUtil.zigZagDecode(bits);

  switch (header & DAY_ENCODING) {
    case SECOND_ENCODING:
      l *= SECOND;
      break;
    case HOUR_ENCODING:
      l *= HOUR;
      break;
    case DAY_ENCODING:
      l *= DAY;
      break;
    case 0:
      // uncompressed
      break;
    default:
      throw new AssertionError();
  }

  return l;
}
 

/**
 * Reads a float in a variable-length format.  Reads between one and
 * five bytes. Small integral values typically take fewer bytes.
 */
static float readZFloat(DataInput in) throws IOException {
  int b = in.readByte() & 0xFF;
  if (b == 0xFF) {
    // negative value
    return Float.intBitsToFloat(in.readInt());
  } else if ((b & 0x80) != 0) {
    // small integer [-1..125]
    return (b & 0x7f) - 1;
  } else {
    // positive float
    int bits = b << 24 | ((in.readShort() & 0xFFFF) << 8) | (in.readByte() & 0xFF);
    return Float.intBitsToFloat(bits);
  }
}
 

public static byte[] decode(DataInput bytesInput, long length) throws IOException {
  length -= ENCODING_OFFSET;
  bytesInput.skipBytes(ENCODING_OFFSET);
  byte[] decodedBytes = new byte[Math.toIntExact(length)];
  for (int i = 0; i < length; i++) {
    decodedBytes[i] = (byte)(bytesInput.readByte() - ENCODING_ROTATION);
  }
  return decodedBytes;
}
 

public static byte[] encode(DataInput bytesInput, int length) throws IOException {
  byte[] encodedBytes = new byte[length + ENCODING_OFFSET];
  for (int i = 0; i < length; i++) {
    encodedBytes[i + ENCODING_OFFSET] = (byte)(bytesInput.readByte() + ENCODING_ROTATION);
  }
  return encodedBytes;
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.fst = new FST<>(input, input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  maxAnalyzedPathsForOneInput = input.readVInt();
  hasPayloads = input.readByte() == 1;
  return true;
}
 

@Override
public TermData read(DataInput in) throws IOException {
  byte[] bytes = null;
  int docFreq = 0;
  long totalTermFreq = -1;
  int bits = in.readByte() & 0xff;
  int bit0 = bits & 1;
  int bit1 = bits & 2;
  int bytesSize = (bits >>> 2);
  if (bit0 > 0 && bytesSize == 0) {  // determine extra length
    bytesSize = in.readVInt();
  }
  if (bit0 > 0) {  // bytes exists
    bytes = new byte[bytesSize];
    in.readBytes(bytes, 0, bytesSize);
  }
  if (bit1 > 0) {  // stats exist
    int code = in.readVInt();
    if (hasPos) {
      totalTermFreq = docFreq = code >>> 1;
      if ((code & 1) == 0) {
        totalTermFreq += in.readVLong();
      }
    } else {
      docFreq = code;
    }
  }
  return new TermData(bytes, docFreq, totalTermFreq);
}
 

private String readNullableString(DataInput in) throws IOException {
  byte b = in.readByte();
  if (b == 0) {
    return null;
  } else if (b == 1) {
    return in.readString();
  } else {
    throw new CorruptIndexException("invalid string lead byte " + b, in);
  }
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.fst = new FST<>(input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  maxAnalyzedPathsForOneInput = input.readVInt();
  hasPayloads = input.readByte() == 1;
  return true;
}
 

private void handleIndexing(Socket socket, AtomicBoolean stop, InputStream is, DataInput in, DataOutput out, BufferedOutputStream bos) throws IOException, InterruptedException {
  Thread.currentThread().setName("indexing");
  message("start handling indexing socket=" + socket);
  while (true) {
    while (true) {
      if (is.available() > 0) {
        break;
      }
      if (stop.get()) {
        return;
      }
      Thread.sleep(10);
    }
    byte cmd;
    try {
      cmd = in.readByte();
    } catch (EOFException eofe) {
      // done
      return;
    }
    //message("INDEXING OP " + cmd);
    if (cmd == CMD_ADD_DOC) {
      handleAddDocument(in, out);
      out.writeByte((byte) 1);
      bos.flush();
    } else if (cmd == CMD_UPDATE_DOC) {
      handleUpdateDocument(in, out);
      out.writeByte((byte) 1);
      bos.flush();
    } else if (cmd == CMD_DELETE_DOC) {
      handleDeleteDocument(in, out);
      out.writeByte((byte) 1);
      bos.flush();
    } else if (cmd == CMD_DELETE_ALL_DOCS) {
      writer.deleteAll();
      out.writeByte((byte) 1);
      bos.flush();
    } else if (cmd == CMD_FORCE_MERGE) {
      writer.forceMerge(1);
      out.writeByte((byte) 1);
      bos.flush();
    } else if (cmd == CMD_INDEXING_DONE) {
      out.writeByte((byte) 1);
      bos.flush();
      break;
    } else {
      throw new IllegalArgumentException("cmd must be add, update or delete; got " + cmd);
    }
  }
}
 

/** Called when another node (replica) wants to copy files from us */
private boolean handleFetchFiles(Random random, Socket socket, DataInput destIn, DataOutput destOut, BufferedOutputStream bos) throws IOException {
  Thread.currentThread().setName("send");

  int replicaID = destIn.readVInt();
  message("top: start fetch for R" + replicaID + " socket=" + socket);
  byte b = destIn.readByte();
  CopyState copyState;
  if (b == 0) {
    // Caller already has CopyState
    copyState = null;
  } else if (b == 1) {
    // Caller does not have CopyState; we pull the latest one:
    copyState = getCopyState();
    Thread.currentThread().setName("send-R" + replicaID + "-" + copyState.version);
  } else {
    // Protocol error:
    throw new IllegalArgumentException("invalid CopyState byte=" + b);
  }

  try {
    if (copyState != null) {
      // Serialize CopyState on the wire to the client:
      writeCopyState(copyState, destOut);
      bos.flush();
    }

    byte[] buffer = new byte[16384];
    int fileCount = 0;
    long totBytesSent = 0;
    while (true) {
      byte done = destIn.readByte();
      if (done == 1) {
        break;
      } else if (done != 0) {
        throw new IllegalArgumentException("expected 0 or 1 byte but got " + done);
      }

      // Name of the file the replica wants us to send:
      String fileName = destIn.readString();

      // Starting offset in the file we should start sending bytes from:
      long fpStart = destIn.readVLong();

      try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
        long len = in.length();
        //message("fetch " + fileName + ": send len=" + len);
        destOut.writeVLong(len);
        in.seek(fpStart);
        long upto = fpStart;
        while (upto < len) {
          int chunk = (int) Math.min(buffer.length, (len-upto));
          in.readBytes(buffer, 0, chunk);
          if (doFlipBitsDuringCopy) {
            if (random.nextInt(3000) == 17 && bitFlipped.contains(fileName) == false) {
              bitFlipped.add(fileName);
              message("file " + fileName + " to R" + replicaID + ": now randomly flipping a bit at byte=" + upto);
              int x = random.nextInt(chunk);
              int bit = random.nextInt(8);
              buffer[x] ^= 1 << bit;
            }
          }
          destOut.writeBytes(buffer, 0, chunk);
          upto += chunk;
          totBytesSent += chunk;
        }
      }

      fileCount++;
    }

    message("top: done fetch files for R" + replicaID + ": sent " + fileCount + " files; sent " + totBytesSent + " bytes");
  } catch (Throwable t) {
    message("top: exception during fetch: " + t.getMessage() + "; now close socket");
    socket.close();
    return false;
  } finally {
    if (copyState != null) {
      message("top: fetch: now release CopyState");
      releaseCopyState(copyState);
    }
  }

  return true;
}