Java Code Examples for org.apache.lucene.store.DataInput#readVLong()

static Map<String,FileMetaData> readFilesMetaData(DataInput in) throws IOException {
  int fileCount = in.readVInt();
  //System.out.println("readFilesMetaData: fileCount=" + fileCount);
  Map<String,FileMetaData> files = new HashMap<>();
  for(int i=0;i<fileCount;i++) {
    String fileName = in.readString();
    //System.out.println("readFilesMetaData: fileName=" + fileName);
    long length = in.readVLong();
    long checksum = in.readVLong();
    byte[] header = new byte[in.readVInt()];
    in.readBytes(header, 0, header.length);
    byte[] footer = new byte[in.readVInt()];
    in.readBytes(footer, 0, footer.length);
    files.put(fileName, new FileMetaData(header, footer, length, checksum));
  }
  return files;
}
 

@Override
public boolean load(DataInput input) throws IOException {
  CodecUtil.checkHeader(input, CODEC_NAME, VERSION_START, VERSION_START);
  count = input.readVLong();
  byte separatorOrig = input.readByte();
  if (separatorOrig != separator) {
    throw new IllegalStateException("separator=" + separator + " is incorrect: original model was built with separator=" + separatorOrig);
  }
  int gramsOrig = input.readVInt();
  if (gramsOrig != grams) {
    throw new IllegalStateException("grams=" + grams + " is incorrect: original model was built with grams=" + gramsOrig);
  }
  totTokens = input.readVLong();

  fst = new FST<>(input, input, PositiveIntOutputs.getSingleton());

  return true;
}
 

public BlockHeader read(DataInput input, BlockHeader reuse) throws IOException {
  int linesCount = input.readVInt();
  if (linesCount <= 0 || linesCount > UniformSplitTermsWriter.MAX_NUM_BLOCK_LINES) {
    throw new CorruptIndexException("Illegal number of lines in block: " + linesCount, input);
  }

  long baseDocsFP = input.readVLong();
  long basePositionsFP = input.readVLong();
  long basePayloadsFP = input.readVLong();

  int termStatesBaseOffset = input.readVInt();
  if (termStatesBaseOffset < 0) {
    throw new CorruptIndexException("Illegal termStatesBaseOffset= " + termStatesBaseOffset, input);
  }
  int middleTermOffset = input.readVInt();
  if (middleTermOffset < 0) {
    throw new CorruptIndexException("Illegal middleTermOffset= " + middleTermOffset, input);
  }

  BlockHeader blockHeader = reuse == null ? new BlockHeader() : reuse;
  return blockHeader.reset(linesCount, baseDocsFP, basePositionsFP, basePayloadsFP, termStatesBaseOffset, middleTermOffset);
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  int bits = in.readByte() & 0xff;
  int bit0 = bits & 1;
  int bit1 = bits & 2;
  int bytesSize = (bits >>> 2);
  if (bit0 > 0 && bytesSize == 0) {  // determine extra length
    bytesSize = in.readVInt();
  }
  if (bit0 > 0) {  // bytes exists
    in.skipBytes(bytesSize);
  }
  if (bit1 > 0) {  // stats exist
    int code = in.readVInt();
    if (hasPos && (code & 1) == 0) {
      in.readVLong();
    }
  }
}
 

/** Pulls CopyState off the wire */
static CopyState readCopyState(DataInput in) throws IOException {

  // Decode a new CopyState
  byte[] infosBytes = new byte[in.readVInt()];
  in.readBytes(infosBytes, 0, infosBytes.length);

  long gen = in.readVLong();
  long version = in.readVLong();
  Map<String,FileMetaData> files = readFilesMetaData(in);

  int count = in.readVInt();
  Set<String> completedMergeFiles = new HashSet<>();
  for(int i=0;i<count;i++) {
    completedMergeFiles.add(in.readString());
  }
  long primaryGen = in.readVLong();

  return new CopyState(files, version, gen, infosBytes, completedMergeFiles, primaryGen, null);
}
 

@Override
public TermData read(DataInput in) throws IOException {
  byte[] bytes = null;
  int docFreq = 0;
  long totalTermFreq = -1;
  int bits = in.readByte() & 0xff;
  int bit0 = bits & 1;
  int bit1 = bits & 2;
  int bytesSize = (bits >>> 2);
  if (bit0 > 0 && bytesSize == 0) {  // determine extra length
    bytesSize = in.readVInt();
  }
  if (bit0 > 0) {  // bytes exists
    bytes = new byte[bytesSize];
    in.readBytes(bytes, 0, bytesSize);
  }
  if (bit1 > 0) {  // stats exist
    int code = in.readVInt();
    if (hasPos) {
      totalTermFreq = docFreq = code >>> 1;
      if ((code & 1) == 0) {
        totalTermFreq += in.readVLong();
      }
    } else {
      docFreq = code;
    }
  }
  return new TermData(bytes, docFreq, totalTermFreq);
}
 

@Override
public void skipOutput(DataInput in) throws IOException {
  int len = in.readVInt();
  in.skipBytes(len);
  in.readVLong();
  in.readVLong();
}
 

protected Collection<FieldMetadata> readEncodedFieldsMetadata(int numFields, DataInput metadataInput, BlockDecoder blockDecoder,
                                                              FieldInfos fieldInfos, FieldMetadata.Serializer fieldMetadataReader,
                                                              int maxNumDocs) throws IOException {
  long encodedLength = metadataInput.readVLong();
  if (encodedLength < 0) {
    throw new CorruptIndexException("Illegal encoded length: " + encodedLength, metadataInput);
  }
  BytesRef decodedBytes = blockDecoder.decode(metadataInput, encodedLength);
  DataInput decodedMetadataInput = new ByteArrayDataInput(decodedBytes.bytes, 0, decodedBytes.length);
  return readUnencodedFieldsMetadata(numFields, decodedMetadataInput, fieldInfos, fieldMetadataReader, maxNumDocs);
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.fst = new FST<>(input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  maxAnalyzedPathsForOneInput = input.readVInt();
  hasPayloads = input.readByte() == 1;
  return true;
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.fst = new FST<>(input, input, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
  maxAnalyzedPathsForOneInput = input.readVInt();
  hasPayloads = input.readByte() == 1;
  return true;
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  TSTNode root = new TSTNode('\0', null);
  readRecursively(input, root);
  trie.setRoot(root);
  return true;
}
 

@Override
public synchronized boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.higherWeightsCompletion = new FSTCompletion(new FST<>(
      input, input, NoOutputs.getSingleton()));
  this.normalCompletion = new FSTCompletion(
      higherWeightsCompletion.getFST(), false, exactMatchFirst);
  return true;
}
 

@Override
public synchronized boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  root = new TernaryTreeNode();
  readRecursively(input, root);
  return true;
}
 

@Override
public Long read(DataInput in) throws IOException {
  long v = in.readVLong();
  if (v == 0) {
    return NO_OUTPUT;
  } else {
    return v;
  }
}
 

@Override
public void decodeTerm(DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
  throws IOException {
  final IDVersionTermState termState = (IDVersionTermState) _termState;
  termState.docID = in.readVInt();
  if (absolute) {
    termState.idVersion = in.readVLong();
  } else {
    termState.idVersion += in.readZLong();
  }
}
 

@Override
public void decodeTerm(DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
  throws IOException {
  final IntBlockTermState termState = (IntBlockTermState) _termState;
  final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
  final boolean fieldHasPayloads = fieldInfo.hasPayloads();

  if (absolute) {
    termState.docStartFP = 0;
    termState.posStartFP = 0;
    termState.payStartFP = 0;
  }

  termState.docStartFP += in.readVLong();
  if (fieldHasPositions) {
    termState.posStartFP += in.readVLong();
    if (fieldHasOffsets || fieldHasPayloads) {
      termState.payStartFP += in.readVLong();
    }
  }
  if (termState.docFreq == 1) {
    termState.singletonDocID = in.readVInt();
  } else {
    termState.singletonDocID = -1;
  }
  if (fieldHasPositions) {
    if (termState.totalTermFreq > BLOCK_SIZE) {
      termState.lastPosBlockOffset = in.readVLong();
    } else {
      termState.lastPosBlockOffset = -1;
    }
  }
  if (termState.docFreq > BLOCK_SIZE) {
    termState.skipOffset = in.readVLong();
  } else {
    termState.skipOffset = -1;
  }
}
 

/**
 * Skip 128 integers.
 */
void skip(DataInput in) throws IOException {
  final int token = Byte.toUnsignedInt(in.readByte());
  final int bitsPerValue = token & 0x1f;
  final int numExceptions = token >>> 5;
  if (bitsPerValue == 0) {
    in.readVLong();
    in.skipBytes((numExceptions << 1));
  } else {
    in.skipBytes(forUtil.numBytes(bitsPerValue) + (numExceptions << 1));
  }
}
 

/** Called when another node (replica) wants to copy files from us */
private boolean handleFetchFiles(Random random, Socket socket, DataInput destIn, DataOutput destOut, BufferedOutputStream bos) throws IOException {
  Thread.currentThread().setName("send");

  int replicaID = destIn.readVInt();
  message("top: start fetch for R" + replicaID + " socket=" + socket);
  byte b = destIn.readByte();
  CopyState copyState;
  if (b == 0) {
    // Caller already has CopyState
    copyState = null;
  } else if (b == 1) {
    // Caller does not have CopyState; we pull the latest one:
    copyState = getCopyState();
    Thread.currentThread().setName("send-R" + replicaID + "-" + copyState.version);
  } else {
    // Protocol error:
    throw new IllegalArgumentException("invalid CopyState byte=" + b);
  }

  try {
    if (copyState != null) {
      // Serialize CopyState on the wire to the client:
      writeCopyState(copyState, destOut);
      bos.flush();
    }

    byte[] buffer = new byte[16384];
    int fileCount = 0;
    long totBytesSent = 0;
    while (true) {
      byte done = destIn.readByte();
      if (done == 1) {
        break;
      } else if (done != 0) {
        throw new IllegalArgumentException("expected 0 or 1 byte but got " + done);
      }

      // Name of the file the replica wants us to send:
      String fileName = destIn.readString();

      // Starting offset in the file we should start sending bytes from:
      long fpStart = destIn.readVLong();

      try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
        long len = in.length();
        //message("fetch " + fileName + ": send len=" + len);
        destOut.writeVLong(len);
        in.seek(fpStart);
        long upto = fpStart;
        while (upto < len) {
          int chunk = (int) Math.min(buffer.length, (len-upto));
          in.readBytes(buffer, 0, chunk);
          if (doFlipBitsDuringCopy) {
            if (random.nextInt(3000) == 17 && bitFlipped.contains(fileName) == false) {
              bitFlipped.add(fileName);
              message("file " + fileName + " to R" + replicaID + ": now randomly flipping a bit at byte=" + upto);
              int x = random.nextInt(chunk);
              int bit = random.nextInt(8);
              buffer[x] ^= 1 << bit;
            }
          }
          destOut.writeBytes(buffer, 0, chunk);
          upto += chunk;
          totBytesSent += chunk;
        }
      }

      fileCount++;
    }

    message("top: done fetch files for R" + replicaID + ": sent " + fileCount + " files; sent " + totBytesSent + " bytes");
  } catch (Throwable t) {
    message("top: exception during fetch: " + t.getMessage() + "; now close socket");
    socket.close();
    return false;
  } finally {
    if (copyState != null) {
      message("top: fetch: now release CopyState");
      releaseCopyState(copyState);
    }
  }

  return true;
}
 

public FieldMetadata read(DataInput input, FieldInfos fieldInfos, int maxNumDocs) throws IOException {
  int fieldId = input.readVInt();
  FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldId);
  if (fieldInfo == null) {
    throw new CorruptIndexException("Illegal field id= " + fieldId, input);
  }
  FieldMetadata fieldMetadata = new FieldMetadata(fieldInfo, 0, false);

  fieldMetadata.numTerms = input.readVLong();
  if (fieldMetadata.numTerms <= 0) {
    throw new CorruptIndexException("Illegal number of terms= " + fieldMetadata.numTerms + " for field= " + fieldId, input);
  }

  fieldMetadata.sumDocFreq = input.readVLong();
  fieldMetadata.sumTotalTermFreq = fieldMetadata.sumDocFreq;
  if (fieldMetadata.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0) {
    fieldMetadata.sumTotalTermFreq += input.readVLong();
    if (fieldMetadata.sumTotalTermFreq < fieldMetadata.sumDocFreq) {
      // #positions must be >= #postings.
      throw new CorruptIndexException("Illegal sumTotalTermFreq= " + fieldMetadata.sumTotalTermFreq
          + " sumDocFreq= " + fieldMetadata.sumDocFreq + " for field= " + fieldId, input);
    }
  }

  fieldMetadata.docCount = input.readVInt();
  if (fieldMetadata.docCount < 0 || fieldMetadata.docCount > maxNumDocs) {
    // #docs with field must be <= #docs.
    throw new CorruptIndexException("Illegal number of docs= " + fieldMetadata.docCount
        + " maxNumDocs= " + maxNumDocs + " for field=" + fieldId, input);
  }
  if (fieldMetadata.sumDocFreq < fieldMetadata.docCount) {
    // #postings must be >= #docs with field.
    throw new CorruptIndexException("Illegal sumDocFreq= " + fieldMetadata.sumDocFreq
        + " docCount= " + fieldMetadata.docCount + " for field= " + fieldId, input);
  }

  fieldMetadata.dictionaryStartFP = input.readVLong();
  fieldMetadata.firstBlockStartFP = input.readVLong();
  fieldMetadata.lastBlockStartFP = input.readVLong();

  int lastTermLength = input.readVInt();
  BytesRef lastTerm = new BytesRef(lastTermLength);
  if (lastTermLength > 0) {
    input.readBytes(lastTerm.bytes, 0, lastTermLength);
    lastTerm.length = lastTermLength;
  } else if (lastTermLength < 0) {
    throw new CorruptIndexException("Illegal last term length= " + lastTermLength + " for field= " + fieldId, input);
  }
  fieldMetadata.setLastTerm(lastTerm);

  return fieldMetadata;
}
 

@Override
public boolean load(DataInput input) throws IOException {
  count = input.readVLong();
  this.fst = new FST<>(input, input, PositiveIntOutputs.getSingleton());
  return true;
}