Java Code Examples for org.apache.lucene.store.DataOutput#writeVLong()

@Override
public void encodeTerm(DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
  IntBlockTermState state = (IntBlockTermState)_state;
  if (absolute) {
    lastState = emptyState;
  }
  out.writeVLong(state.docStartFP - lastState.docStartFP);
  if (writePositions) {
    out.writeVLong(state.posStartFP - lastState.posStartFP);
    if (writePayloads || writeOffsets) {
      out.writeVLong(state.payStartFP - lastState.payStartFP);
    }
  }
  if (state.singletonDocID != -1) {
    out.writeVInt(state.singletonDocID);
  }
  if (writePositions) {
    if (state.lastPosBlockOffset != -1) {
      out.writeVLong(state.lastPosBlockOffset);
    }
  }
  if (state.skipOffset != -1) {
    out.writeVLong(state.skipOffset);
  }
  lastState = state;
}
 

@Override
public boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  if (fst == null) {
    return false;
  }

  fst.save(output);
  output.writeVInt(maxAnalyzedPathsForOneInput);
  output.writeByte((byte) (hasPayloads ? 1 : 0));
  return true;
}
 

static void writeFilesMetaData(DataOutput out, Map<String,FileMetaData> files) throws IOException {
  out.writeVInt(files.size());
  for(Map.Entry<String,FileMetaData> ent : files.entrySet()) {
    out.writeString(ent.getKey());

    FileMetaData fmd = ent.getValue();
    out.writeVLong(fmd.length);
    out.writeVLong(fmd.checksum);
    out.writeVInt(fmd.header.length);
    out.writeBytes(fmd.header, 0, fmd.header.length);
    out.writeVInt(fmd.footer.length);
    out.writeBytes(fmd.footer, 0, fmd.footer.length);
  }
}
 

/** Pushes CopyState on the wire */
private static void writeCopyState(CopyState state, DataOutput out) throws IOException {
  // TODO (opto): we could encode to byte[] once when we created the copyState, and then just send same byts to all replicas...
  out.writeVInt(state.infosBytes.length);
  out.writeBytes(state.infosBytes, 0, state.infosBytes.length);
  out.writeVLong(state.gen);
  out.writeVLong(state.version);
  SimpleServer.writeFilesMetaData(out, state.files);

  out.writeVInt(state.completedMergeFiles.size());
  for(String fileName : state.completedMergeFiles) {
    out.writeString(fileName);
  }
  out.writeVLong(state.primaryGen);
}
 

private void verifyAtLeastMarkerCount(int expectedAtLeastCount, DataOutput out) throws IOException {
  IndexSearcher searcher = mgr.acquire();
  try {
    long version = ((DirectoryReader) searcher.getIndexReader()).getVersion();
    int hitCount = searcher.count(new TermQuery(new Term("marker", "marker")));

    if (hitCount < expectedAtLeastCount) {
      message("marker search: expectedAtLeastCount=" + expectedAtLeastCount + " but hitCount=" + hitCount);
      TopDocs hits = searcher.search(new TermQuery(new Term("marker", "marker")), expectedAtLeastCount);
      List<Integer> seen = new ArrayList<>();
      for(ScoreDoc hit : hits.scoreDocs) {
        Document doc = searcher.doc(hit.doc);
        seen.add(Integer.parseInt(doc.get("docid").substring(1)));
      }
      Collections.sort(seen);
      message("saw markers:");
      for(int marker : seen) {
        message("saw m" + marker);
      }
      throw new IllegalStateException("at flush: marker count " + hitCount + " but expected at least " + expectedAtLeastCount + " version=" + version);
    }

    if (out != null) {
      out.writeVLong(version);
      out.writeVInt(hitCount);
    }
  } finally {
    mgr.release(searcher);
  }
}
 

@Override
public void write(Output prefix, DataOutput out) throws IOException {
  out.writeVInt(prefix.bytes.length);
  out.writeBytes(prefix.bytes.bytes, prefix.bytes.offset, prefix.bytes.length);
  out.writeVLong(prefix.startOrd);
  out.writeVLong(prefix.endOrd);
}
 

@Override
public void write(DataOutput output, BlockEncoder blockEncoder) throws IOException {
  if (blockEncoder == null) {
    fst.save(output, output);
  } else {
    ByteBuffersDataOutput bytesDataOutput = ByteBuffersDataOutput.newResettableInstance();
    fst.save(bytesDataOutput, bytesDataOutput);
    BlockEncoder.WritableBytes encodedBytes = blockEncoder.encode(bytesDataOutput.toDataInput(), bytesDataOutput.size());
    output.writeVLong(encodedBytes.size());
    encodedBytes.writeTo(output);
  }
}
 

@Override
public boolean store(DataOutput output) throws IOException {
  CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT);
  output.writeVLong(count);
  output.writeByte(separator);
  output.writeVInt(grams);
  output.writeVLong(totTokens);
  fst.save(output, output);
  return true;
}
 

@Override
public boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  if (fst == null) {
    return false;
  }

  fst.save(output, output);
  output.writeVInt(maxAnalyzedPathsForOneInput);
  output.writeByte((byte) (hasPayloads ? 1 : 0));
  return true;
}
 

@Override
public boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  TSTNode root = trie.getRoot();
  if (root == null) { // empty tree
    return false;
  }
  writeRecursively(output, root);
  return true;
}
 

@Override
public synchronized boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  if (normalCompletion == null || normalCompletion.getFST() == null) {
    return false;
  }
  normalCompletion.getFST().save(output, output);
  return true;
}
 

@Override
public boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  if (fst == null) {
    return false;
  }
  fst.save(output, output);
  return true;
}
 

@Override
public void encodeTerm(DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
  IDVersionTermState state = (IDVersionTermState) _state;
  out.writeVInt(state.docID);
  if (absolute) {
    out.writeVLong(state.idVersion);
  } else {
    long delta = state.idVersion - lastEncodedVersion;
    out.writeZLong(delta);
  }
  lastEncodedVersion = state.idVersion;
}
 

/** 
 * Writes a long in a variable-length format.  Writes between one and 
 * ten bytes. Small values or values representing timestamps with day,
 * hour or second precision typically require fewer bytes.
 * <p>
 * ZLong --&gt; Header, Bytes*?
 * <ul>
 *    <li>Header --&gt; The first two bits indicate the compression scheme:
 *       <ul>
 *          <li>00 - uncompressed
 *          <li>01 - multiple of 1000 (second)
 *          <li>10 - multiple of 3600000 (hour)
 *          <li>11 - multiple of 86400000 (day)
 *       </ul>
 *       Then the next bit is a continuation bit, indicating whether more
 *       bytes need to be read, and the last 5 bits are the lower bits of
 *       the encoded value. In order to reconstruct the value, you need to
 *       combine the 5 lower bits of the header with a vLong in the next
 *       bytes (if the continuation bit is set to 1). Then
 *       {@link BitUtil#zigZagDecode(int) zigzag-decode} it and finally
 *       multiply by the multiple corresponding to the compression scheme.
 *    <li>Bytes --&gt; Potential additional bytes to read depending on the
 *       header.
 * </ul>
 */
// T for "timestamp"
static void writeTLong(DataOutput out, long l) throws IOException {
  int header; 
  if (l % SECOND != 0) {
    header = 0;
  } else if (l % DAY == 0) {
    // timestamp with day precision
    header = DAY_ENCODING;
    l /= DAY;
  } else if (l % HOUR == 0) {
    // timestamp with hour precision, or day precision with a timezone
    header = HOUR_ENCODING;
    l /= HOUR;
  } else {
    // timestamp with second precision
    header = SECOND_ENCODING;
    l /= SECOND;
  }

  final long zigZagL = BitUtil.zigZagEncode(l);
  header |= (zigZagL & 0x1F); // last 5 bits
  final long upperBits = zigZagL >>> 5;
  if (upperBits != 0) {
    header |= 0x20;
  }
  out.writeByte((byte) header);
  if (upperBits != 0) {
    out.writeVLong(upperBits);
  }
}
 

/** Called when another node (replica) wants to copy files from us */
private boolean handleFetchFiles(Random random, Socket socket, DataInput destIn, DataOutput destOut, BufferedOutputStream bos) throws IOException {
  Thread.currentThread().setName("send");

  int replicaID = destIn.readVInt();
  message("top: start fetch for R" + replicaID + " socket=" + socket);
  byte b = destIn.readByte();
  CopyState copyState;
  if (b == 0) {
    // Caller already has CopyState
    copyState = null;
  } else if (b == 1) {
    // Caller does not have CopyState; we pull the latest one:
    copyState = getCopyState();
    Thread.currentThread().setName("send-R" + replicaID + "-" + copyState.version);
  } else {
    // Protocol error:
    throw new IllegalArgumentException("invalid CopyState byte=" + b);
  }

  try {
    if (copyState != null) {
      // Serialize CopyState on the wire to the client:
      writeCopyState(copyState, destOut);
      bos.flush();
    }

    byte[] buffer = new byte[16384];
    int fileCount = 0;
    long totBytesSent = 0;
    while (true) {
      byte done = destIn.readByte();
      if (done == 1) {
        break;
      } else if (done != 0) {
        throw new IllegalArgumentException("expected 0 or 1 byte but got " + done);
      }

      // Name of the file the replica wants us to send:
      String fileName = destIn.readString();

      // Starting offset in the file we should start sending bytes from:
      long fpStart = destIn.readVLong();

      try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
        long len = in.length();
        //message("fetch " + fileName + ": send len=" + len);
        destOut.writeVLong(len);
        in.seek(fpStart);
        long upto = fpStart;
        while (upto < len) {
          int chunk = (int) Math.min(buffer.length, (len-upto));
          in.readBytes(buffer, 0, chunk);
          if (doFlipBitsDuringCopy) {
            if (random.nextInt(3000) == 17 && bitFlipped.contains(fileName) == false) {
              bitFlipped.add(fileName);
              message("file " + fileName + " to R" + replicaID + ": now randomly flipping a bit at byte=" + upto);
              int x = random.nextInt(chunk);
              int bit = random.nextInt(8);
              buffer[x] ^= 1 << bit;
            }
          }
          destOut.writeBytes(buffer, 0, chunk);
          upto += chunk;
          totBytesSent += chunk;
        }
      }

      fileCount++;
    }

    message("top: done fetch files for R" + replicaID + ": sent " + fileCount + " files; sent " + totBytesSent + " bytes");
  } catch (Throwable t) {
    message("top: exception during fetch: " + t.getMessage() + "; now close socket");
    socket.close();
    return false;
  } finally {
    if (copyState != null) {
      message("top: fetch: now release CopyState");
      releaseCopyState(copyState);
    }
  }

  return true;
}
 

@Override
public synchronized boolean store(DataOutput output) throws IOException {
  output.writeVLong(count);
  writeRecursively(output, root);
  return true;
}