Java Code Examples for org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader#getSerializedSize()

The following examples show how to use org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader#getSerializedSize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FanOutOneBlockAsyncDFSOutput.java    From hbase with Apache License 2.0 6 votes vote down vote up 
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
  if (evt instanceof IdleStateEvent) {
    IdleStateEvent e = (IdleStateEvent) evt;
    if (e.state() == READER_IDLE) {
      failed(ctx.channel(),
        () -> new IOException("Timeout(" + timeoutMs + "ms) waiting for response"));
    } else if (e.state() == WRITER_IDLE) {
      PacketHeader heartbeat = new PacketHeader(4, 0, HEART_BEAT_SEQNO, false, 0, false);
      int len = heartbeat.getSerializedSize();
      ByteBuf buf = alloc.buffer(len);
      heartbeat.putInBuffer(buf.nioBuffer(0, len));
      buf.writerIndex(len);
      ctx.channel().writeAndFlush(buf);
    }
    return;
  }
  super.userEventTriggered(ctx, evt);
}
Example 2
Source File: BlockSender.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Write packet header into {@code pkt},
 * return the length of the header written.
 */
private int writePacketHeader(ByteBuffer pkt, int dataLen, int packetLen) {
  pkt.clear();
  // both syncBlock and syncPacket are false
  PacketHeader header = new PacketHeader(packetLen, offset, seqno,
      (dataLen == 0), dataLen, false);
  
  int size = header.getSerializedSize();
  pkt.position(PacketHeader.PKT_MAX_HEADER_LEN - size);
  header.putInBuffer(pkt);
  return size;
}
Example 3
Source File: BlockSender.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Write packet header into {@code pkt},
 * return the length of the header written.
 */
private int writePacketHeader(ByteBuffer pkt, int dataLen, int packetLen) {
  pkt.clear();
  // both syncBlock and syncPacket are false
  PacketHeader header = new PacketHeader(packetLen, offset, seqno,
      (dataLen == 0), dataLen, false);
  
  int size = header.getSerializedSize();
  pkt.position(PacketHeader.PKT_MAX_HEADER_LEN - size);
  header.putInBuffer(pkt);
  return size;
}
Example 4
Source File: FanOutOneBlockAsyncDFSOutput.java    From hbase with Apache License 2.0 5 votes vote down vote up 
private void flushBuffer(CompletableFuture<Long> future, ByteBuf dataBuf,
    long nextPacketOffsetInBlock, boolean syncBlock) {
  int dataLen = dataBuf.readableBytes();
  int chunkLen = summer.getBytesPerChecksum();
  int trailingPartialChunkLen = dataLen % chunkLen;
  int numChecks = dataLen / chunkLen + (trailingPartialChunkLen != 0 ? 1 : 0);
  int checksumLen = numChecks * summer.getChecksumSize();
  ByteBuf checksumBuf = alloc.directBuffer(checksumLen);
  summer.calculateChunkedSums(dataBuf.nioBuffer(), checksumBuf.nioBuffer(0, checksumLen));
  checksumBuf.writerIndex(checksumLen);
  PacketHeader header = new PacketHeader(4 + checksumLen + dataLen, nextPacketOffsetInBlock,
      nextPacketSeqno, false, dataLen, syncBlock);
  int headerLen = header.getSerializedSize();
  ByteBuf headerBuf = alloc.buffer(headerLen);
  header.putInBuffer(headerBuf.nioBuffer(0, headerLen));
  headerBuf.writerIndex(headerLen);
  Callback c = new Callback(future, nextPacketOffsetInBlock + dataLen, datanodeList);
  waitingAckQueue.addLast(c);
  // recheck again after we pushed the callback to queue
  if (state != State.STREAMING && waitingAckQueue.peekFirst() == c) {
    future.completeExceptionally(new IOException("stream already broken"));
    // it's the one we have just pushed or just a no-op
    waitingAckQueue.removeFirst();
    return;
  }
  // TODO: we should perhaps measure time taken per DN here;
  //       we could collect statistics per DN, and/or exclude bad nodes in createOutput.
  datanodeList.forEach(ch -> {
    ch.write(headerBuf.retainedDuplicate());
    ch.write(checksumBuf.retainedDuplicate());
    ch.writeAndFlush(dataBuf.retainedDuplicate());
  });
  checksumBuf.release();
  headerBuf.release();
  dataBuf.release();
  nextPacketSeqno++;
}