org.apache.ratis.thirdparty.com.google.protobuf.ByteString Java Examples

@Override
public CompletableFuture<Integer> write(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  final CompletableFuture<Integer> f = files.write(entry.getIndex(),
      h.getPath().toStringUtf8(), h.getClose(), h.getOffset(), smLog.getStateMachineEntry().getStateMachineData());
  // sync only if closing the file
  return h.getClose()? f: null;
}
 

@Override
public CompletableFuture<ByteString> read(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  CompletableFuture<ExamplesProtos.ReadReplyProto> reply =
      files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength());

  return reply.thenApply(ExamplesProtos.ReadReplyProto::getData);
}
 

/**
 * Gets a response to the read small file call.
 * @param msg - Msg
 * @param data  - Data
 * @param info  - Info
 * @return    Response.
 */
public static ContainerCommandResponseProto getGetSmallFileResponseSuccess(
    ContainerCommandRequestProto msg, ByteString data, ChunkInfo info) {

  Preconditions.checkNotNull(msg);

  ReadChunkResponseProto.Builder readChunk =
      ReadChunkResponseProto.newBuilder()
          .setChunkData(info)
          .setData((data))
          .setBlockID(msg.getGetSmallFile().getBlock().getBlockID());

  GetSmallFileResponseProto.Builder getSmallFile =
      GetSmallFileResponseProto.newBuilder().setData(readChunk);

  return getSuccessResponseBuilder(msg)
      .setCmdType(Type.GetSmallFile)
      .setGetSmallFile(getSmallFile)
      .build();
}
 

@Override
public TransactionContext startTransaction(RaftClientRequest request) throws IOException {
  final ByteString content = request.getMessage().getContent();
  final FileStoreRequestProto proto = FileStoreRequestProto.parseFrom(content);
  final TransactionContext.Builder b = TransactionContext.newBuilder()
      .setStateMachine(this)
      .setClientRequest(request);

  if (proto.getRequestCase() == FileStoreRequestProto.RequestCase.WRITE) {
    final WriteRequestProto write = proto.getWrite();
    final FileStoreRequestProto newProto = FileStoreRequestProto.newBuilder()
        .setWriteHeader(write.getHeader()).build();
    b.setLogData(newProto.toByteString()).setStateMachineData(write.getData());
  } else {
    b.setLogData(content);
  }
  return b.build();
}
 

@Override
public void run() throws Exception {
  RaftProperties raftProperties = new RaftProperties();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);


}
 

private static <OUTPUT, THROWABLE extends Throwable> OUTPUT writeImpl(
    CheckedFunction<ByteString, OUTPUT, THROWABLE> sendFunction,
    String path, long offset, boolean close, ByteBuffer data)
    throws THROWABLE {
  final WriteRequestHeaderProto.Builder header = WriteRequestHeaderProto.newBuilder()
      .setPath(ProtoUtils.toByteString(path))
      .setOffset(offset)
      .setLength(data.position())
      .setClose(close);

  final WriteRequestProto.Builder write = WriteRequestProto.newBuilder()
      .setHeader(header)
      .setData(ByteString.copyFrom(data));

  final FileStoreRequestProto request = FileStoreRequestProto.newBuilder().setWrite(write).build();
  return sendFunction.apply(request.toByteString());
}
 

public static ContainerCommandRequestProto toProto(
    ByteString bytes, RaftGroupId groupId)
    throws InvalidProtocolBufferException {
  final int i = Integer.BYTES + bytes.substring(0, Integer.BYTES)
      .asReadOnlyByteBuffer().getInt();
  final ContainerCommandRequestProto header
      = ContainerCommandRequestProto
      .parseFrom(bytes.substring(Integer.BYTES, i));
  // TODO: setting pipeline id can be avoided if the client is sending it.
  //       In such case, just have to validate the pipeline id.
  final ContainerCommandRequestProto.Builder b = header.toBuilder();
  if (groupId != null) {
    b.setPipelineID(groupId.getUuid().toString());
  }
  final ByteString data = bytes.substring(i);
  if (header.getCmdType() == Type.WriteChunk) {
    b.setWriteChunk(b.getWriteChunkBuilder().setData(data));
  } else if (header.getCmdType() == Type.PutSmallFile) {
    b.setPutSmallFile(b.getPutSmallFileBuilder().setData(data));
  }
  return b.build();
}
 

private static <OUTPUT, THROWABLE extends Throwable> OUTPUT writeImpl(
    CheckedFunction<ByteString, OUTPUT, THROWABLE> sendFunction,
    String path, long offset, boolean close, ByteBuffer data)
    throws THROWABLE {
  final WriteRequestHeaderProto.Builder header = WriteRequestHeaderProto.newBuilder()
      .setPath(ProtoUtils.toByteString(path))
      .setOffset(offset)
      .setLength(data.position())
      .setClose(close);

  final WriteRequestProto.Builder write = WriteRequestProto.newBuilder()
      .setHeader(header)
      .setData(ByteString.copyFrom(data));

  final FileStoreRequestProto request = FileStoreRequestProto.newBuilder().setWrite(write).build();
  return sendFunction.apply(request.toByteString());
}
 

/**
 * Calls the container protocol to write a chunk.
 *
 * @param xceiverClient client to perform call
 * @param chunk information about chunk to write
 * @param blockID ID of the block
 * @param data the data of the chunk to write
 * @throws IOException if there is an I/O error while performing the call
 */
public static XceiverClientReply writeChunkAsync(
    XceiverClientSpi xceiverClient, ChunkInfo chunk, BlockID blockID,
    ByteString data)
    throws IOException, ExecutionException, InterruptedException {
  WriteChunkRequestProto.Builder writeChunkRequest =
      WriteChunkRequestProto.newBuilder()
          .setBlockID(blockID.getDatanodeBlockIDProtobuf())
          .setChunkData(chunk).setData(data);
  String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
  ContainerCommandRequestProto.Builder builder =
      ContainerCommandRequestProto.newBuilder().setCmdType(Type.WriteChunk)
          .setContainerID(blockID.getContainerID())
          .setDatanodeUuid(id).setWriteChunk(writeChunkRequest);
  String encodedToken = getEncodedBlockToken(new Text(blockID.
      getContainerBlockID().toString()));
  if (encodedToken != null) {
    builder.setEncodedToken(encodedToken);
  }
  ContainerCommandRequestProto request = builder.build();
  return xceiverClient.sendCommandAsync(request);
}
 

public static LogServiceRequestProto toAppendBBEntryLogRequestProto(LogName name,
    List<ByteBuffer> entries) {
  LogNameProto logNameProto =
      LogNameProto.newBuilder().setName(name.getName()).build();
  AppendLogEntryRequestProto.Builder builder = AppendLogEntryRequestProto.newBuilder();
  builder.setLogName(logNameProto);
  for (int i=0; i < entries.size(); i++) {
    ByteBuffer currentBuf = entries.get(i);
    // Save the current position
    int pos = currentBuf.position();
    builder.addData(ByteString.copyFrom(currentBuf));
    // Reset it after we're done reading the bytes
    currentBuf.position(pos);
  }
  return LogServiceRequestProto.newBuilder().setAppendRequest(builder.build()).build();
}
 

/**
 * Returns the next log entry. Ensure {@link #hasNext()} returns true before
 * calling this method.
 */
@Override
public byte[] next() throws RaftLogIOException, InvalidProtocolBufferException {
  if (currentRecord == null) {
    throw new NoSuchElementException();
  }
  ByteString current = currentRecord;
  currentRecord = null;
  loadNext();
  return current.toByteArray();
}
 

private <REQUEST, REPLY> REPLY processRequest(
    REQUEST request, ByteString replyId,
    CheckedFunction<RaftServerProtocolPB, REPLY, ServiceException> f)
    throws IOException {
  CodeInjectionForTesting.execute(SEND_SERVER_REQUEST, getId(), null, request);

  final RaftServerProtocolPB proxy = getProxies().getProxy(
      RaftPeerId.valueOf(replyId)).getProtocol();
  try {
    return f.apply(proxy);
  } catch (ServiceException se) {
    throw ProtoUtils.toIOException(se);
  }
}
 

Writer verify() throws IOException {
  final Random r = new Random(seed);
  final int size = fileSize.getSizeInt();

  for(int offset = 0; offset < size; ) {
    final int remaining = size - offset;
    final int n = Math.min(remaining, buffer.length);
    final ByteString read = client.read(fileName, offset, n);
    final ByteBuffer expected = randomBytes(n, r);
    verify(read, offset, n, expected);
    offset += n;
  }
  return this;
}
 

static StateMachineLogEntryProto toStateMachineLogEntryProto(
    ClientId clientId, long callId, ByteString logData, ByteString stateMachineData) {
  final StateMachineLogEntryProto.Builder b = StateMachineLogEntryProto.newBuilder()
      .setClientId(clientId.toByteString())
      .setCallId(callId)
      .setLogData(logData);
  if (stateMachineData != null) {
    b.setStateMachineEntry(toStateMachineEntryProtoBuilder(stateMachineData));
  }
  return b.build();
}
 

static StateMachineLogEntryProto toStateMachineLogEntryProto(
    RaftClientRequest request, ByteString logData, ByteString stateMachineData) {
  if (logData == null) {
    logData = request.getMessage().getContent();
  }
  return toStateMachineLogEntryProto(request.getClientId(), request.getCallId(), logData, stateMachineData);
}
 

@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setMaxCachedSegmentNum(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setSchedulerThreads(raftProperties, 10);
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 

CompletableFuture<Integer> submitCreate(
    CheckedFunction<Path, Path, IOException> resolver, ByteString data, boolean close,
    ExecutorService executor, RaftPeerId id, long index) {
  final Supplier<String> name = () -> "create(" + getRelativePath() + ", "
      + close + ") @" + id + ":" + index;
  final CheckedSupplier<Integer, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    if (out == null) {
      out = new FileOut(resolver.apply(getRelativePath()));
    }
    return write(0L, data, close);
  }, name);
  return submitWrite(task, executor, id, index);
}
 

private static ByteString toByteString(UUID uuid) {
  Objects.requireNonNull(uuid, "uuid == null");
  final ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
  buf.putLong(uuid.getMostSignificantBits());
  buf.putLong(uuid.getLeastSignificantBits());
  return ByteString.copyFrom(buf.array());
}
 

@Test
public void testRaftPeerId() {
  final RaftPeerId id = RaftPeerId.valueOf("abc");
  final ByteString bytes = id.toByteString();
  Assert.assertEquals(bytes, id.toByteString());
  Assert.assertEquals(id, RaftPeerId.valueOf(bytes));
}
 

public static ReadLogReplyProto toReadLogReplyProto(List<byte[]> entries, Throwable t) {
  ReadLogReplyProto.Builder builder = ReadLogReplyProto.newBuilder();
  if (t != null) {
    builder.setException(toLogException(t));
  } else {
    for(byte[] record: entries) {
      builder.addLogRecord( ByteString.copyFrom(record));
    }
  }
  return builder.build();
}
 

@Override
public ByteString getContent() {
  final int length = variable.length() + expression.length();
  final byte[] bytes = new byte[length];
  final int offset = variable.toBytes(bytes, 0);
  expression.toBytes(bytes, offset);
  return toByteString(bytes);
}
 

private static Function<ByteBuffer, ByteString> newChecksumByteBufferFunction(
    Supplier<ChecksumByteBuffer> constructor) {
  final ChecksumByteBuffer algorithm = constructor.get();
  return  data -> {
    algorithm.reset();
    algorithm.update(data);
    return int2ByteString((int)algorithm.getValue());
  };
}
 

static StateMachineLogEntryProto toStateMachineLogEntryProto(
    ClientId clientId, long callId, ByteString logData, ByteString stateMachineData) {
  final StateMachineLogEntryProto.Builder b = StateMachineLogEntryProto.newBuilder()
      .setClientId(clientId.toByteString())
      .setCallId(callId)
      .setLogData(logData);
  if (stateMachineData != null) {
    b.setStateMachineEntry(toStateMachineEntryProtoBuilder(stateMachineData));
  }
  return b.build();
}
 

/**
 * Compute checksum using the algorithm for the data upto the max length.
 * @param data input data
 * @param function the checksum function
 * @param maxLength the max length of data
 * @return computed checksum ByteString
 */
private static ByteString computeChecksum(ByteBuffer data,
    Function<ByteBuffer, ByteString> function, int maxLength) {
  final int limit = data.limit();
  try {
    final int maxIndex = data.position() + maxLength;
    if (limit > maxIndex) {
      data.limit(maxIndex);
    }
    return function.apply(data);
  } finally {
    data.limit(limit);
  }
}
 

private static ByteString toByteString(UUID uuid) {
  Objects.requireNonNull(uuid, "uuid == null");
  final ByteBuffer buf = ByteBuffer.wrap(new byte[BYTE_LENGTH]);
  buf.putLong(uuid.getMostSignificantBits());
  buf.putLong(uuid.getLeastSignificantBits());
  return ByteString.copyFrom(buf.array());
}
 

private int write(long offset, ByteString data, boolean close) throws IOException {
  if (offset != writeSize) {
    throw new IOException("Offset/size mismatched: offset = " + offset
        + " != writeSize = " + writeSize + ", path=" + getRelativePath());
  }
  if (out == null) {
    throw new IOException("File output is not initialized, path=" + getRelativePath());
  }

  synchronized (out) {
    int n = 0;
    if (data != null) {
      final ByteBuffer buffer = data.asReadOnlyByteBuffer();
      try {
        for (; buffer.remaining() > 0; ) {
          n += out.write(buffer);
        }
      } finally {
        writeSize += n;
      }
    }

    if (close) {
      out.close();
    }
    return n;
  }
}
 

private int write(long offset, ByteString data, boolean close) throws IOException {
  // If leader finish write data with offset = 4096 and writeSize become 8192,
  // and 2 follower has not written the data with offset = 4096,
  // then start a leader election. So client will retry send the data with offset = 4096,
  // then offset < writeSize in leader.
  if (offset < writeSize) {
    return data.size();
  }
  if (offset != writeSize) {
    throw new IOException("Offset/size mismatched: offset = " + offset
        + " != writeSize = " + writeSize + ", path=" + getRelativePath());
  }
  if (out == null) {
    throw new IOException("File output is not initialized, path=" + getRelativePath());
  }

  synchronized (out) {
    int n = 0;
    if (data != null) {
      final ByteBuffer buffer = data.asReadOnlyByteBuffer();
      try {
        for (; buffer.remaining() > 0; ) {
          n += out.write(buffer);
        }
      } finally {
        writeSize += n;
      }
    }

    if (close) {
      out.close();
    }
    return n;
  }
}
 

CompletableFuture<ReadReplyProto> read(String relative, long offset, long length) {
  final Supplier<String> name = () -> "read(" + relative
      + ", " + offset + ", " + length + ") @" + getId();
  final CheckedSupplier<ReadReplyProto, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    final FileInfo info = files.get(relative);
    final ReadReplyProto.Builder reply = ReadReplyProto.newBuilder()
        .setResolvedPath(FileStoreCommon.toByteString(info.getRelativePath()))
        .setOffset(offset);

    final ByteString bytes = info.read(this::resolve, offset, length);
    return reply.setData(bytes).build();
  }, name);
  return submit(task, reader);
}
 

static void runTest(int length,
    BiFunction<BlockID, ByteString, ContainerCommandRequestProto> method)
    throws Exception {
  System.out.println("length=" + length);
  final BlockID blockID = new BlockID(RANDOM.nextLong(), RANDOM.nextLong());
  final ByteString data = newData(length);

  final ContainerCommandRequestProto original = method.apply(blockID, data);
  final ContainerCommandRequestMessage message
      = ContainerCommandRequestMessage.toMessage(original, null);
  final ContainerCommandRequestProto computed
      = ContainerCommandRequestMessage.toProto(message.getContent(), null);
  Assert.assertEquals(original, computed);
}
 

public static List<byte[]> toListByteArray(List<ByteString> list) {
  List<byte[]> retVal = new ArrayList<byte[]>(list.size());
  for(int i=0; i < list.size(); i++) {
    retVal.add(list.get(i).toByteArray());
  }
  return retVal;
}