org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException Java Examples

public static ContainerCommandRequestProto toProto(
    ByteString bytes, RaftGroupId groupId)
    throws InvalidProtocolBufferException {
  final int i = Integer.BYTES + bytes.substring(0, Integer.BYTES)
      .asReadOnlyByteBuffer().getInt();
  final ContainerCommandRequestProto header
      = ContainerCommandRequestProto
      .parseFrom(bytes.substring(Integer.BYTES, i));
  // TODO: setting pipeline id can be avoided if the client is sending it.
  //       In such case, just have to validate the pipeline id.
  final ContainerCommandRequestProto.Builder b = header.toBuilder();
  if (groupId != null) {
    b.setPipelineID(groupId.getUuid().toString());
  }
  final ByteString data = bytes.substring(i);
  if (header.getCmdType() == Type.WriteChunk) {
    b.setWriteChunk(b.getWriteChunkBuilder().setData(data));
  } else if (header.getCmdType() == Type.PutSmallFile) {
    b.setPutSmallFile(b.getPutSmallFileBuilder().setData(data));
  }
  return b.build();
}
 

@Override
public CompletableFuture<Integer> write(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  final CompletableFuture<Integer> f = files.write(entry.getIndex(),
      h.getPath().toStringUtf8(), h.getClose(), h.getOffset(), smLog.getStateMachineEntry().getStateMachineData());
  // sync only if closing the file
  return h.getClose()? f: null;
}
 

@Override
public CompletableFuture<ByteString> read(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  CompletableFuture<ExamplesProtos.ReadReplyProto> reply =
      files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength());

  return reply.thenApply(ExamplesProtos.ReadReplyProto::getData);
}
 

/**
 * Positions this reader just before the current recordId. Use {@link #next()} to get that
 * element, but take care to check if a value is present using {@link #hasNext()} first.
 */
@Override
public void seek(long recordId) throws RaftLogIOException, InvalidProtocolBufferException {
  LOG.trace("Seeking to recordId={}", recordId);
  // RaftLog starting index
  currentRaftIndex = raftLog.getStartIndex();
  currentRecordId = 0;

  currentLogEntry = null;
  currentLogEntryOffset = -1;
  currentRecord = null;

  loadNext();
  while (currentRecordId < recordId && hasNext()) {
    next();
    currentRecordId++;
  }
}
 

@Override
public CompletableFuture<ByteString> readStateMachineData(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  CompletableFuture<ExamplesProtos.ReadReplyProto> reply =
      files.read(h.getPath().toStringUtf8(), h.getOffset(), h.getLength());

  return reply.thenApply(ExamplesProtos.ReadReplyProto::getData);
}
 

@Override
public CompletableFuture<Integer> writeStateMachineData(LogEntryProto entry) {
  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final ByteString data = smLog.getLogData();
  final FileStoreRequestProto proto;
  try {
    proto = FileStoreRequestProto.parseFrom(data);
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(
        entry.getIndex(), "Failed to parse data, entry=" + entry, e);
  }
  if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) {
    return null;
  }

  final WriteRequestHeaderProto h = proto.getWriteHeader();
  final CompletableFuture<Integer> f = files.write(entry.getIndex(),
      h.getPath().toStringUtf8(), h.getClose(), h.getOffset(), smLog.getStateMachineEntry().getStateMachineData());
  // sync only if closing the file
  return h.getClose()? f: null;
}
 

private static ContainerCommandRequestProto getContainerCommandRequestProto(
    RaftGroupId id, ByteString request)
    throws InvalidProtocolBufferException {
  // TODO: We can avoid creating new builder and set pipeline Id if
  // the client is already sending the pipeline id, then we just have to
  // validate the pipeline Id.
  return ContainerCommandRequestProto.newBuilder(
      ContainerCommandRequestProto.parseFrom(request))
      .setPipelineID(id.getUuid().toString()).build();
}
 

@Override
public CompletableFuture<Message> query(Message request) {
  final ReadRequestProto proto;
  try {
    proto = ReadRequestProto.parseFrom(request.getContent());
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally("Failed to parse " + request, e);
  }

  final String path = proto.getPath().toStringUtf8();
  return files.read(path, proto.getOffset(), proto.getLength())
      .thenApply(reply -> Message.valueOf(reply.toByteString()));
}
 

@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();

  final long index = entry.getIndex();
  updateLastAppliedTermIndex(entry.getTerm(), index);

  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final FileStoreRequestProto request;
  try {
    request = FileStoreRequestProto.parseFrom(smLog.getLogData());
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(index,
        "Failed to parse logData in" + smLog, e);
  }

  switch(request.getRequestCase()) {
    case DELETE:
      return delete(index, request.getDelete());
    case WRITEHEADER:
      return writeCommit(index, request.getWriteHeader(), smLog.getStateMachineEntry().getStateMachineData().size());
    case WRITE:
      // WRITE should not happen here since
      // startTransaction converts WRITE requests to WRITEHEADER requests.
    default:
      LOG.error(getId() + ": Unexpected request case " + request.getRequestCase());
      return FileStoreCommon.completeExceptionally(index,
          "Unexpected request case " + request.getRequestCase());
  }
}
 

@Override
public TransactionContext applyTransactionSerial(TransactionContext trx) {
    RaftProtos.LogEntryProto x = trx.getLogEntry();
    MetaSMRequestProto req = null;
    try {
        req = MetaSMRequestProto.parseFrom(x.getStateMachineLogEntry().getLogData());
    } catch (InvalidProtocolBufferException e) {
        e.printStackTrace();
    }
    switch (req.getTypeCase()) {
        case REGISTERREQUEST:
            LogServiceRegisterLogRequestProto r = req.getRegisterRequest();
            LogName logname = LogServiceProtoUtil.toLogName(r.getLogname());
            RaftGroup rg = MetaServiceProtoUtil.toRaftGroup(r.getRaftGroup());
            map.put(logname, rg);
            LOG.info("Log {} registered at {} with group {} ", logname, getId(), rg );
            break;
        case UNREGISTERREQUEST:
            LogServiceUnregisterLogRequestProto unregReq = req.getUnregisterRequest();
            logname = LogServiceProtoUtil.toLogName(unregReq.getLogname());
            map.remove(logname);
            break;
        case PINGREQUEST:
            LogServicePingRequestProto pingRequest = req.getPingRequest();
            RaftPeer peer = MetaServiceProtoUtil.toRaftPeer(pingRequest.getPeer());
            if (peers.contains(peer)) {
                //Do Nothing, that's just heartbeat
            } else {
                peers.add(peer);
                avail.add(new PeerGroups(peer));
            }
            break;

        default:
    }
    return super.applyTransactionSerial(trx);
}
 

/**
 * Returns the next log entry. Ensure {@link #hasNext()} returns true before
 * calling this method.
 */
@Override
public byte[] next() throws RaftLogIOException, InvalidProtocolBufferException {
  if (currentRecord == null) {
    throw new NoSuchElementException();
  }
  ByteString current = currentRecord;
  currentRecord = null;
  loadNext();
  return current.toByteArray();
}
 

@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();

  final long index = entry.getIndex();
  updateLastAppliedTermIndex(entry.getTerm(), index);

  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final FileStoreRequestProto request;
  try {
    request = FileStoreRequestProto.parseFrom(smLog.getLogData());
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(index,
        "Failed to parse logData in" + smLog, e);
  }

  switch(request.getRequestCase()) {
    case DELETE:
      return delete(index, request.getDelete());
    case WRITEHEADER:
      return writeCommit(index, request.getWriteHeader(), smLog.getStateMachineEntry().getStateMachineData().size());
    case WRITE:
      // WRITE should not happen here since
      // startTransaction converts WRITE requests to WRITEHEADER requests.
    default:
      LOG.error(getId() + ": Unexpected request case " + request.getRequestCase());
      return FileStoreCommon.completeExceptionally(index,
          "Unexpected request case " + request.getRequestCase());
  }
}
 

@Override
public CompletableFuture<Message> query(Message request) {
  final ReadRequestProto proto;
  try {
    proto = ReadRequestProto.parseFrom(request.getContent());
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally("Failed to parse " + request, e);
  }

  final String path = proto.getPath().toStringUtf8();
  return files.read(path, proto.getOffset(), proto.getLength())
      .thenApply(reply -> Message.valueOf(reply.toByteString()));
}
 

void runTestStaleReadAsync(CLUSTER cluster) throws Exception {
  final int numMesssages = 10;
  try (RaftClient client = cluster.createClient()) {
    RaftTestUtil.waitForLeader(cluster);

    // submit some messages
    final List<CompletableFuture<RaftClientReply>> futures = new ArrayList<>();
    for (int i = 0; i < numMesssages; i++) {
      final String s = "" + i;
      LOG.info("sendAsync " + s);
      futures.add(client.sendAsync(new SimpleMessage(s)));
    }
    Assert.assertEquals(numMesssages, futures.size());
    final List<RaftClientReply> replies = new ArrayList<>();
    for (CompletableFuture<RaftClientReply> f : futures) {
      final RaftClientReply r = f.join();
      Assert.assertTrue(r.isSuccess());
      replies.add(r);
    }
    futures.clear();

    // Use a follower with the max commit index
    final RaftClientReply lastWriteReply = replies.get(replies.size() - 1);
    final RaftPeerId leader = lastWriteReply.getServerId();
    LOG.info("leader = " + leader);
    final Collection<CommitInfoProto> commitInfos = lastWriteReply.getCommitInfos();
    LOG.info("commitInfos = " + commitInfos);
    final CommitInfoProto followerCommitInfo = commitInfos.stream()
        .filter(info -> !RaftPeerId.valueOf(info.getServer().getId()).equals(leader))
        .max(Comparator.comparing(CommitInfoProto::getCommitIndex)).get();
    final RaftPeerId follower = RaftPeerId.valueOf(followerCommitInfo.getServer().getId());
    final long followerCommitIndex = followerCommitInfo.getCommitIndex();
    LOG.info("max follower = {}, commitIndex = {}", follower, followerCommitIndex);

    // test a failure case
    testFailureCaseAsync("sendStaleReadAsync(..) with a larger commit index",
        () -> client.sendStaleReadAsync(
            new SimpleMessage("" + Long.MAX_VALUE),
            followerCommitInfo.getCommitIndex(), follower),
        StateMachineException.class, IndexOutOfBoundsException.class);

    // test sendStaleReadAsync
    for (int i = 0; i < numMesssages; i++) {
      final RaftClientReply reply = replies.get(i);
      final String query = "" + i;
      LOG.info("query=" + query + ", reply=" + reply);
      final Message message = new SimpleMessage(query);
      final CompletableFuture<RaftClientReply> readFuture = client.sendReadOnlyAsync(message);

      futures.add(readFuture.thenCompose(r -> {
        if (reply.getLogIndex() <= followerCommitIndex) {
          LOG.info("sendStaleReadAsync, query=" + query);
          return client.sendStaleReadAsync(message, followerCommitIndex, follower);
        } else {
          return CompletableFuture.completedFuture(null);
        }
      }).thenApply(staleReadReply -> {
        if (staleReadReply == null) {
          return null;
        }

        final ByteString expected = readFuture.join().getMessage().getContent();
        final ByteString computed = staleReadReply.getMessage().getContent();
        try {
          LOG.info("query " + query + " returns "
              + LogEntryProto.parseFrom(expected).getStateMachineLogEntry().getLogData().toStringUtf8());
        } catch (InvalidProtocolBufferException e) {
          throw new CompletionException(e);
        }

        Assert.assertEquals("log entry mismatch for query=" + query, expected, computed);
        return null;
      }));
    }
    JavaUtils.allOf(futures).join();
  }
}
 

public static LogName parseFrom(ByteString logName)
    throws InvalidProtocolBufferException {
  LogServiceProtos.LogNameProto logNameProto = LogServiceProtos.LogNameProto.parseFrom(logName);
  return new LogName(logNameProto.getName());
}
 

void runTestStaleReadAsync(CLUSTER cluster) throws Exception {
  final int numMesssages = 10;
  try (RaftClient client = cluster.createClient()) {
    RaftTestUtil.waitForLeader(cluster);

    // submit some messages
    final List<CompletableFuture<RaftClientReply>> futures = new ArrayList<>();
    for (int i = 0; i < numMesssages; i++) {
      final String s = "" + i;
      LOG.info("sendAsync " + s);
      futures.add(client.sendAsync(new SimpleMessage(s)));
    }
    Assert.assertEquals(numMesssages, futures.size());
    final List<RaftClientReply> replies = new ArrayList<>();
    for (CompletableFuture<RaftClientReply> f : futures) {
      final RaftClientReply r = f.join();
      Assert.assertTrue(r.isSuccess());
      replies.add(r);
    }
    futures.clear();

    // Use a follower with the max commit index
    final RaftClientReply lastWriteReply = replies.get(replies.size() - 1);
    final RaftPeerId leader = lastWriteReply.getServerId();
    LOG.info("leader = " + leader);
    final Collection<CommitInfoProto> commitInfos = lastWriteReply.getCommitInfos();
    LOG.info("commitInfos = " + commitInfos);
    final CommitInfoProto followerCommitInfo = commitInfos.stream()
        .filter(info -> !RaftPeerId.valueOf(info.getServer().getId()).equals(leader))
        .max(Comparator.comparing(CommitInfoProto::getCommitIndex)).get();
    final RaftPeerId follower = RaftPeerId.valueOf(followerCommitInfo.getServer().getId());
    final long followerCommitIndex = followerCommitInfo.getCommitIndex();
    LOG.info("max follower = {}, commitIndex = {}", follower, followerCommitIndex);

    // test a failure case
    testFailureCaseAsync("sendStaleReadAsync(..) with a larger commit index",
        () -> client.sendStaleReadAsync(
            new SimpleMessage("" + Long.MAX_VALUE),
            followerCommitInfo.getCommitIndex(), follower),
        StateMachineException.class, IndexOutOfBoundsException.class);

    // test sendStaleReadAsync
    for (int i = 0; i < numMesssages; i++) {
      final RaftClientReply reply = replies.get(i);
      final String query = "" + i;
      LOG.info("query=" + query + ", reply=" + reply);
      final Message message = new SimpleMessage(query);
      final CompletableFuture<RaftClientReply> readFuture = client.sendReadOnlyAsync(message);

      futures.add(readFuture.thenCompose(r -> {
        if (reply.getLogIndex() <= followerCommitIndex) {
          LOG.info("sendStaleReadAsync, query=" + query);
          return client.sendStaleReadAsync(message, followerCommitIndex, follower);
        } else {
          return CompletableFuture.completedFuture(null);
        }
      }).thenApply(staleReadReply -> {
        if (staleReadReply == null) {
          return null;
        }

        final ByteString expected = readFuture.join().getMessage().getContent();
        final ByteString computed = staleReadReply.getMessage().getContent();
        try {
          LOG.info("query " + query + " returns "
              + LogEntryProto.parseFrom(expected).getStateMachineLogEntry().getLogData().toStringUtf8());
        } catch (InvalidProtocolBufferException e) {
          throw new CompletionException(e);
        }

        Assert.assertEquals("log entry mismatch for query=" + query, expected, computed);
        return null;
      }));
    }
    JavaUtils.allOf(futures).join();
  }
}
 

@Override
public TransactionContext applyTransactionSerial(TransactionContext trx) {
    RaftProtos.LogEntryProto x = trx.getLogEntry();
    MetaSMRequestProto req = null;
    try {
        req = MetaSMRequestProto.parseFrom(x.getStateMachineLogEntry().getLogData());
    } catch (InvalidProtocolBufferException e) {
        e.printStackTrace();
    }
    switch (req.getTypeCase()) {
        case REGISTERREQUEST:
            LogServiceRegisterLogRequestProto r = req.getRegisterRequest();
            LogName logname = LogServiceProtoUtil.toLogName(r.getLogname());
            RaftGroup rg = MetaServiceProtoUtil.toRaftGroup(r.getRaftGroup());
            rg.getPeers().stream().forEach(raftPeer -> {
                Set<LogName> logNames;
                if(!peerLogs.containsKey(raftPeer)) {
                    logNames = new HashSet<>();
                    peerLogs.put(raftPeer, logNames);
                } else {
                    logNames = peerLogs.get(raftPeer);
                }
                logNames.add(logname);

            });
            map.put(logname, rg);

            LOG.info("Log {} registered at {} with group {} ", logname, getId(), rg );
            break;
        case UNREGISTERREQUEST:
            LogServiceUnregisterLogRequestProto unregReq = req.getUnregisterRequest();
            logname = LogServiceProtoUtil.toLogName(unregReq.getLogname());
            map.remove(logname);
            break;
        case PINGREQUEST:
            LogServicePingRequestProto pingRequest = req.getPingRequest();
            RaftPeer peer = MetaServiceProtoUtil.toRaftPeer(pingRequest.getPeer());
            //If Set<RaftPeer> contains peer then do nothing as that's just heartbeat else add the peer to the set.
            if (!peers.contains(peer)) {
                peers.add(peer);
                avail.add(new PeerGroups(peer));
                heartbeatInfo.put(peer,  System.currentTimeMillis());
            }
            break;
        case HEARTBEATREQUEST:
            MetaServiceProtos.LogServiceHeartbeatRequestProto heartbeatRequest = req.getHeartbeatRequest();
            RaftPeer heartbeatPeer = MetaServiceProtoUtil.toRaftPeer(heartbeatRequest.getPeer());
            heartbeatInfo.put(heartbeatPeer,  System.currentTimeMillis());
            break;
        default:
    }
    return super.applyTransactionSerial(trx);
}
 

/**
 * Finds the next record from the RaftLog and sets it as {@link #currentRecord}.
 */
private void loadNext() throws RaftLogIOException, InvalidProtocolBufferException {
  // Clear the old "current" record
  currentRecord = null;
  if (LOG.isTraceEnabled()) {
    LOG.trace("Loading next value: raftIndex={}, recordId={}, proto='{}', offset={}",
        currentRaftIndex, currentRecordId,
        currentLogEntry == null ? "null" : TextFormat.shortDebugString(currentLogEntry),
            currentLogEntryOffset);
  }
  // Continue iterating over the current entry.
  if (currentLogEntry != null) {
    assert currentLogEntryOffset != -1;
    currentLogEntryOffset++;

    // We have an element to read from our current entry
    if (currentLogEntryOffset < currentLogEntry.getDataCount()) {
      currentRecord = currentLogEntry.getData(currentLogEntryOffset);
      return;
    }
    // We don't have an element in our current entry so null it out.
    currentLogEntry = null;
    currentLogEntryOffset = -1;
    // Also, increment to the next element in the RaftLog
    currentRaftIndex++;
  }

  // Make sure we don't read off the end of the Raft log
  for (; currentRaftIndex <= raftLog.getLastCommittedIndex(); currentRaftIndex++) {
    try {
      LogEntryProto entry = raftLog.get(currentRaftIndex);
      if (LOG.isTraceEnabled()) {
        LOG.trace("Raft Index: {} Entry: {}", currentRaftIndex,
            TextFormat.shortDebugString(entry));
      }
      if (entry == null || entry.hasConfigurationEntry()) {
        continue;
      }

      LogServiceRequestProto logServiceProto =
          LogServiceRequestProto.parseFrom(entry.getStateMachineLogEntry().getLogData());
      // TODO is it possible to get LogService messages that aren't appends?
      if (RequestCase.APPENDREQUEST != logServiceProto.getRequestCase()) {
        continue;
      }

      currentLogEntry = logServiceProto.getAppendRequest();
      currentLogEntryOffset = 0;
      if (currentLogEntry.getDataCount() > 0) {
        currentRecord = currentLogEntry.getData(currentLogEntryOffset);
        return;
      }
      currentLogEntry = null;
      currentLogEntryOffset = -1;
    } catch (RaftLogIOException e) {
      LOG.error("Caught exception reading from RaftLog", e);
      throw e;
    } catch (InvalidProtocolBufferException e) {
      LOG.error("Caught exception reading LogService protobuf from RaftLog", e);
      throw e;
    }
  }
  // If we make it here, we've read off the end of the RaftLog.
}
 

/**
 * Returns true if there is a log entry to read.
 */
@Override
public boolean hasNext() throws RaftLogIOException, InvalidProtocolBufferException {
  return currentRecord != null;
}
 

public static LogName parseFrom(ByteString logName)
    throws InvalidProtocolBufferException {
  LogServiceProtos.LogNameProto logNameProto = LogServiceProtos.LogNameProto.parseFrom(logName);
  return new LogName(logNameProto.getName());
}
 

private ContainerCommandRequestProto message2ContainerCommandRequestProto(
    Message message) throws InvalidProtocolBufferException {
  return ContainerCommandRequestMessage.toProto(message.getContent(), gid);
}