Java Code Examples for org.apache.ratis.proto.RaftProtos.LogEntryProto#getIndex()

@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();
  final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData());

  final long index = entry.getIndex();
  final Double result;
  try(AutoCloseableLock writeLock = writeLock()) {
    result = assignment.evaluate(variables);
    updateLastAppliedTermIndex(entry.getTerm(), index);
  }
  final Expression r = Expression.Utils.double2Expression(result);
  final CompletableFuture<Message> f = CompletableFuture.completedFuture(Expression.Utils.toMessage(r));

  final RaftPeerRole role = trx.getServerRole();
  if (role == RaftPeerRole.LEADER) {
    LOG.info("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  } else {
    LOG.debug("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}-{}: variables={}", getId(), index, variables);
  }
  return f;
}
 

private void append(boolean keepEntryInCache, LogEntryProto entry) {
  Objects.requireNonNull(entry, "entry == null");
  if (records.isEmpty()) {
    Preconditions.assertTrue(entry.getIndex() == startIndex,
        "gap between start index %s and first entry to append %s",
        startIndex, entry.getIndex());
  }

  final LogRecord currentLast = getLastRecord();
  if (currentLast != null) {
    Preconditions.assertTrue(entry.getIndex() == currentLast.getTermIndex().getIndex() + 1,
        "gap between entries %s and %s", entry.getIndex(), currentLast.getTermIndex().getIndex());
  }

  final LogRecord record = new LogRecord(totalSize, entry);
  records.add(record);
  if (keepEntryInCache) {
    entryCache.put(record.getTermIndex(), entry);
  }
  if (entry.hasConfigurationEntry()) {
    configEntries.add(record.getTermIndex());
  }
  totalSize += getEntrySize(entry);
  endIndex = entry.getIndex();
}
 

@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();
  final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData());

  final long index = entry.getIndex();
  final Double result;
  try(final AutoCloseableLock writeLock = writeLock()) {
    result = assignment.evaluate(variables);
    updateLastAppliedTermIndex(entry.getTerm(), index);
  }
  final Expression r = Expression.Utils.double2Expression(result);
  final CompletableFuture<Message> f = CompletableFuture.completedFuture(Expression.Utils.toMessage(r));

  final RaftPeerRole role = trx.getServerRole();
  if (role == RaftPeerRole.LEADER) {
    LOG.info("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  } else {
    LOG.debug("{}:{}-{}: {} = {}", role, getId(), index, assignment, r);
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}-{}: variables={}", getId(), index, variables);
  }
  return f;
}
 

static void assertLogEntries(List<LogEntryProto> entries, long expectedTerm, SimpleMessage... expectedMessages) {
  long logIndex = 0;
  Assert.assertEquals(expectedMessages.length, entries.size());
  for (int i = 0; i < expectedMessages.length; i++) {
    final LogEntryProto e = entries.get(i);
    Assert.assertTrue(e.getTerm() >= expectedTerm);
    if (e.getTerm() > expectedTerm) {
      expectedTerm = e.getTerm();
    }
    Assert.assertTrue(e.getIndex() > logIndex);
    logIndex = e.getIndex();
    Assert.assertArrayEquals(expectedMessages[i].getContent().toByteArray(),
        e.getStateMachineLogEntry().getLogData().toByteArray());
  }
}
 

static void assertLogEntries(List<LogEntryProto> entries, long expectedTerm, SimpleMessage... expectedMessages) {
  long logIndex = 0;
  Assert.assertEquals(expectedMessages.length, entries.size());
  for (int i = 0; i < expectedMessages.length; i++) {
    final LogEntryProto e = entries.get(i);
    Assert.assertTrue(e.getTerm() >= expectedTerm);
    if (e.getTerm() > expectedTerm) {
      expectedTerm = e.getTerm();
    }
    Assert.assertTrue(e.getIndex() > logIndex);
    logIndex = e.getIndex();
    Assert.assertArrayEquals(expectedMessages[i].getContent().toByteArray(),
        e.getStateMachineLogEntry().getLogData().toByteArray());
  }
}
 

private void append(boolean keepEntryInCache, LogEntryProto... entries) {
  Preconditions.assertTrue(entries != null && entries.length > 0);
  final long term = entries[0].getTerm();
  if (records.isEmpty()) {
    Preconditions.assertTrue(entries[0].getIndex() == startIndex,
        "gap between start index %s and first entry to append %s",
        startIndex, entries[0].getIndex());
  }
  for (LogEntryProto entry : entries) {
    // all these entries should be of the same term
    Preconditions.assertTrue(entry.getTerm() == term,
        "expected term:%s, term of the entry:%s", term, entry.getTerm());
    final LogRecord currentLast = getLastRecord();
    if (currentLast != null) {
      Preconditions.assertTrue(
          entry.getIndex() == currentLast.getTermIndex().getIndex() + 1,
          "gap between entries %s and %s", entry.getIndex(),
          currentLast.getTermIndex().getIndex());
    }

    final LogRecord record = new LogRecord(totalSize, entry);
    records.add(record);
    if (keepEntryInCache) {
      entryCache.put(record.getTermIndex(), entry);
    }
    if (entry.hasConfigurationEntry()) {
      configEntries.add(record.getTermIndex());
    }
    totalSize += getEntrySize(entry);
    endIndex = entry.getIndex();
  }
}
 

/**
 * Find the last valid entry index in the stream.
 * If there are invalid or corrupt entries in the middle of the stream,
 * scanEditLog will skip over them.
 *
 * This reads through the stream but does not close it.
 *
 * @param maxIndexToScan Maximum entry index to try to scan. The scan returns
 *                       after reading this or a higher index. The file
 *                       portion beyond this index is potentially being
 *                       updated.
 */
static LogValidation scanEditLog(LogInputStream in, long maxIndexToScan) {
  long lastPos = 0;
  long end = INVALID_LOG_INDEX;
  long numValid = 0;
  boolean hitError = false;
  while (end < maxIndexToScan) {
    long index;
    lastPos = in.getPosition();
    try {
      if (hitError) {
        LogEntryProto entry = in.nextEntry();
        index = entry != null ? entry.getIndex() : INVALID_LOG_INDEX;
        LOG.warn("After resync, position is " + in.getPosition());
      } else {
        index = in.scanNextEntry();
      }
      if (index == INVALID_LOG_INDEX) {
        break;
      } else {
        hitError = false;
      }
    } catch (Throwable t) {
      LOG.warn("Caught exception after scanning through {} ops from {}"
          + " while determining its valid length. Position was "
          + lastPos, numValid, in, t);
      hitError = true;
      continue;
    }
    if (end == INVALID_LOG_INDEX || index > end) {
      end = index;
    }
    numValid++;
  }
  return new LogValidation(lastPos, end, false);
}
 

public LogEntryProto nextEntry() throws IOException {
  if (state.isUnopened()) {
      try {
        init();
      } catch (Throwable e) {
        LOG.error("caught exception initializing " + this, e);
        throw IOUtils.asIOException(e);
      }
  }

  Preconditions.assertTrue(!state.isUnopened());
  if (state.isOpened()) {
      final LogEntryProto entry = reader.readEntry();
      if (entry != null) {
        long index = entry.getIndex();
        if (!isOpen() && index >= endIndex) {
          /**
           * The end index may be derived from the segment recovery
           * process. It is possible that we still have some uncleaned garbage
           * in the end. We should skip them.
           */
          long skipAmt = logFile.length() - reader.getPos();
          if (skipAmt > 0) {
            LOG.debug("skipping {} bytes at the end of log '{}': reached" +
                " entry {} out of {}", skipAmt, getName(), index, endIndex);
            reader.skipFully(skipAmt);
          }
        }
      }
      return entry;
  } else if (state.isClosed()) {
    return null;
  }
  throw new IOException("Failed to get next entry from " + this, state.getThrowable());
}
 

private CompletableFuture<Message> processAppendRequest(TransactionContext trx,
    LogServiceRequestProto logProto) {

  final LogEntryProto entry = trx.getLogEntry();
  AppendLogEntryRequestProto proto = logProto.getAppendRequest();
  final long index = entry.getIndex();
  long total = 0;
  Throwable t = verifyState(State.OPEN);
  if (t == null) {
    try (final AutoCloseableLock writeLock = writeLock()) {
        List<byte[]> entries = LogServiceProtoUtil.toListByteArray(proto.getDataList());
        for (byte[] bb : entries) {
          total += bb.length;
        }
        this.length += total;
        // TODO do we need this for other write request (close, sync)
        updateLastAppliedTermIndex(entry.getTerm(), index);
    }
  }
  List<Long> ids = new ArrayList<Long>();
  ids.add(index);
  final CompletableFuture<Message> f =
      CompletableFuture.completedFuture(
        Message.valueOf(LogServiceProtoUtil.toAppendLogReplyProto(ids, t).toByteString()));
  final RaftProtos.RaftPeerRole role = trx.getServerRole();
  LOG.debug("{}:{}-{}: {} new length {}", role, getId(), index, proto, length);
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}-{}: variables={}", getId(), index, length);
  }
  return f;
}
 

@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();

  final long index = entry.getIndex();
  updateLastAppliedTermIndex(entry.getTerm(), index);

  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final FileStoreRequestProto request;
  try {
    request = FileStoreRequestProto.parseFrom(smLog.getLogData());
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(index,
        "Failed to parse logData in" + smLog, e);
  }

  switch(request.getRequestCase()) {
    case DELETE:
      return delete(index, request.getDelete());
    case WRITEHEADER:
      return writeCommit(index, request.getWriteHeader(), smLog.getStateMachineEntry().getStateMachineData().size());
    case WRITE:
      // WRITE should not happen here since
      // startTransaction converts WRITE requests to WRITEHEADER requests.
    default:
      LOG.error(getId() + ": Unexpected request case " + request.getRequestCase());
      return FileStoreCommon.completeExceptionally(index,
          "Unexpected request case " + request.getRequestCase());
  }
}
 

@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  LogEntryProto entry = Objects.requireNonNull(trx.getLogEntry());
  put(entry);
  updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex());

  final SimpleMessage m = new SimpleMessage(entry.getIndex() + " OK");
  return collecting.collect(Collecting.Type.APPLY_TRANSACTION, m);
}
 

/**
 * Find the last valid entry index in the stream.
 * If there are invalid or corrupt entries in the middle of the stream,
 * scanEditLog will skip over them.
 *
 * This reads through the stream but does not close it.
 *
 * @param maxIndexToScan Maximum entry index to try to scan. The scan returns
 *                       after reading this or a higher index. The file
 *                       portion beyond this index is potentially being
 *                       updated.
 */
static LogValidation scanEditLog(SegmentedRaftLogInputStream in, long maxIndexToScan) {
  long lastPos = 0;
  long end = INVALID_LOG_INDEX;
  long numValid = 0;
  boolean hitError = false;
  while (end < maxIndexToScan) {
    long index;
    lastPos = in.getPosition();
    try {
      if (hitError) {
        LogEntryProto entry = in.nextEntry();
        index = entry != null ? entry.getIndex() : INVALID_LOG_INDEX;
        LOG.warn("After resync, position is " + in.getPosition());
      } else {
        index = in.scanNextEntry();
      }
      if (index == INVALID_LOG_INDEX) {
        break;
      } else {
        hitError = false;
      }
    } catch (Throwable t) {
      LOG.warn("Caught exception after scanning through {} ops from {}"
          + " while determining its valid length. Position was "
          + lastPos, numValid, in, t);
      hitError = true;
      continue;
    }
    if (end == INVALID_LOG_INDEX || index > end) {
      end = index;
    }
    numValid++;
  }
  return new LogValidation(lastPos, end, false);
}
 

public LogEntryProto nextEntry() throws IOException {
  if (state.isUnopened()) {
      try {
        init();
      } catch (Throwable e) {
        LOG.error("caught exception initializing " + this, e);
        throw IOUtils.asIOException(e);
      }
  }

  Preconditions.assertTrue(!state.isUnopened());
  if (state.isOpened()) {
      final LogEntryProto entry = reader.readEntry();
      if (entry != null) {
        long index = entry.getIndex();
        if (!isOpen() && index >= endIndex) {
          /**
           * The end index may be derived from the segment recovery
           * process. It is possible that we still have some uncleaned garbage
           * in the end. We should skip them.
           */
          long skipAmt = logFile.length() - reader.getPos();
          if (skipAmt > 0) {
            LOG.debug("skipping {} bytes at the end of log '{}': reached" +
                " entry {} out of {}", skipAmt, getName(), index, endIndex);
            reader.skipFully(skipAmt);
          }
        }
      }
      return entry;
  } else if (state.isClosed()) {
    return null;
  }
  throw new IOException("Failed to get next entry from " + this, state.getThrowable());
}
 

private CompletableFuture<Message> processAppendRequest(TransactionContext trx,
    LogServiceRequestProto logProto) {

  final LogEntryProto entry = trx.getLogEntry();
  AppendLogEntryRequestProto proto = logProto.getAppendRequest();
  final long index = entry.getIndex();
  long newSize = 0;
  Throwable t = verifyState(State.OPEN);
  final List<Long> ids = new ArrayList<Long>();
  if (t == null) {
    try (AutoCloseableLock writeLock = writeLock()) {
        List<byte[]> entries = LogServiceProtoUtil.toListByteArray(proto.getDataList());
        for (byte[] bb : entries) {
          ids.add(this.length);
          newSize += bb.length;
          this.length++;
        }
        this.dataRecordsSize += newSize;
        // TODO do we need this for other write request (close, sync)
        updateLastAppliedTermIndex(entry.getTerm(), index);
    }
  }
  final CompletableFuture<Message> f =
      CompletableFuture.completedFuture(
        Message.valueOf(LogServiceProtoUtil.toAppendLogReplyProto(ids, t).toByteString()));
  final RaftProtos.RaftPeerRole role = trx.getServerRole();
  if (LOG.isTraceEnabled()) {
    LOG.trace("{}:{}-{}: {} new length {}", role, getId(), index,
        TextFormat.shortDebugString(proto), dataRecordsSize);
  }
  return f;
}
 

@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
  final LogEntryProto entry = trx.getLogEntry();

  final long index = entry.getIndex();
  updateLastAppliedTermIndex(entry.getTerm(), index);

  final StateMachineLogEntryProto smLog = entry.getStateMachineLogEntry();
  final FileStoreRequestProto request;
  try {
    request = FileStoreRequestProto.parseFrom(smLog.getLogData());
  } catch (InvalidProtocolBufferException e) {
    return FileStoreCommon.completeExceptionally(index,
        "Failed to parse logData in" + smLog, e);
  }

  switch(request.getRequestCase()) {
    case DELETE:
      return delete(index, request.getDelete());
    case WRITEHEADER:
      return writeCommit(index, request.getWriteHeader(), smLog.getStateMachineEntry().getStateMachineData().size());
    case WRITE:
      // WRITE should not happen here since
      // startTransaction converts WRITE requests to WRITEHEADER requests.
    default:
      LOG.error(getId() + ": Unexpected request case " + request.getRequestCase());
      return FileStoreCommon.completeExceptionally(index,
          "Unexpected request case " + request.getRequestCase());
  }
}
 

public static StreamObserver<AppendEntriesRequestProto> appendEntries(
    RaftPeerId raftPeerId,
    StreamObserver<AppendEntriesReplyProto> responseHandler) {
  return new StreamObserver<AppendEntriesRequestProto>() {
    private long maxIndex = -1L;

    @Override
    public void onNext(AppendEntriesRequestProto value) {

      for (LogEntryProto entry : value.getEntriesList()) {
        if (entry.getIndex() > maxIndex) {
          maxIndex = entry.getIndex();
        }
      }

      long maxCommitted = value.getCommitInfosList()
          .stream()
          .mapToLong(CommitInfoProto::getCommitIndex)
          .max().orElseGet(() -> 0L);

      maxCommitted = Math.min(maxCommitted, maxIndex);

      AppendEntriesReplyProto response = AppendEntriesReplyProto.newBuilder()
          .setNextIndex(maxIndex + 1)
          .setFollowerCommit(maxCommitted)
          .setResult(AppendResult.SUCCESS)
          .setTerm(value.getLeaderTerm())
          .setMatchIndex(maxIndex)
          .setServerReply(RaftRpcReplyProto.newBuilder()
              .setSuccess(true)
              .setRequestorId(value.getServerRequest().getRequestorId())
              .setReplyId(raftPeerId.toByteString())
              .setCallId(value.getServerRequest().getCallId())
              .setRaftGroupId(value.getServerRequest().getRaftGroupId()))
          .build();
      maxCommitted = Math.min(value.getLeaderCommit(), maxIndex);
      addLatency();
      responseHandler.onNext(response);
    }

    @Override
    public void onError(Throwable t) {

    }

    @Override
    public void onCompleted() {

    }
  };
}
 

/**
 * Scan and validate a log entry.
 * @return the index of the log entry
 */
long scanEntry() throws IOException {
  LogEntryProto entry = decodeEntry();
  return entry != null ? entry.getIndex() : RaftServerConstants.INVALID_LOG_INDEX;
}
 

/**
 * Scan and validate a log entry.
 * @return the index of the log entry
 */
long scanEntry() throws IOException {
  LogEntryProto entry = decodeEntry();
  return entry != null ? entry.getIndex() : RaftServerConstants.INVALID_LOG_INDEX;
}