org.apache.ratis.util.Preconditions Java Examples

/**
 * Validate the term and index of entry w.r.t RaftLog
 */
protected void validateLogEntry(LogEntryProto entry) {
  if (entry.hasMetadataEntry()) {
    return;
  }
  long latestSnapshotIndex = getSnapshotIndex();
  TermIndex lastTermIndex = getLastEntryTermIndex();
  if (lastTermIndex != null) {
    long lastIndex = lastTermIndex.getIndex() > latestSnapshotIndex ?
        lastTermIndex.getIndex() : latestSnapshotIndex;
    Preconditions.assertTrue(entry.getTerm() >= lastTermIndex.getTerm(),
        "Entry term less than RaftLog's last term: %d, entry: %s", lastTermIndex.getTerm(), entry);
    Preconditions.assertTrue(entry.getIndex() == lastIndex + 1,
        "Difference between entry index and RaftLog's last index %d (or snapshot index %d) " +
            "is greater than 1, entry: %s",
        lastTermIndex.getIndex(), latestSnapshotIndex, entry);
  } else {
    Preconditions.assertTrue(entry.getIndex() == latestSnapshotIndex + 1,
        "Difference between entry index and RaftLog's latest snapshot index %d is greater than 1 " +
            "and in between log entries are not present, entry: %s",
        latestSnapshotIndex, entry);
  }
}
 

boolean tryAcquire(int... permits) {
  Preconditions.assertTrue(permits.length == resources.size(),
      () -> "items.length = " + permits.length + " != resources.size() = "
          + resources.size());
  int i = 0;
  // try acquiring all resources
  for (; i < permits.length; i++) {
    if (!resources.get(i).tryAcquire(permits[i])) {
      break;
    }
  }
  if (i == permits.length) {
    return true; // successfully acquired all resources
  }

  // failed at i, releasing all previous resources
  for(i--; i >= 0; i--) {
    resources.get(i).release(permits[i]);
  }
  return false;
}
 

private CompletableFuture<RaftClientReply> groupAddAsync(GroupManagementRequest request, RaftGroup newGroup) {
  if (!request.getRaftGroupId().equals(newGroup.getGroupId())) {
    return JavaUtils.completeExceptionally(new GroupMismatchException(
        getId() + ": Request group id (" + request.getRaftGroupId() + ") does not match the new group " + newGroup));
  }
  return impls.addNew(newGroup)
      .thenApplyAsync(newImpl -> {
        LOG.debug("{}: newImpl = {}", getId(), newImpl);
        final boolean started = newImpl.start();
        Preconditions.assertTrue(started, () -> getId()+ ": failed to start a new impl: " + newImpl);
        return new RaftClientReply(request, newImpl.getCommitInfos());
      })
      .whenComplete((_1, throwable) -> {
        if (throwable != null) {
          impls.remove(newGroup.getGroupId());
          LOG.warn(getId() + ": Failed groupAdd* " + request, throwable);
        }
      });
}
 

/**
 * Force close the container replica(s) with highest sequence Id.
 *
 * <p>
 *   Note: We should force close the container only if >50% (quorum)
 *   of replicas with unique originNodeId are in QUASI_CLOSED state.
 * </p>
 *
 * @param container ContainerInfo
 * @param replicas Set of ContainerReplicas
 */
private void forceCloseContainer(final ContainerInfo container,
                                 final Set<ContainerReplica> replicas) {
  Preconditions.assertTrue(container.getState() ==
      LifeCycleState.QUASI_CLOSED);

  final List<ContainerReplica> quasiClosedReplicas = replicas.stream()
      .filter(r -> r.getState() == State.QUASI_CLOSED)
      .collect(Collectors.toList());

  final Long sequenceId = quasiClosedReplicas.stream()
      .map(ContainerReplica::getSequenceId)
      .max(Long::compare)
      .orElse(-1L);

  LOG.info("Force closing container {} with BCSID {}," +
      " which is in QUASI_CLOSED state.",
      container.containerID(), sequenceId);

  quasiClosedReplicas.stream()
      .filter(r -> sequenceId != -1L)
      .filter(replica -> replica.getSequenceId().equals(sequenceId))
      .forEach(replica -> sendCloseCommand(
          container, replica.getDatanodeDetails(), true));
}
 

/**
 * Run the given operation sequentially.
 * This method can be invoked by different threads but only one thread at any given time is allowed.
 * The same thread can call this methed multiple times.
 *
 * @throws IllegalStateException if this runner is already running another operation.
 */
<OUTPUT, THROWABLE extends Throwable> OUTPUT runSequentially(
    CheckedSupplier<OUTPUT, THROWABLE> operation) throws THROWABLE {
  final Thread current = Thread.currentThread();
  // update only if the runner is null
  final Thread previous = runner.getAndUpdate(prev -> prev != null? prev: current);
  if (previous == null) {
    // The current thread becomes the runner.
    try {
      return operation.get();
    } finally {
      // prev is expected to be current
      final Thread got = runner.getAndUpdate(prev -> prev != current? prev: null);
      Preconditions.assertTrue(got == current,
          () -> name + ": Unexpected runner " + got + " != " + current);
    }
  } else if (previous == current) {
    // The current thread is already the runner.
    return operation.get();
  } else {
    throw new IllegalStateException(
        name + ": Already running a method by " + previous + ", current=" + current);
  }
}
 

/**
 * Read header from the log file:
 *
 * (1) The header in file is verified successfully.
 *     Then, return true.
 *
 * (2) The header in file is partially written.
 *     Then, return false.
 *
 * (3) The header in file is corrupted or there is some other {@link IOException}.
 *     Then, throw an exception.
 */
boolean verifyHeader() throws IOException {
  final int headerLength = SegmentedRaftLogFormat.getHeaderLength();
  final int readLength = in.read(temp, 0, headerLength);
  Preconditions.assertTrue(readLength <= headerLength);
  final int matchLength = SegmentedRaftLogFormat.matchHeader(temp, 0, readLength);
  Preconditions.assertTrue(matchLength <= readLength);

  if (readLength == headerLength && matchLength == readLength) {
    // The header is matched successfully
    return true;
  } else if (SegmentedRaftLogFormat.isTerminator(temp, matchLength, readLength - matchLength)) {
    // The header is partially written
    return false;
  }
  // The header is corrupted
  throw new CorruptedFileException(file, "Log header mismatched: expected header length="
      + SegmentedRaftLogFormat.getHeaderLength() + ", read length=" + readLength + ", match length=" + matchLength
      + ", header in file=" + StringUtils.bytes2HexString(temp, 0, readLength)
      + ", expected header=" + SegmentedRaftLogFormat.applyHeaderTo(StringUtils::bytes2HexString));
}
 

synchronized CompletableFuture<RaftServerImpl> addNew(RaftGroup group) {
  if (isClosed) {
    return JavaUtils.completeExceptionally(new AlreadyClosedException(
        getId() + ": Failed to add " + group + " since the server is already closed"));
  }
  if (containsGroup(group.getGroupId())) {
    return JavaUtils.completeExceptionally(new AlreadyExistsException(
        getId() + ": Failed to add " + group + " since the group already exists in the map."));
  }
  final RaftGroupId groupId = group.getGroupId();
  final CompletableFuture<RaftServerImpl> newImpl = newRaftServerImpl(group);
  final CompletableFuture<RaftServerImpl> previous = map.put(groupId, newImpl);
  Preconditions.assertNull(previous, "previous");
  LOG.info("{}: addNew {} returns {}", getId(), group, toString(groupId, newImpl));
  return newImpl;
}
 

private CompletableFuture<RaftClientReply> groupAddAsync(GroupManagementRequest request, RaftGroup newGroup) {
  if (!request.getRaftGroupId().equals(newGroup.getGroupId())) {
    return JavaUtils.completeExceptionally(new GroupMismatchException(
        getId() + ": Request group id (" + request.getRaftGroupId() + ") does not match the new group " + newGroup));
  }
  return impls.addNew(newGroup)
      .thenApplyAsync(newImpl -> {
        LOG.debug("{}: newImpl = {}", getId(), newImpl);
        final boolean started = newImpl.start();
        Preconditions.assertTrue(started, () -> getId()+ ": failed to start a new impl: " + newImpl);
        return new RaftClientReply(request, newImpl.getCommitInfos());
      }, implExecutor)
      .whenComplete((raftClientReply, throwable) -> {
        if (throwable != null) {
          if (!(throwable.getCause() instanceof AlreadyExistsException)) {
            impls.remove(newGroup.getGroupId());
            LOG.warn(getId() + ": Failed groupAdd* " + request, throwable);
          } else {
            if (LOG.isDebugEnabled()) {
              LOG.debug(getId() + ": Failed groupAdd* " + request, throwable);
            }
          }
        }
      });
}
 

public static void runTestPythagorean(
    RaftClient client, int start, int count) throws IOException {
  Preconditions.assertTrue(count > 0, () -> "count = " + count + " <= 0");
  Preconditions.assertTrue(start >= 2, () -> "start = " + start + " < 2");

  final Variable a = new Variable("a");
  final Variable b = new Variable("b");
  final Variable c = new Variable("c");
  final Expression pythagorean = SQRT.apply(ADD.apply(SQUARE.apply(a), SQUARE.apply(b)));

  final int end = start + 2*count;
  for(int n = (start & 1) == 0? start + 1: start; n < end; n += 2) {
    int n2 = n*n;
    int half_n2 = n2/2;

    assign(client, a, n);
    assign(client, b, half_n2);
    assign(client, c, pythagorean, (double)half_n2 + 1);

    assignNull(client, a);
    assignNull(client, b);
    assignNull(client, c);
  }
}
 

public RaftStorage(File dir, RaftServerConstants.StartupOption option, CorruptionPolicy logCorruptionPolicy)
    throws IOException {
  this.storageDir = new RaftStorageDirectory(dir);
  if (option == RaftServerConstants.StartupOption.FORMAT) {
    if (storageDir.analyzeStorage(false) == StorageState.NON_EXISTENT) {
      throw new IOException("Cannot format " + storageDir);
    }
    storageDir.lock();
    format();
    state = storageDir.analyzeStorage(false);
    Preconditions.assertTrue(state == StorageState.NORMAL);
  } else {
    state = analyzeAndRecoverStorage(true); // metaFile is initialized here
    if (state != StorageState.NORMAL) {
      storageDir.unlock();
      throw new IOException("Cannot load " + storageDir
          + ". Its state: " + state);
    }
  }
  this.logCorruptionPolicy = logCorruptionPolicy;
}
 

private StorageState analyzeAndRecoverStorage(boolean toLock)
    throws IOException {
  StorageState storageState = storageDir.analyzeStorage(toLock);
  if (storageState == StorageState.NORMAL) {
    metaFile = new MetaFile(storageDir.getMetaFile());
    Preconditions.assertTrue(metaFile.exists(),
        () -> "Meta file " + metaFile + " does not exists.");
    metaFile.readFile();
    // Existence of raft-meta.tmp means the change of votedFor/term has not
    // been committed. Thus we should delete the tmp file.
    cleanMetaTmpFile();
    return StorageState.NORMAL;
  } else if (storageState == StorageState.NOT_FORMATTED &&
      storageDir.isCurrentEmpty()) {
    format();
    return StorageState.NORMAL;
  } else {
    return storageState;
  }
}
 

protected boolean updateLastAppliedTermIndex(long term, long index) {
  final TermIndex newTI = TermIndex.newTermIndex(term, index);
  final TermIndex oldTI = lastAppliedTermIndex.getAndSet(newTI);
  if (!newTI.equals(oldTI)) {
    LOG.trace("{}: update lastAppliedTermIndex from {} to {}", getId(), oldTI, newTI);
    if (oldTI != null) {
      Preconditions.assertTrue(newTI.compareTo(oldTI) >= 0,
          () -> getId() + ": Failed updateLastAppliedTermIndex: newTI = "
              + newTI + " < oldTI = " + oldTI);
    }
    return true;
  }

  synchronized (transactionFutures) {
    for(long i; !transactionFutures.isEmpty() && (i = transactionFutures.firstKey()) <= index; ) {
      transactionFutures.remove(i).complete(null);
    }
  }
  return false;
}
 

public TransactionContext build() {
  Objects.requireNonNull(serverRole, "serverRole == null");
  Objects.requireNonNull(stateMachine, "stateMachine == null");
  if (clientRequest != null) {
    Preconditions.assertTrue(serverRole == RaftPeerRole.LEADER,
        () -> "serverRole MUST be LEADER since clientRequest != null, serverRole is " + serverRole);
    Preconditions.assertNull(logEntry, () -> "logEntry MUST be null since clientRequest != null");
    if (stateMachineLogEntry == null) {
      stateMachineLogEntry = ServerProtoUtils.toStateMachineLogEntryProto(clientRequest, logData, stateMachineData);
    }
    return new TransactionContextImpl(stateMachine, clientRequest, stateMachineLogEntry, stateMachineContext);
  } else {
    Objects.requireNonNull(logEntry, "logEntry MUST NOT be null since clientRequest == null");
    Preconditions.assertTrue(logEntry.hasStateMachineLogEntry(),
        () -> "Unexpected logEntry: stateMachineLogEntry not found, logEntry=" + logEntry);
    return new TransactionContextImpl(serverRole, stateMachine, logEntry);
  }
}
 

private void assertSegment(long expectedStart, int expectedEntryCount, boolean corrupted, long expectedEnd) {
  Preconditions.assertSame(expectedStart, getStartIndex(), "Segment start index");
  Preconditions.assertSame(expectedEntryCount, records.size(), "Number of records");

  final long expectedLastIndex = expectedStart + expectedEntryCount - 1;
  Preconditions.assertSame(expectedLastIndex, getEndIndex(), "Segment end index");

  final LogRecord last = getLastRecord();
  if (last != null) {
    Preconditions.assertSame(expectedLastIndex, last.getTermIndex().getIndex(), "Index at the last record");
    Preconditions.assertSame(expectedStart, records.get(0).getTermIndex().getIndex(), "Index at the first record");
  }
  if (!corrupted) {
    Preconditions.assertSame(expectedEnd, expectedLastIndex, "End/last Index");
  }
}
 

private StorageState analyzeAndRecoverStorage(boolean toLock)
    throws IOException {
  StorageState storageState = storageDir.analyzeStorage(toLock);
  if (storageState == StorageState.NORMAL) {
    metaFile = new MetaFile(storageDir.getMetaFile());
    Preconditions.assertTrue(metaFile.exists(),
        () -> "Meta file " + metaFile + " does not exists.");
    metaFile.readFile();
    // Existence of raft-meta.tmp means the change of votedFor/term has not
    // been committed. Thus we should delete the tmp file.
    cleanMetaTmpFile();
    return StorageState.NORMAL;
  } else if (storageState == StorageState.NOT_FORMATTED &&
      storageDir.isCurrentEmpty()) {
    format();
    return StorageState.NORMAL;
  } else {
    return storageState;
  }
}
 

EntryIterator(long start) {
  this.nextIndex = start;
  segmentIndex = closedSegments.binarySearch(nextIndex);
  if (segmentIndex >= 0) {
    currentSegment = closedSegments.get(segmentIndex);
  } else {
    segmentIndex = -segmentIndex - 1;
    if (segmentIndex == closedSegments.size()) {
      currentSegment = openSegment;
    } else {
      // the start index is smaller than the first closed segment's start
      // index. We no longer keep the log entry (because of the snapshot) or
      // the start index is invalid.
      Preconditions.assertTrue(segmentIndex == 0);
      throw new IndexOutOfBoundsException();
    }
  }
}
 

static void assertCorruptedLogHeader(RaftPeerId id, File openLogFile, int partialLength,
    MiniRaftCluster cluster, Logger LOG) throws Exception {
  Preconditions.assertTrue(partialLength < SegmentedRaftLogFormat.getHeaderLength());
  try(final RandomAccessFile raf = new RandomAccessFile(openLogFile, "rw")) {
    SegmentedRaftLogFormat.applyHeaderTo(header -> {
      LOG.info("header    = {}", StringUtils.bytes2HexString(header));
      final byte[] corrupted = new byte[header.length];
      System.arraycopy(header, 0, corrupted, 0, partialLength);
      LOG.info("corrupted = {}", StringUtils.bytes2HexString(corrupted));
      raf.write(corrupted);
      return null;
    });
  }
  final RaftServerImpl server = cluster.restartServer(id, false);
  server.getProxy().close();
}
 

static LogSegment loadSegment(RaftStorage storage, File file,
    long start, long end, boolean isOpen,
    boolean keepEntryInCache, Consumer<LogEntryProto> logConsumer)
    throws IOException {
  final LogSegment segment = isOpen ?
      LogSegment.newOpenSegment(storage, start) :
      LogSegment.newCloseSegment(storage, start, end);

  final int entryCount = readSegmentFile(file, start, end, isOpen, entry -> {
    segment.append(keepEntryInCache || isOpen, entry);
    if (logConsumer != null) {
      logConsumer.accept(entry);
    }
  });
  LOG.info("Successfully read {} entries from segment file {}", entryCount, file);

  if (entryCount == 0) {
    // The segment does not have any entries, delete the file.
    FileUtils.deleteFile(file);
    return null;
  } else if (file.length() > segment.getTotalSize()) {
    // The segment has extra padding, truncate it.
    FileUtils.truncateFile(file, segment.getTotalSize());
  }

  Preconditions.assertTrue(start == segment.getStartIndex());
  if (!segment.records.isEmpty()) {
    Preconditions.assertTrue(start == segment.records.get(0).getTermIndex().getIndex());
  }
  if (!isOpen) {
    Preconditions.assertTrue(segment.getEndIndex() == end);
  }
  return segment;
}
 

@Override
public LogEntryProto initLogEntry(long term, long index) {
  Preconditions.assertTrue(serverRole == RaftPeerRole.LEADER);
  Preconditions.assertNull(logEntry, "logEntry");
  Objects.requireNonNull(smLogEntryProto, "smLogEntryProto == null");
  return logEntry = ServerProtoUtils.toLogEntryProto(smLogEntryProto, term, index);
}
 

private void validateAdding(LogSegment segment) {
  final LogSegment lastClosed = closedSegments.getLast();
  if (lastClosed != null) {
    Preconditions.assertTrue(!lastClosed.isOpen());
    Preconditions.assertTrue(lastClosed.getEndIndex() + 1 == segment.getStartIndex());
  }
}
 

TruncateIndices computeTruncateIndices(Consumer<TermIndex> failClientRequest, LogEntryProto... entries) {
  int arrayIndex = 0;
  long truncateIndex = -1;

  try(AutoCloseableLock readLock = closedSegments.readLock()) {
    final Iterator<TermIndex> i = iterator(entries[0].getIndex());
    for(; i.hasNext() && arrayIndex < entries.length; arrayIndex++) {
      final TermIndex storedEntry = i.next();
      Preconditions.assertTrue(storedEntry.getIndex() == entries[arrayIndex].getIndex(),
          "The stored entry's index %s is not consistent with the received entries[%s]'s index %s",
          storedEntry.getIndex(), arrayIndex, entries[arrayIndex].getIndex());

      if (storedEntry.getTerm() != entries[arrayIndex].getTerm()) {
        // we should truncate from the storedEntry's arrayIndex
        truncateIndex = storedEntry.getIndex();
        if (LOG.isTraceEnabled()) {
          LOG.trace("{}: truncate to {}, arrayIndex={}, ti={}, storedEntry={}, entries={}",
              name, truncateIndex, arrayIndex,
              ServerProtoUtils.toTermIndex(entries[arrayIndex]), storedEntry,
              ServerProtoUtils.toString(entries));
        }

        // fail all requests starting at truncateIndex
        failClientRequest.accept(storedEntry);
        for(; i.hasNext(); ) {
          failClientRequest.accept(i.next());
        }
        break;
      }
    }
  }
  return new TruncateIndices(arrayIndex, truncateIndex);
}
 

/**
 * Validate the term and index of entry w.r.t RaftLog
 */
void validateLogEntry(LogEntryProto entry) {
  if (entry.hasMetadataEntry()) {
    return;
  }
  TermIndex lastTermIndex = getLastEntryTermIndex();
  if (lastTermIndex != null) {
    Preconditions.assertTrue(entry.getTerm() >= lastTermIndex.getTerm(),
        "Entry term less than RaftLog's last term: %d, entry: %s", lastTermIndex.getTerm(), entry);
    Preconditions.assertTrue(entry.getIndex() == lastTermIndex.getIndex() + 1,
        "Difference between entry index and RaftLog's last index %d greater than 1, entry: %s", lastTermIndex.getIndex(), entry);
  }
}
 

public void setConfiguration(RaftPeer... peers) throws IOException {
  try(RaftClient client = createClient()) {
    LOG.info("Start changing the configuration: {}", Arrays.asList(peers));
    final RaftClientReply reply = client.setConfiguration(peers);
    Preconditions.assertTrue(reply.isSuccess());
  }
}
 

public static int readSegmentFile(File file, long start, long end,
    boolean isOpen, CorruptionPolicy corruptionPolicy,
    RaftLogMetrics raftLogMetrics, Consumer<LogEntryProto> entryConsumer) throws
    IOException {
  int count = 0;
  try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file, start, end, isOpen, raftLogMetrics)) {
    for(LogEntryProto prev = null, next; (next = in.nextEntry()) != null; prev = next) {
      if (prev != null) {
        Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1,
            "gap between entry %s and entry %s", prev, next);
      }

      if (entryConsumer != null) {
        entryConsumer.accept(next);
      }
      count++;
    }
  } catch (IOException ioe) {
    switch (corruptionPolicy) {
      case EXCEPTION: throw ioe;
      case WARN_AND_RETURN:
        LOG.warn("Failed to read segment file {} (start={}, end={}, isOpen? {}): only {} entries read successfully",
            file, start, end, isOpen, count, ioe);
        break;
      default:
        throw new IllegalStateException("Unexpected enum value: " + corruptionPolicy
            + ", class=" + CorruptionPolicy.class);
    }
  }

  return count;
}
 

public Builder setLogEntryIndex(long logEntryIndex) {
  Preconditions.assertTrue(
      logEntryIndex != RaftServerConstants.INVALID_LOG_INDEX);
  Preconditions.assertTrue(
      this.logEntryIndex == RaftServerConstants.INVALID_LOG_INDEX,
      "logEntryIndex is already set.");
  this.logEntryIndex = logEntryIndex;
  return this;
}
 

@Override
public LogEntryProto initLogEntry(long term, long index) {
  Preconditions.assertTrue(serverRole == RaftPeerRole.LEADER);
  Preconditions.assertNull(logEntry, "logEntry");
  Objects.requireNonNull(smLogEntryProto, "smLogEntryProto == null");
  return logEntry = ServerProtoUtils.toLogEntryProto(smLogEntryProto, term, index);
}
 

public void sendReply(REQUEST request, REPLY reply, IOException ioe)
    throws IOException {
  if (reply != null) {
    Preconditions.assertTrue(
        request.getRequestorId().equals(reply.getRequestorId()));
    Preconditions.assertTrue(
        request.getReplierId().equals(reply.getReplierId()));
  }
  simulateLatency();
  final String qid = request.getReplierId();
  EventQueue<REQUEST, REPLY> q = queues.get(qid);
  if (q != null) {
    q.reply(request, reply, ioe);
  }
}
 

static CompletableFuture<ByteString> sendAsync(
    ByteString request, Function<Message, CompletableFuture<RaftClientReply>> sendFunction)
    {
  return sendFunction.apply(() -> request
  ).thenApply(reply -> {
    final StateMachineException sme = reply.getStateMachineException();
    if (sme != null) {
      throw new CompletionException("Failed to send request " + request, sme);
    }
    Preconditions.assertTrue(reply.isSuccess(), () -> "Failed " + request + ", reply=" + reply);
    return reply.getMessage().getContent();
  });
}
 

static String extractString(byte[] buf, int offset) {
  Preconditions.assertTrue(buf[offset] == Type.VARIABLE.byteValue());
  final int length = buf[offset + 1];
  final byte[] stringBytes = new byte[length];
  System.arraycopy(buf, offset + 2, stringBytes, 0, length);
  return new String(stringBytes, AssignmentMessage.UTF8);
}
 

/**
 * Remove records from the given index (inclusive)
 */
synchronized void truncate(long fromIndex) {
  Preconditions.assertTrue(fromIndex >= startIndex && fromIndex <= endIndex);
  for (long index = endIndex; index >= fromIndex; index--) {
    LogRecord removed = records.remove(Math.toIntExact(index - startIndex));
    entryCache.remove(removed.getTermIndex());
    configEntries.remove(removed.getTermIndex());
    totalSize = removed.offset;
  }
  isOpen = false;
  this.endIndex = fromIndex - 1;
}