io.fabric8.kubernetes.api.model.ContainerStateTerminated Java Examples

@Override
public void eventReceived(io.fabric8.kubernetes.client.Watcher.Action action, Pod pod) {
  log.info("Event received for pod: {}, action: {}", podName, action);
  PodStatus status = pod.getStatus();
  List<ContainerStatus> containerStatuses = status.getContainerStatuses();
  if (!containerStatuses.isEmpty()) {
    ContainerStatus containerStatus = containerStatuses.get(0);
    ContainerState state = containerStatus.getState();
    ContainerStateTerminated terminated = state.getTerminated();
    if (terminated != null) {
      Integer exitCode = terminated.getExitCode();
      log.info("Container exit code for pod {}: {}", podName, exitCode);
      if (exitCode == 0) {
        exitCodeFuture.complete(0);
      } else {
        exitCodeFuture.completeExceptionally(new RuntimeException("Completed with non zero exit code: " + exitCode));
      }
      resource.delete();
      watch.close();
    } else {
      log.warn("ContainerStateTerminated was null for pod: {}, action {}", podName, action);
    }
  } else {
    log.warn("ContainerStatus list was empty for pod: {}, action {}", podName, action);
  }
}
 

@Override
public void onEvent(@NonNull Action action, @NonNull KubernetesSlave node, @NonNull Pod pod) throws IOException, InterruptedException {
    if (action != Action.MODIFIED) {
        return;
    }
    List<ContainerStatus> terminatedContainers = PodUtils.getTerminatedContainers(pod);
    if (!terminatedContainers.isEmpty()) {
        String ns = pod.getMetadata().getNamespace();
        String name = pod.getMetadata().getName();
        TaskListener runListener = node.getTemplate().getListener();
        terminatedContainers.forEach(c -> {
            ContainerStateTerminated t = c.getState().getTerminated();
            LOGGER.info(() -> ns + "/" + name + " Container " + c.getName() + " was just terminated, so removing the corresponding Jenkins agent");
            runListener.getLogger().printf("%s/%s Container %s was terminated (Exit Code: %d, Reason: %s)%n", ns, name, c.getName(), t.getExitCode(), t.getReason());
        });
        node.terminate();
    }
}
 

private ContainerState terminatedStateWithExitCode(int exitCode) {
  ContainerStateTerminated terminated = new ContainerStateTerminated();
  terminated.setExitCode(exitCode);
  ContainerState state = new ContainerState();
  state.setTerminated(terminated);
  return state;
}
 

private static Optional<Event> isInErrorState(WorkflowInstance workflowInstance, Pod pod,
                                              Optional<ContainerStatus> mainContainerStatusOpt) {
  final PodStatus status = pod.getStatus();
  final String phase = status.getPhase();

  if ("NodeLost".equals(pod.getStatus().getReason())) {
    return Optional.of(Event.runError(workflowInstance, "Lost node running pod"));
  }

  switch (phase) {
    case "Pending":
      // check if one or more docker contains failed to pull their image, a possible silent error
      return mainContainerStatusOpt
          .flatMap(KubernetesPodEventTranslator::imageError)
          .map(msg -> Event.runError(workflowInstance, msg));

    case "Succeeded":
    case "Failed":
      if (mainContainerStatusOpt.isEmpty()) {
        return Optional.of(Event.runError(workflowInstance, "Could not find our container in pod"));
      }

      final ContainerStatus containerStatus = mainContainerStatusOpt.get();
      final ContainerStateTerminated terminated = containerStatus.getState().getTerminated();
      if (terminated == null) {
        return Optional.of(Event.runError(workflowInstance, "Unexpected null terminated status"));
      }
      return Optional.empty();

    case "Unknown":
      return Optional.of(Event.runError(workflowInstance, "Pod entered Unknown phase"));

    default:
      return Optional.empty();
  }
}
 

private boolean isNonDeletePeriodExpired(ContainerStatus cs) {
  final ContainerStateTerminated t = cs.getState().getTerminated();
  if (t.getFinishedAt() == null) {
    return true;
  }
  final Instant finishedAt;
  try {
    finishedAt = Instant.parse(t.getFinishedAt());
  } catch (DateTimeParseException e) {
    LOG.warn("Failed to parse container state terminated finishedAt: '{}'", t.getFinishedAt(), e);
    return true;
  }
  final Instant deadline = time.get().minus(podDeletionDelay);
  return finishedAt.isBefore(deadline);
}
 

private static Optional<Integer> getExitCodeIfValid(String workflowInstance,
                                                    Pod pod,
                                                    ContainerStatus status,
                                                    Stats stats) {
  final ContainerStateTerminated terminated = status.getState().getTerminated();

  // Check termination log exit code, if available
  if (Optional.ofNullable(pod.getMetadata().getAnnotations())
      .map(annotations -> "true".equals(annotations.get(DOCKER_TERMINATION_LOGGING_ANNOTATION)))
      .orElse(false)) {
    if (terminated.getMessage() == null) {
      LOG.warn("Missing termination log message for workflow instance {} container {}",
               workflowInstance, status.getContainerID());
      stats.recordTerminationLogMissing();
    } else {
      try {
        final TerminationLogMessage message = Json.deserialize(
            ByteString.encodeUtf8(terminated.getMessage()), TerminationLogMessage.class);

        if (!Objects.equals(message.exitCode, terminated.getExitCode())) {
          LOG.warn("Exit code mismatch for workflow instance {} container {}. Container exit code: {}. "
              + "Termination log exit code: {}",
              workflowInstance, status.getContainerID(), terminated.getExitCode(),
              message.exitCode);
          stats.recordExitCodeMismatch();
        }

        if (terminated.getExitCode() != null && message.exitCode == 0) {
          // If we have a non-zero container exit code but a zero termination log exit code,
          // return the container exit code to indicate failure. This guards against jobs that
          // incorrectly write a successful exit code to the termination log _before_ running
          // the actual job, which then fails. We could then still incorrectly get a zero exit
          // code from docker, but there is not a lot we can do about that.
          return Optional.of(terminated.getExitCode());
        } else {
          return Optional.of(message.exitCode);
        }
      } catch (IOException e) {
        stats.recordTerminationLogInvalid();
        LOG.warn("Unexpected termination log message for workflow instance {} container {}",
            workflowInstance, status.getContainerID(), e);
      }
    }

    // If there's no termination log exit code, fall back to k8s exit code if it is not zero.
    // Rationale: It is important for users to be able to get the exit code of the container to be
    // able to debug failures, but at the same time we must be careful about using the use the k8s
    // exit code when checking whether the execution was successful as dockerd some times returns
    // incorrect exit codes.
    // TODO: consider separating execution status and debugging info in the "terminate" event.
    if (terminated.getExitCode() != null && terminated.getExitCode() != 0) {
      return Optional.of(terminated.getExitCode());
    } else {
      return Optional.empty();
    }
  }

  // No termination log expected, use k8s exit code
  if (terminated.getExitCode() == null) {
    LOG.warn("Missing exit code for workflow instance {} container {}", workflowInstance,
             status.getContainerID());
    return Optional.empty();
  } else {
    // there are cases k8s marks the pod failed but with exitCode 0
    if ("Failed".equals(pod.getStatus().getPhase()) && terminated.getExitCode() == 0) {
      return Optional.empty();
    }
    return Optional.of(terminated.getExitCode());
  }
}
 

static ContainerState terminatedContainerState(Integer exitCode, String message) {
  return new ContainerState(null, new ContainerStateTerminated("", exitCode, "", message, "", 0, ""), null);
}