io.fabric8.kubernetes.api.model.ContainerStatus Java Examples

@Override
public void onEvent(@NonNull Action action, @NonNull KubernetesSlave node, @NonNull Pod pod) throws IOException, InterruptedException {
    if (action != Action.MODIFIED) {
        return;
    }
    List<ContainerStatus> terminatedContainers = PodUtils.getTerminatedContainers(pod);
    if (!terminatedContainers.isEmpty()) {
        String ns = pod.getMetadata().getNamespace();
        String name = pod.getMetadata().getName();
        TaskListener runListener = node.getTemplate().getListener();
        terminatedContainers.forEach(c -> {
            ContainerStateTerminated t = c.getState().getTerminated();
            LOGGER.info(() -> ns + "/" + name + " Container " + c.getName() + " was just terminated, so removing the corresponding Jenkins agent");
            runListener.getLogger().printf("%s/%s Container %s was terminated (Exit Code: %d, Reason: %s)%n", ns, name, c.getName(), t.getExitCode(), t.getReason());
        });
        node.terminate();
    }
}
 

@VisibleForTesting
boolean shouldDeletePodWithRunState(WorkflowInstance workflowInstance, Pod pod, RunState runState) {
  // Precondition: The run states were fetched after the pods
  final Optional<ContainerStatus> containerStatus = getMainContainerStatus(pod);
  if (containerStatus.isEmpty()) {
    // Do nothing, let the RunState time out if the pod fails to start. Then shouldDeletePodWithoutRunState will
    // mark it for deletion.
    // Note: It is natural for pods to not have any container statuses for a while after creation.
    return false;
  }
  if (wantsPod(runState)) {
    // Do not delete the pod if the workflow instance still wants it, e.g. it is still RUNNING.
    return false;
  }
  if (isTerminated(containerStatus.get())) {
    return shouldDeletePodIfNonDeletePeriodExpired(workflowInstance, pod);
  } else if (imageError(containerStatus.get()).isPresent()) {
    return shouldDeletePod(workflowInstance, pod, "Pull image error");
  }
  return false;
}
 

public static boolean isPodReady(final Pod pod, final boolean doLog) {

        if (!"Running".equals(pod.getStatus().getPhase())) {
            if (doLog) {
                log.info("POD {} in status : {}", pod.getMetadata().getName(), pod.getStatus().getPhase());
            }
            return false;
        }

        var nonReadyContainers = pod.getStatus().getContainerStatuses().stream()
                .filter(cs -> !Boolean.TRUE.equals(cs.getReady()))
                .map(ContainerStatus::getName)
                .collect(Collectors.toList());

        if (!nonReadyContainers.isEmpty()) {
            if (doLog) {
                log.info("POD {} non-ready containers: [{}]", pod.getMetadata().getName(), String.join(", ", nonReadyContainers));
            }
            return false;
        }

        return true;
    }
 

static Optional<String> imageError(ContainerStatus cs) {
  return Optional.ofNullable(cs.getState().getWaiting()).flatMap(waiting ->
      Optional.ofNullable(waiting.getReason()).flatMap(reason -> {
        var message = Optional.ofNullable(waiting.getMessage()).orElse("");
        switch (reason) {
          // https://github.com/kubernetes/kubernetes/blob/8327e433590f9e867b1e31a4dc32316685695729/pkg/kubelet/images/types.go#L26
          case "ImageInspectError":
          case "PullImageError":
          case "ErrImagePull":
          case "ErrImageNeverPull":
          case "ImagePullBackOff":
          case "RegistryUnavailable":
            return Optional.of(String.format("Failed to pull image %s of container %s, reason: %s, message: %s",
                cs.getImage(), cs.getName(), reason, message));
          case "InvalidImageName":
            return Optional.of(String.format("Container %s has invalid image name %s, message: %s",
                cs.getName(), cs.getImage(), message));
          default:
            return Optional.empty();
        }
      }));
}
 

private static boolean isExited(Pod pod, Optional<ContainerStatus> mainContainerStatusOpt) {
  switch (pod.getStatus().getPhase()) {
    case "Running":
      // Check if the main container has exited
      if (mainContainerStatusOpt.map(ContainerStatus::getState)
          .map(ContainerState::getTerminated)
          .isPresent()) {
        return true;
      }

      break;

    case "Succeeded":
    case "Failed":
      return true;

    default:
      // do nothing
      break;
  }

  return false;
}
 

private static List<Event> handleExited(WorkflowInstance workflowInstance, RunState state,
                                              Pod pod,
                                              Optional<ContainerStatus> mainContainerStatusOpt,
                                              Stats stats) {
  final List<Event> generatedEvents = Lists.newArrayList();

  switch (state.state()) {
    case PREPARE:
    case SUBMITTED:
      generatedEvents.add(Event.started(workflowInstance));
      // intentional fall-through

    case RUNNING:
      final Optional<Integer> exitCode = mainContainerStatusOpt.flatMap(cs ->
          getExitCodeIfValid(workflowInstance.toKey(), pod, cs, stats));
      generatedEvents.add(Event.terminate(workflowInstance, exitCode));
      break;

    default:
      // no event
      break;
  }

  return ImmutableList.copyOf(generatedEvents);
}
 

static List<Event> translate(
    WorkflowInstance workflowInstance,
    RunState state,
    Pod pod,
    Stats stats) {

  final Optional<ContainerStatus> mainContainerStatusOpt = getMainContainerStatus(pod);

  final Optional<Event> hasError = isInErrorState(workflowInstance, pod, mainContainerStatusOpt);
  if (hasError.isPresent()) {
    return handleError(state, hasError.get());
  }

  if (isExited(pod, mainContainerStatusOpt)) {
    return handleExited(workflowInstance, state, pod, mainContainerStatusOpt, stats);
  }

  if (isStarted(pod, mainContainerStatusOpt)) {
    return handleStarted(workflowInstance, state);
  }

  return List.of();
}
 

@Override
public void onEvent(@NonNull Action action, @NonNull KubernetesSlave node, @NonNull Pod pod) throws IOException, InterruptedException {
    List<ContainerStatus> backOffContainers = PodUtils.getContainers(pod, cs -> {
        ContainerStateWaiting waiting = cs.getState().getWaiting();
        return waiting != null && waiting.getMessage() != null && waiting.getMessage().contains("Back-off pulling image");
    });
    if (backOffContainers.isEmpty()) {
        return;
    }
    backOffContainers.forEach(cs -> {
        TaskListener runListener = node.getTemplate().getListener();
        runListener.error("Unable to pull Docker image \""+cs.getImage()+"\". Check if image tag name is spelled correctly.");
    });
    Queue q = Jenkins.get().getQueue();
    String runUrl = pod.getMetadata().getAnnotations().get("runUrl");
    for (Queue.Item item: q.getItems()) {
        if (item.task.getUrl().equals(runUrl)) {
            q.cancel(item);
            break;
        }
    }
    node.terminate();
}
 

public DeploymentState resolve(ContainerStatus containerStatus) {

		Stream<Predicate<ContainerStatus>> conditionsStream = Stream.of(conditions);
		Boolean allConditionsMet = conditionsStream.reduce((x, y) -> x.and(y)).get().test(containerStatus);

		if (allConditionsMet) {
			logger.debug("deployment state is " + resolvedState.name());
			return this.resolvedState;
		}
		else {
			Stream<ContainerStatusCondition> report = Stream.of(conditions);
			report.filter(c -> {
				boolean result= false;
				try {
					result = c.test(containerStatus);
				}
				catch (NullPointerException e) {

				}
				return !result;

			}).forEach(c -> logger.debug(c + " is not satisfied"));

		}
		return null;
	}
 

@Override
public void eventReceived(io.fabric8.kubernetes.client.Watcher.Action action, Pod pod) {
  log.info("Event received for pod: {}, action: {}", podName, action);
  PodStatus status = pod.getStatus();
  List<ContainerStatus> containerStatuses = status.getContainerStatuses();
  if (!containerStatuses.isEmpty()) {
    ContainerStatus containerStatus = containerStatuses.get(0);
    ContainerState state = containerStatus.getState();
    ContainerStateTerminated terminated = state.getTerminated();
    if (terminated != null) {
      Integer exitCode = terminated.getExitCode();
      log.info("Container exit code for pod {}: {}", podName, exitCode);
      if (exitCode == 0) {
        exitCodeFuture.complete(0);
      } else {
        exitCodeFuture.completeExceptionally(new RuntimeException("Completed with non zero exit code: " + exitCode));
      }
      resource.delete();
      watch.close();
    } else {
      log.warn("ContainerStateTerminated was null for pod: {}, action {}", podName, action);
    }
  } else {
    log.warn("ContainerStatus list was empty for pod: {}, action {}", podName, action);
  }
}
 

/**
 * Log the last lines of containers logs
 */
private void logLastLines(@CheckForNull List<ContainerStatus> containers, String podId, String namespace, KubernetesSlave slave,
        Map<String, Integer> errors, KubernetesClient client) {
    if (containers != null) {
        for (ContainerStatus containerStatus : containers) {
            String containerName = containerStatus.getName();
            PrettyLoggable<String, LogWatch> tailingLines = client.pods().inNamespace(namespace).withName(podId)
                    .inContainer(containerStatus.getName()).tailingLines(30);
            String log = tailingLines.getLog();
            if (!StringUtils.isBlank(log)) {
                String msg = errors != null ? String.format(" exited with error %s", errors.get(containerName)) : "";
                LOGGER.log(Level.SEVERE, "Error in provisioning; agent={0}, template={1}. Container {2}{3}. Logs: {4}",
                        new Object[]{slave, slave.getTemplate(), containerName, msg, tailingLines.getLog()});
            }
        }
    }
}
 

public static GoCDContainerDetails fromContainer(Container container, ContainerStatus containerStatus) {
    GoCDContainerDetails containerDetails = new GoCDContainerDetails();

    containerDetails.name = container.getName();
    containerDetails.image = container.getImage();
    containerDetails.imagePullPolicy = container.getImagePullPolicy();

    containerDetails.command = container.getCommand();
    containerDetails.env = new ArrayList<>();
    for (EnvVar var : container.getEnv()) {
        containerDetails.env.add(new EnvironmentVariable(var.getName(), var.getValue()));
    }
    if (containerStatus != null) {
        containerDetails.ready = containerStatus.getReady();
        containerDetails.restartCount = containerStatus.getRestartCount();
    }
    else {
        containerDetails.ready = false;
        containerDetails.restartCount = 0;
    }

    return containerDetails;
}
 

public static String getDockerContainerID(Pod pod) {
    PodStatus status = pod.getStatus();
    if (status != null) {
        List<ContainerStatus> containerStatuses = status.getContainerStatuses();
        if (containerStatuses != null) {
            for (ContainerStatus containerStatus : containerStatuses) {
                String containerID = containerStatus.getContainerID();
                if (StringUtils.isNotBlank(containerID)) {
                    String prefix = "://";
                    int idx = containerID.indexOf(prefix);
                    if (idx > 0) {
                        return containerID.substring(idx + prefix.length());
                    }
                    return containerID;
                }
            }
        }
    }
    return null;
}
 

@Override
public DeploymentState resolve(ContainerStatus containerStatus) {
	for (RunningPhaseDeploymentStateResolver resolver: delegates) {
		DeploymentState deploymentState = resolver.resolve(containerStatus);
		if (deploymentState != null) {
			return deploymentState;
		}
	}
	return null;
}
 

protected AppStatus buildAppStatus(String id, PodList podList, ServiceList services) {
	AppStatus.Builder statusBuilder = AppStatus.of(id);
	Service service = null;
	if (podList != null && podList.getItems() != null) {
		for (Pod pod : podList.getItems()) {
			String deploymentKey = pod.getMetadata().getLabels().get(SPRING_DEPLOYMENT_KEY);
			for (Service svc : services.getItems()) {
				if (svc.getMetadata().getName().equals(deploymentKey)) {
					service = svc;
					break;
				}
			}
			//find the container with the correct env var
			for(Container container : pod.getSpec().getContainers()) {
				if(container.getEnv().stream().anyMatch(envVar -> "SPRING_CLOUD_APPLICATION_GUID".equals(envVar.getName()))) {
					//find container status for this container
					Optional<ContainerStatus> containerStatusOptional =
						pod.getStatus().getContainerStatuses()
						   .stream().filter(containerStatus -> container.getName().equals(containerStatus.getName()))
						   .findFirst();

					statusBuilder.with(new KubernetesAppInstanceStatus(pod, service, properties, containerStatusOptional.orElse(null)));

					break;
				}
			}
		}
	}
	return statusBuilder.build();
}
 

public KubernetesAppInstanceStatus(Pod pod, Service service, KubernetesDeployerProperties properties, ContainerStatus containerStatus) {
	this.pod = pod;
	this.service = service;
	this.properties = properties;
	this.containerStatus = containerStatus;
	this.runningPhaseDeploymentStateResolver = new DefaultRunningPhaseDeploymentStateResolver(properties);
}
 

ContainerReady(KubernetesDeployerProperties properties) {
	super(properties, DeploymentState.deployed, new ContainerStatusCondition("container ready") {
		@Override
		public boolean test(ContainerStatus containerStatus) {
			return containerStatus.getReady();
		}
	});
}
 

public static void waitForPod(String name) {
    LOGGER.info("Waiting when Pod {} will be ready", name);

    TestUtils.waitFor("pod " + name + " to be ready", Constants.POLL_INTERVAL_FOR_RESOURCE_READINESS, Duration.ofMinutes(6).toMillis(),
        () -> {
            List<ContainerStatus> statuses =  kubeClient().getPod(name).getStatus().getContainerStatuses();
            for (ContainerStatus containerStatus : statuses) {
                if (!containerStatus.getReady()) {
                    return false;
                }
            }
            return true;
        });
    LOGGER.info("Pod {} is ready", name);
}
 

@Test
public void
    shouldCompleteExceptionallyFutureForWaitingPodIfStatusIsRunningButSomeContainersAreTerminated() {
  // given
  ContainerStatus containerStatus = mock(ContainerStatus.class);
  when(containerStatus.getName()).thenReturn("FailingContainer");
  when(containerStatus.getState())
      .thenReturn(
          new ContainerStateBuilder()
              .withNewTerminated()
              .withReason("Completed")
              .endTerminated()
              .build());

  when(status.getPhase()).thenReturn(POD_STATUS_PHASE_RUNNING);
  when(status.getContainerStatuses()).thenReturn(singletonList(containerStatus));
  CompletableFuture<?> future = kubernetesDeployments.waitRunningAsync(POD_NAME);

  // when
  verify(podResource).watch(watcherCaptor.capture());
  Watcher<Pod> watcher = watcherCaptor.getValue();
  watcher.eventReceived(Watcher.Action.MODIFIED, pod);

  // then
  assertTrue(future.isDone());
  assertTrue(future.isCompletedExceptionally());
}
 

@Test
public void
    shouldCompleteExceptionallyFutureForWaitingPodIfStatusIsRunningButSomeContainersAreWaitingAndTerminatedBefore() {
  // given
  ContainerStatus containerStatus = mock(ContainerStatus.class);
  when(containerStatus.getName()).thenReturn("FailingContainer");
  when(containerStatus.getState())
      .thenReturn(
          new ContainerStateBuilder().withNewWaiting().withMessage("bah").endWaiting().build());
  when(containerStatus.getLastState())
      .thenReturn(
          new ContainerStateBuilder()
              .withNewTerminated()
              .withReason("Completed")
              .endTerminated()
              .build());

  when(status.getPhase()).thenReturn(POD_STATUS_PHASE_RUNNING);
  when(status.getContainerStatuses()).thenReturn(singletonList(containerStatus));
  CompletableFuture<?> future = kubernetesDeployments.waitRunningAsync(POD_NAME);

  // when
  verify(podResource).watch(watcherCaptor.capture());
  Watcher<Pod> watcher = watcherCaptor.getValue();
  watcher.eventReceived(Watcher.Action.MODIFIED, pod);

  // then
  assertTrue(future.isDone());
  assertTrue(future.isCompletedExceptionally());
}
 

public static List<ContainerStatus> getContainerStatus(Pod pod) {
    PodStatus podStatus = pod.getStatus();
    if (podStatus == null) {
        return Collections.emptyList();
    }
    return podStatus.getContainerStatuses();
}
 

private void checkTerminatedContainers(List<ContainerStatus> terminatedContainers, String podId, String namespace,
        KubernetesSlave slave, KubernetesClient client) {
    if (!terminatedContainers.isEmpty()) {
        Map<String, Integer> errors = terminatedContainers.stream().collect(Collectors
                .toMap(ContainerStatus::getName, (info) -> info.getState().getTerminated().getExitCode()));

        // Print the last lines of failed containers
        logLastLines(terminatedContainers, podId, namespace, slave, errors, client);
        throw new IllegalStateException("Containers are terminated with exit codes: " + errors);
    }
}
 

/**
 * Wait until all pod containers are running
 * 
 * @return the pod
 * @throws IllegalStateException
 *             if pod or containers are no longer running
 * @throws KubernetesClientTimeoutException
 *             if time ran out
 */
private Pod periodicAwait(int i, long started, long interval, long amount) {
    Pod pod = client.pods().inNamespace(this.pod.getMetadata().getNamespace())
            .withName(this.pod.getMetadata().getName()).get();
    if (pod == null) {
        throw new IllegalStateException(String.format("Pod is no longer available: %s/%s",
                this.pod.getMetadata().getNamespace(), this.pod.getMetadata().getName()));
    } else {
        LOGGER.finest(() -> "Updating pod for " + this.pod.getMetadata().getNamespace() + "/" + this.pod.getMetadata().getName() + " : " + Serialization.asYaml(pod));
        this.pod = pod;
    }
    List<ContainerStatus> terminatedContainers = PodUtils.getTerminatedContainers(pod);
    if (!terminatedContainers.isEmpty()) {
        throw new IllegalStateException(String.format("Pod has terminated containers: %s/%s (%s)",
                this.pod.getMetadata().getNamespace(),
                this.pod.getMetadata().getName(),
                terminatedContainers.stream()
                        .map(ContainerStatus::getName)
                        .collect(joining(", ")
                        )));
    }
    if (areAllContainersRunning(pod)) {
        return pod;
    }
    try {
        return awaitWatcher(interval, TimeUnit.MILLISECONDS);
    } catch (KubernetesClientTimeoutException e) {
        if (i <= 0) {
            throw e;
        }
    }

    long remaining = (started + amount) - System.currentTimeMillis();
    long next = Math.max(0, Math.min(remaining, interval));
    return periodicAwait(i - 1, started, next, amount);
}
 

@Test
public void shouldNotCleanupRunningPodWithoutRunStateIfTerminatedAfterRefresh() {
  final ContainerStatus runningMainContainer = new ContainerStatusBuilder()
      .withName(MAIN_CONTAINER_NAME)
      .withNewState().withNewRunning().endRunning().endState()
      .build();

  final ContainerStatus terminatedMainContainer = new ContainerStatusBuilder()
      .withName(MAIN_CONTAINER_NAME)
      .withNewState()
      .withNewTerminated()
      .withFinishedAt(FIXED_INSTANT.minus(Duration.ofDays(1)).toString())
      .endTerminated()
      .endState()
      .build();

  createdPod.setStatus(new PodStatusBuilder()
      .withContainerStatuses(runningMainContainer, keepaliveContainerStatus)
      .build());

  final Pod refreshedPod = new PodBuilder(createdPod)
      .withStatus(new PodStatusBuilder()
          .withContainerStatuses(terminatedMainContainer, keepaliveContainerStatus)
          .build())
      .build();

  // Return terminated container when refreshing by name
  when(k8sClient.getPod(POD_NAME)).thenReturn(Optional.of(refreshedPod));

  var shouldDelete = kdr.shouldDeletePodWithoutRunState(WORKFLOW_INSTANCE, createdPod);

  // Leave the pod to expire and be deleted by a later poll tick
  assertThat(shouldDelete, is(false));
}
 

private PodStatus terminatedPostStatus(int exitCode) {
  ContainerStatus containerStatus = new ContainerStatus();
  containerStatus.setState(terminatedStateWithExitCode(exitCode));
  PodStatus podStatus = new PodStatus();
  podStatus.setContainerStatuses(Collections.singletonList(containerStatus));
  return podStatus;
}
 

public static KubernetesElasticAgent fromPod(KubernetesClient client, Pod pod, JobIdentifier jobIdentifier) {
    KubernetesElasticAgent agent = new KubernetesElasticAgent();
    agent.jobIdentifier = getJobIdentifier(pod, jobIdentifier);
    agent.elasticAgentId = pod.getMetadata().getName();
    agent.podDetails = KubernetesPodDetails.fromPod(pod);
    List<ContainerStatus> containerStatuses = pod.getStatus().getContainerStatuses();
    agent.containerDetails = getContainerDetails(pod.getSpec().getContainers(), containerStatuses);
    agent.events = getAllEventsForPod(pod, client);
    agent.containerLogs = getContainerLogs(client, pod, containerStatuses);
    agent.configuration = getPodConfiguration(pod);
    return agent;
}
 

private static ArrayList<GoCDContainerDetails> getContainerDetails(List<Container> containers, List<ContainerStatus> containerStatuses) {
    ArrayList<GoCDContainerDetails> details = new ArrayList<>();
    for(ContainerStatus status: containerStatuses) {
        details.add(GoCDContainerDetails.fromContainer(getContainerSpecByName(containers, status.getName()), status));
    }
    return details;
}
 

private static ArrayList<GoCDContainerLog> getContainerLogs(KubernetesClient client, Pod pod, List<ContainerStatus> containerStatuses) {
    ArrayList<GoCDContainerLog> logs = new ArrayList<>();
    for(ContainerStatus containerStatus: containerStatuses) {
        logs.add(new GoCDContainerLog(containerStatus.getName(), getPodLogs(pod, client, containerStatus.getName())));
    }
    return logs;
}
 

private boolean allContainersReady(List<ContainerStatus> containerStatuses) {
    boolean errorPresent = containerStatuses.stream()
            .filter(cs -> !cs.getReady().booleanValue())
            .findFirst()
            .isPresent();

    return !errorPresent ;
}
 

private static Optional<Event> isInErrorState(WorkflowInstance workflowInstance, Pod pod,
                                              Optional<ContainerStatus> mainContainerStatusOpt) {
  final PodStatus status = pod.getStatus();
  final String phase = status.getPhase();

  if ("NodeLost".equals(pod.getStatus().getReason())) {
    return Optional.of(Event.runError(workflowInstance, "Lost node running pod"));
  }

  switch (phase) {
    case "Pending":
      // check if one or more docker contains failed to pull their image, a possible silent error
      return mainContainerStatusOpt
          .flatMap(KubernetesPodEventTranslator::imageError)
          .map(msg -> Event.runError(workflowInstance, msg));

    case "Succeeded":
    case "Failed":
      if (mainContainerStatusOpt.isEmpty()) {
        return Optional.of(Event.runError(workflowInstance, "Could not find our container in pod"));
      }

      final ContainerStatus containerStatus = mainContainerStatusOpt.get();
      final ContainerStateTerminated terminated = containerStatus.getState().getTerminated();
      if (terminated == null) {
        return Optional.of(Event.runError(workflowInstance, "Unexpected null terminated status"));
      }
      return Optional.empty();

    case "Unknown":
      return Optional.of(Event.runError(workflowInstance, "Pod entered Unknown phase"));

    default:
      return Optional.empty();
  }
}