io.kubernetes.client.openapi.models.V1Node Java Examples

/**
 * Returns true if there is {@link KubeConstants#TAINT_SCHEDULER} taint with {@link KubeConstants#TAINT_SCHEDULER_VALUE_FENZO} value
 * or this taint is missing (no explicit scheduler taint == Fenzo).
 */
public static boolean hasFenzoSchedulerTaint(V1Node node) {
    List<V1Taint> taints = node.getSpec().getTaints();
    if (CollectionsExt.isNullOrEmpty(taints)) {
        return true;
    }
    Set<String> schedulerTaintValues = taints.stream()
            .filter(t -> KubeConstants.TAINT_SCHEDULER.equals(t.getKey()))
            .map(t -> StringExt.safeTrim(t.getValue()))
            .collect(Collectors.toSet());

    if (schedulerTaintValues.isEmpty()) {
        return true;
    }

    return schedulerTaintValues.size() == 1 && KubeConstants.TAINT_SCHEDULER_VALUE_FENZO.equalsIgnoreCase(CollectionsExt.first(schedulerTaintValues));
}
 

public static boolean isFarzoneNode(List<String> farzones, V1Node node) {
    Map<String, String> labels = node.getMetadata().getLabels();
    if (CollectionsExt.isNullOrEmpty(labels)) {
        return false;
    }
    String nodeZone = labels.get(KubeConstants.NODE_LABEL_ZONE);
    if (StringExt.isEmpty(nodeZone)) {
        logger.debug("Node without zone label: {}", node.getMetadata().getName());
        return false;
    }
    for (String farzone : farzones) {
        if (farzone.equalsIgnoreCase(nodeZone)) {
            logger.debug("Farzone node: nodeId={}, zoneId={}", node.getMetadata().getName(), nodeZone);
            return true;
        }
    }
    logger.debug("Non-farzone node: nodeId={}, zoneId={}", node.getMetadata().getName(), nodeZone);
    return false;
}
 

private void reconcileNodesAndPods() {
    if (!mesosConfiguration.isReconcilerEnabled() || !kubeApiFacade.getNodeInformer().hasSynced() || !kubeApiFacade.getPodInformer().hasSynced()) {
        return;
    }

    List<V1Node> nodes = kubeApiFacade.getNodeInformer().getIndexer().list();
    List<V1Pod> pods = kubeApiFacade.getPodInformer().getIndexer().list();
    List<Task> tasks = v3JobOperations.getTasks();
    Map<String, Task> currentTasks = tasks.stream().collect(Collectors.toMap(Task::getId, Function.identity()));

    gcTimedOutNodes(nodes);
    gcOrphanedPodsWithoutValidNodes(nodes, pods);
    gcTerminalPods(pods, currentTasks);
    gcUnknownPods(pods, currentTasks);
    gcPodsPastDeletionTimestamp(pods);
    gcPendingPodsWithDeletionTimestamp(pods);
}
 

@Test
public void taintNotToleratedInConfiguration() {
    AgentInstance agentInstance = createAgentInstance(INSTANCE_ID, INSTANCE_GROUP_ID);
    when(agentManagementService.findAgentInstance(INSTANCE_ID)).thenReturn(Optional.of(agentInstance));

    V1Taint taint = createTaint("not_tolerated_taint_key", "NoSchedule", "not_tolerated_taint_value");
    V1Node node = createNode(INSTANCE_ID, true, Collections.singletonList(taint));
    when(indexer.getByKey(INSTANCE_ID)).thenReturn(node);

    ConstraintEvaluator.Result result = kubeConstraint.evaluate(
            createTaskRequest(TASK_ID),
            createVirtualMachineCurrentStateMock(INSTANCE_ID, Collections.emptyList(), Collections.emptyList()),
            taskTrackerState);
    assertThat(result.isSuccessful()).isFalse();
    assertThat(result.getFailureReason()).isEqualToIgnoringCase(KubeConstraint.TAINT_NOT_TOLERATED_IN_CONFIGURATION_REASON);
}
 

private SharedIndexInformer<V1Node> createNodeInformer(SharedInformerFactory sharedInformerFactory) {
    return sharedInformerFactory.sharedIndexInformerFor(
            (CallGeneratorParams params) -> coreV1Api.listNodeCall(
                    null,
                    null,
                    null,
                    null,
                    null,
                    null,
                    params.resourceVersion,
                    params.timeoutSeconds,
                    params.watch,
                    null
            ),
            V1Node.class,
            V1NodeList.class,
            configuration.getKubeApiServerIntegratorRefreshIntervalMs()
    );
}
 

private void reconcile() {
    if (!mesosConfiguration.isReconcilerEnabled()) {
        logger.info("Skipping the job management / Kube reconciliation cycle: reconciler disabled");
        return;
    }
    if (!kubeApiFacade.getNodeInformer().hasSynced() || !kubeApiFacade.getPodInformer().hasSynced()) {
        logger.info("Skipping the job management / Kube reconciliation cycle: Kube informers not ready (node={}, pod={})",
                kubeApiFacade.getNodeInformer().hasSynced(), kubeApiFacade.getPodInformer().hasSynced()
        );
        return;
    }

    List<V1Node> nodes = kubeApiFacade.getNodeInformer().getIndexer().list();
    List<V1Pod> pods = kubeApiFacade.getPodInformer().getIndexer().list();
    List<Task> tasks = v3JobOperations.getTasks();

    Map<String, V1Node> nodesById = nodes.stream().collect(Collectors.toMap(
            node -> node.getMetadata().getName(),
            Function.identity()
    ));
    Map<String, Task> currentTasks = tasks.stream().collect(Collectors.toMap(Task::getId, Function.identity()));
    Set<String> currentPodNames = pods.stream().map(p -> p.getMetadata().getName()).collect(Collectors.toSet());

    transitionOrphanedTasks(currentTasks, currentPodNames, nodesById);
}
 

private void nodeUpdated(V1Node node) {
    try {
        boolean notOwnedByFenzo = !KubeUtil.isNodeOwnedByFenzo(directKubeConfiguration.getFarzones(), node);
        if (notOwnedByFenzo) {
            String nodeName = node.getMetadata().getName();
            logger.debug("Ignoring node: {} as it is not owned by fenzo", nodeName);
        } else {
            VirtualMachineLease lease = nodeToLease(node);
            if (lease != null) {
                logger.debug("Adding lease: {}", lease.getId());
                leaseHandler.call(Collections.singletonList(lease));
            }
        }
    } catch (Exception e) {
        logger.warn("Exception on node update: {}", node, e);
    }
}
 

/**
 * GC orphaned pods on nodes that are no longer valid/available.
 */
private void gcOrphanedPodsWithoutValidNodes(List<V1Node> nodes, List<V1Pod> pods) {
    Set<String> currentNodeNames = nodes.stream().map(n -> n.getMetadata().getName()).collect(Collectors.toSet());
    List<V1Pod> orphanedPodsWithoutValidNodesToGc = pods.stream()
            .filter(p -> {
                String nodeName = p.getSpec().getNodeName();
                return StringExt.isNotEmpty(nodeName) && !currentNodeNames.contains(nodeName);
            })
            .collect(Collectors.toList());

    logger.info("Attempting to GC {} orphaned pods: {} without valid nodes", orphanedPodsWithoutValidNodesToGc.size(),
            orphanedPodsWithoutValidNodesToGc);
    orphanedPodsWithoutValidNodesToGcGauge.set(orphanedPodsWithoutValidNodesToGc.size());
    for (V1Pod pod : orphanedPodsWithoutValidNodesToGc) {
        gcPod(pod);
    }
    logger.info("Finished orphaned pod GC without valid nodes");
}
 

private boolean isNodeReadyForGc(V1Node node) {
    Optional<V1NodeCondition> stoppedConditionOpt = node.getStatus().getConditions().stream()
            .filter(c -> c.getType().equalsIgnoreCase(STOPPED) && Boolean.parseBoolean(c.getStatus()))
            .findAny();
    if (stoppedConditionOpt.isPresent()) {
        return true;
    }

    Optional<V1NodeCondition> readyConditionOpt = node.getStatus().getConditions().stream()
            .filter(c -> c.getType().equalsIgnoreCase(READY))
            .findAny();
    if (!readyConditionOpt.isPresent()) {
        return false;
    }
    V1NodeCondition readyCondition = readyConditionOpt.get();
    boolean status = Boolean.parseBoolean(readyCondition.getStatus());
    DateTime lastHeartbeatTime = readyCondition.getLastHeartbeatTime();
    return !status &&
            lastHeartbeatTime != null &&
            clock.isPast(lastHeartbeatTime.getMillis() + directKubeConfiguration.getNodeGcTtlMs());
}
 

private Protos.Offer nodeToOffer(V1Node node) {
    try {
        V1ObjectMeta metadata = node.getMetadata();
        V1NodeStatus status = node.getStatus();
        String nodeName = metadata.getName();
        boolean hasTrueReadyCondition = status.getConditions().stream()
                .anyMatch(c -> c.getType().equalsIgnoreCase(READY) && Boolean.parseBoolean(c.getStatus()));
        if (hasTrueReadyCondition) {
            List<Protos.Attribute> nodeAttributes = nodeToAttributes(node);
            if (hasRequiredNodeAttributes(nodeAttributes)) {
                return Protos.Offer.newBuilder()
                        .setId(Protos.OfferID.newBuilder().setValue(nodeName).build())
                        .setSlaveId(Protos.SlaveID.newBuilder().setValue(nodeName).build())
                        .setHostname(nodeName)
                        .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("TitusFramework").build())
                        .addAllResources(nodeToResources(node))
                        .addAllAttributes(nodeAttributes)
                        .build();
            } else {
                logger.debug("Ignoring node {}, as not all required attributes are set: nodeAttributes={}",
                        node.getMetadata().getName(), nodeAttributes);
            }
        }
    } catch (Exception ignore) {
        logger.info("Failed to convert node to offer for node {}", node, ignore);
    }
    return null;
}
 

private Optional<V1Node> findNode(Task task, Map<String, V1Node> nodes) {
    // Node name may be different from agent instance id. We use the instance id attribute only as a fallback.
    String nodeName = task.getTaskContext().getOrDefault(
            TaskAttributes.TASK_ATTRIBUTES_KUBE_NODE_NAME,
            task.getTaskContext().get(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID)
    );
    if (nodeName == null) {
        return Optional.empty();
    }
    return Optional.ofNullable(nodes.get(nodeName));
}
 

public void handleStringMap(Map<String, String>  map, String prefix, Map<String, Object> data) {
    if (map != null) {
        for (String key : map.keySet()) {
            String columnName = (prefix + key).toLowerCase();
            data.put(columnName, map.get(key));
            if (!getKubeColumn().containsKey(columnName)) {
                UpdateColumns(columnName, new KubeColumn<V1Node>(columnName, VarcharType.createUnboundedVarcharType(), key));
            }
        }
    }
}
 

private VirtualMachineLease nodeToLease(V1Node node) {
    Protos.Offer offer = nodeToOffer(node);
    if (offer == null) {
        return null;
    }
    return new VMLeaseObject(offer);
}
 

private void nodeDeleted(V1Node node) {
    try {
        String leaseId = node.getMetadata().getName();
        logger.debug("Removing lease on node delete: {}", leaseId);
        rescindLeaseHandler.call(Collections.singletonList(LeaseRescindedEvent.leaseIdEvent(leaseId)));
    } catch (Exception e) {
        logger.warn("Exception on node delete: {}", node, e);
    }
}
 

@Override
public Result reconcile(Request request) {
  V1Node node = this.nodeLister.get(request.getName());
  System.out.println("triggered reconciling " + node.getMetadata().getName());
  this.eventRecorder.event(
      node,
      EventType.Normal,
      "Print Node",
      "Successfully printed %s",
      node.getMetadata().getName());
  return new Result(false);
}
 

public static Optional<String> getNodeIpV4Address(V1Node node) {
    return Optional.ofNullable(node.getStatus().getAddresses())
            .map(Collection::stream)
            .orElseGet(Stream::empty)
            .filter(a -> a.getType().equalsIgnoreCase(TYPE_INTERNAL_IP) && NetworkExt.isIpV4(a.getAddress()))
            .findFirst()
            .map(V1NodeAddress::getAddress);
}
 

/**
 * A node is owned by Fenzo if:
 * <ul>
 *     <li>There is no taint with {@link KubeConstants#TAINT_SCHEDULER} key and it is not a farzone node</li>
 *     <li>There is one taint with {@link KubeConstants#TAINT_SCHEDULER} key and 'fenzo' value</li>
 * </ul>
 */
public static boolean isNodeOwnedByFenzo(List<String> farzones, V1Node node) {
    if (isFarzoneNode(farzones, node)) {
        logger.debug("Not owned by fenzo (farzone node): {}", node.getMetadata().getName());
        return false;
    }

    if (!hasFenzoSchedulerTaint(node)) {
        logger.debug("Not owned by fenzo (non Fenzo scheduler taint): {}", node.getMetadata().getName());
        return false;
    }

    logger.debug("Owned by fenzo");
    return true;
}
 

private Iterable<? extends Protos.Resource> nodeToResources(V1Node node) {
    V1NodeStatus status = node.getStatus();
    Map<String, Quantity> allocatableResources = status.getAllocatable();
    List<Protos.Resource> resources = new ArrayList<>();
    resources.add(createResource("cpus", allocatableResources.getOrDefault("cpu", DEFAULT_QUANTITY).getNumber().doubleValue()));
    resources.add(createResource("mem", allocatableResources.getOrDefault("memory", DEFAULT_QUANTITY).getNumber().doubleValue()));
    resources.add(createResource("disk", allocatableResources.getOrDefault("storage", DEFAULT_QUANTITY).getNumber().doubleValue()));
    resources.add(createResource("network", allocatableResources.getOrDefault("network", DEFAULT_QUANTITY).getNumber().doubleValue()));
    resources.add(createResource("gpu", allocatableResources.getOrDefault("gpu", DEFAULT_QUANTITY).getNumber().doubleValue()));
    return resources;
}
 

private List<Protos.Attribute> nodeToAttributes(V1Node node) {
    V1ObjectMeta metadata = node.getMetadata();

    List<Protos.Attribute> attributes = new ArrayList<>();
    KubeUtil.getNodeIpV4Address(node).ifPresent(nodeIp -> attributes.add(createAttribute(NODE_ATTRIBUTE_HOST_IP, nodeIp)));

    for (Map.Entry<String, String> entry : metadata.getAnnotations().entrySet()) {
        attributes.add(createAttribute(entry.getKey(), entry.getValue()));

    }
    return attributes;
}
 

private Optional<V1Node> findNode(V1Pod pod) {
    String nodeName = pod.getSpec().getNodeName();
    if (StringExt.isEmpty(nodeName)) {
        return Optional.empty();
    }
    return Optional.ofNullable(kubeApiFacade.getNodeInformer().getIndexer().getByKey(nodeName));
}
 

/**
 * GC nodes that have timed out due to not publishing a heartbeat
 */
private void gcTimedOutNodes(List<V1Node> nodes) {
    List<V1Node> timedOutNodesToGc = nodes.stream()
            .filter(this::isNodeReadyForGc)
            .collect(Collectors.toList());

    logger.info("Attempting to GC {} timed out nodes: {}", timedOutNodesToGc.size(), timedOutNodesToGc);
    timedOutNodesToGcGauge.set(timedOutNodesToGc.size());
    for (V1Node node : timedOutNodesToGc) {
        gcNode(node);
    }
    logger.info("Finished timed out node GC");
}
 

private Mono<Void> updateTaskStatus(PodWrapper pod,
                                    Task task,
                                    TaskState newTaskState,
                                    Optional<TitusExecutorDetails> executorDetailsOpt,
                                    Optional<V1Node> node) {
    return ReactorExt.toMono(v3JobOperations.updateTask(
            task.getId(),
            currentTask -> Optional.of(updateTaskStatus(pod, newTaskState, executorDetailsOpt, node, currentTask)),
            V3JobOperations.Trigger.Kube,
            "Kube pod notification",
            KUBE_CALL_METADATA
    ));
}
 

private static Task attachNodeMetadata(Task task, V1Node node) {
    Map<String, String> annotations = node.getMetadata().getAnnotations();
    if (CollectionsExt.isNullOrEmpty(annotations)) {
        return task;
    }

    Map<String, String> agentAttributes = new HashMap<>();

    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "ami"), ami -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_AMI, ami));
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "asg"), asg -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_ASG, asg));
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "cluster"), cluster -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_CLUSTER, cluster));
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "id"), id -> {
        agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_INSTANCE_ID, id);
        agentAttributes.put("agent.id", id);
    });
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "itype"), itype -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_ITYPE, itype));
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "region"), region -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_REGION, region));
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "res"), res -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_RES, res));
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "stack"), stack -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_STACK, stack));
    acceptNotNull(annotations.get(TITUS_NODE_DOMAIN + "zone"), zone -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_ZONE, zone));

    acceptNotNull(node.getMetadata().getName(), nodeName -> agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_KUBE_NODE_NAME, nodeName));

    String nodeIpAddress = KubeUtil.getNodeIpV4Address(node).orElse("UnknownIpAddress");
    agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_HOST, nodeIpAddress);
    agentAttributes.put(TaskAttributes.TASK_ATTRIBUTES_AGENT_HOST_IP, nodeIpAddress);

    return task.toBuilder()
            .withTaskContext(CollectionsExt.merge(task.getTaskContext(), agentAttributes))
            .build();
}
 

private V1Node newNodeWithoutZone(V1Taint... taints) {
    V1Node node = new V1Node()
            .metadata(new V1ObjectMeta().labels(Collections.emptyMap()))
            .spec(new V1NodeSpec().taints(new ArrayList<>()));
    for (V1Taint taint : taints) {
        node.getSpec().getTaints().add(taint);
    }
    return node;
}
 

@Test
public void nodeNotReady() {
    AgentInstance agentInstance = createAgentInstance(INSTANCE_ID, INSTANCE_GROUP_ID);
    when(agentManagementService.findAgentInstance(INSTANCE_ID)).thenReturn(Optional.of(agentInstance));

    V1Node node = createNode(INSTANCE_ID, false, Collections.emptyList());
    when(indexer.getByKey(INSTANCE_ID)).thenReturn(node);

    ConstraintEvaluator.Result result = kubeConstraint.evaluate(
            createTaskRequest(TASK_ID),
            createVirtualMachineCurrentStateMock(INSTANCE_ID, Collections.emptyList(), Collections.emptyList()),
            taskTrackerState);
    assertThat(result.isSuccessful()).isFalse();
    assertThat(result.getFailureReason()).isEqualToIgnoringCase(KubeConstraint.NODE_NOT_READY_REASON);
}
 

@Test
public void successfullyScheduled() {
    AgentInstance agentInstance = createAgentInstance(INSTANCE_ID, INSTANCE_GROUP_ID);
    when(agentManagementService.findAgentInstance(INSTANCE_ID)).thenReturn(Optional.of(agentInstance));

    V1Taint taint = createTaint("tolerated_taint_key", "NoSchedule", "tolerated_taint_value");
    V1Node node = createNode(INSTANCE_ID, true, Collections.singletonList(taint));
    when(indexer.getByKey(INSTANCE_ID)).thenReturn(node);

    ConstraintEvaluator.Result result = kubeConstraint.evaluate(
            createTaskRequest(TASK_ID),
            createVirtualMachineCurrentStateMock(INSTANCE_ID, Collections.emptyList(), Collections.emptyList()),
            taskTrackerState);
    assertThat(result.isSuccessful()).isTrue();
}
 

private KubeApiFacade createKubeApiFacade() {
    Indexer<V1Node> indexer = mock(Indexer.class);
    SharedIndexInformer<V1Node> nodeInformer = mock(SharedIndexInformer.class);
    KubeApiFacade kubeApiFacade = mock(KubeApiFacade.class);

    when(kubeApiFacade.getNodeInformer()).thenReturn(nodeInformer);
    when(nodeInformer.getIndexer()).thenReturn(indexer);

    return kubeApiFacade;
}
 

private V1Node createNode(String name, Boolean ready, List<V1Taint> taints) {
    return new V1Node()
            .metadata(
                    new V1ObjectMeta().name(name)
            )
            .status(
                    new V1NodeStatus()
                            .addConditionsItem(new V1NodeCondition().type(KubeConstraint.READY).status(ready.toString()))
            ).spec(
                    new V1NodeSpec().taints(taints)
            );
}
 

PodUpdatedEvent(V1Pod oldPod, V1Pod newPod, Optional<V1Node> node) {
    super(newPod);
    this.oldPod = oldPod;
    this.node = node;
}
 

@Override
public Result evaluate(TaskRequest taskRequest, VirtualMachineCurrentState targetVM, TaskTrackerState taskTrackerState) {
    if (!mesosConfiguration.isKubeApiServerIntegrationEnabled()) {
        return VALID;
    }

    Optional<AgentInstance> instanceOpt = SchedulerUtils.findInstance(agentManagementService, schedulerConfiguration.getInstanceAttributeName(), targetVM);
    if (!instanceOpt.isPresent()) {
        return Failure.INSTANCE_NOT_FOUND.toResult();
    }

    String instanceId = instanceOpt.get().getId();
    V1Node node = kubeApiFacade.getNodeInformer().getIndexer().getByKey(instanceId);
    if (node == null) {
        return Failure.NODE_NOT_FOUND.toResult();
    }

    V1NodeCondition readyCondition = null;
    if (node.getStatus() != null) {
        List<V1NodeCondition> conditions = node.getStatus().getConditions();
        if (conditions != null && !conditions.isEmpty()) {
            for (V1NodeCondition condition : node.getStatus().getConditions()) {
                if (condition.getType().equalsIgnoreCase(READY)) {
                    readyCondition = condition;
                    break;
                }
            }
        }
    }

    if (readyCondition == null || !Boolean.parseBoolean(readyCondition.getStatus())) {
        return Failure.NODE_NOT_READY.toResult();
    }

    Set<String> toleratedTaints = mesosConfiguration.getFenzoTaintTolerations();
    if (toleratedTaints == null || toleratedTaints.isEmpty()) {
        return VALID;
    }

    if (node.getSpec() != null) {
        List<V1Taint> taints = node.getSpec().getTaints();
        if (taints != null && !taints.isEmpty()) {
            for (V1Taint taint : taints) {
                String taintKey = taint.getKey();
                if (!toleratedTaints.contains(taintKey)) {
                    return Failure.TAINT_NOT_TOLERATED_IN_CONFIGURATION.toResult();
                }
            }
        }
    }

    return VALID;
}