Java Code Examples for hudson.slaves.NodeProvisioner#PlannedNode

@Override
public NodeProvisioner.PlannedNode build() {
    KubernetesCloud cloud = getCloud();
    PodTemplate t = getTemplate();
    Future f;
    String displayName;
    try {
        KubernetesSlave agent = KubernetesSlave
                .builder()
                .podTemplate(cloud.getUnwrappedTemplate(t))
                .cloud(cloud)
                .build();
        displayName = agent.getDisplayName();
        f = Futures.immediateFuture(agent);
    } catch (IOException | Descriptor.FormException e) {
        displayName = null;
        f = Futures.immediateFailedFuture(e);
    }
    return new NodeProvisioner.PlannedNode(Util.fixNull(displayName), f, getNumExecutors());
}
 

@Test
public void provision_shouldProvisionNoneIfNotYetUpdated() {
    // given
    when(ec2Api.connect(any(String.class), any(String.class), anyString())).thenReturn(amazonEC2);

    PowerMockito.when(FleetStateStats.readClusterState(any(AmazonEC2.class), anyString(), anyString()))
            .thenReturn(new FleetStateStats("", 0, "active",
                    Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    EC2FleetCloud fleetCloud = new EC2FleetCloud(null, null, "credId", null, "region",
            "", "", "", null, null, false,
            false, 0, 0, 1, 1, false,
            false, false, 0, 0, false,
            10, false);

    // when
    Collection<NodeProvisioner.PlannedNode> r = fleetCloud.provision(null, 1);

    // then
    assertEquals(0, r.size());
    assertEquals(0, fleetCloud.getToAdd());
}
 

@Test
public void provision_shouldProvisionNoMoreMax() {
    // given
    when(ec2Api.connect(any(String.class), any(String.class), anyString())).thenReturn(amazonEC2);

    PowerMockito.when(FleetStateStats.readClusterState(any(AmazonEC2.class), anyString(), anyString()))
            .thenReturn(new FleetStateStats("", 0, "active",
                    Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    EC2FleetCloud fleetCloud = new EC2FleetCloud(null, null, "credId", null, "region",
            "", "", "", null, null, false,
            false, 0, 0, 10, 1, false,
            false, false, 0, 0, false,
            10, false);

    fleetCloud.setStats(new FleetStateStats("", 5, "active",
            Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    // when
    Collection<NodeProvisioner.PlannedNode> r = fleetCloud.provision(null, 10);

    // then
    assertEquals(5, r.size());
    assertEquals(5, fleetCloud.getToAdd());
}
 

@Test
public void provision_shouldProvisionIfBelowMax() {
    // given
    when(ec2Api.connect(any(String.class), any(String.class), anyString())).thenReturn(amazonEC2);

    PowerMockito.when(FleetStateStats.readClusterState(any(AmazonEC2.class), anyString(), anyString()))
            .thenReturn(new FleetStateStats("", 0, "active",
                    Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    EC2FleetCloud fleetCloud = new EC2FleetCloud(null, null, "credId", null, "region",
            "", "", "", null, null, false,
            false, 0, 0, 10, 1, false,
            false, false, 0, 0, false,
            10, false);

    fleetCloud.setStats(new FleetStateStats("", 5, "active",
            Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    // when
    Collection<NodeProvisioner.PlannedNode> r = fleetCloud.provision(null, 1);

    // then
    assertEquals(1, r.size());
    assertEquals(1, fleetCloud.getToAdd());
}
 

@Test
public void provision_shouldProvisionNoneWhenExceedMax() {
    // given
    when(ec2Api.connect(any(String.class), any(String.class), anyString())).thenReturn(amazonEC2);

    PowerMockito.when(FleetStateStats.readClusterState(any(AmazonEC2.class), anyString(), anyString()))
            .thenReturn(new FleetStateStats("", 0, "active",
                    Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    EC2FleetCloud fleetCloud = new EC2FleetCloud(null, null, "credId", null, "region",
            "", "", "", null, null, false,
            false, 0, 0, 9, 1, false,
            false, false, 0, 0, false,
            10, false);

    fleetCloud.setStats(new FleetStateStats("", 10, "active",
            Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    // when
    Collection<NodeProvisioner.PlannedNode> r = fleetCloud.provision(null, 1);

    // then
    assertEquals(0, r.size());
    assertEquals(0, fleetCloud.getToAdd());
}
 

@Test
public void provision_shouldProvisionNoneWhenMaxReached() {
    // given
    when(ec2Api.connect(any(String.class), any(String.class), anyString())).thenReturn(amazonEC2);

    PowerMockito.when(FleetStateStats.readClusterState(any(AmazonEC2.class), anyString(), anyString()))
            .thenReturn(new FleetStateStats("", 0, "active",
                    Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    EC2FleetCloud fleetCloud = new EC2FleetCloud(null, null, "credId", null, "region",
            "", "", "", null, null, false,
            false, 0, 0, 10, 1, false,
            false, false, 0, 0, false,
            10, false);

    fleetCloud.setStats(new FleetStateStats("", 10, "active",
            Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    // when
    Collection<NodeProvisioner.PlannedNode> r = fleetCloud.provision(null, 1);

    // then
    assertEquals(0, r.size());
    assertEquals(0, fleetCloud.getToAdd());
}
 

@Test
public void testProvisionCreateThenTerminatePod() throws Exception {
    KafkaKubernetesCloud cloud = new KafkaKubernetesCloud("kafka-kubernetes");
    cloud.setServerUrl(k.getMockServer().url("/").toString());
    cloud.setSkipTlsVerify(true);
    j.jenkins.clouds.add(cloud);

    Collection<NodeProvisioner.PlannedNode> provisionedNodes = cloud.provision(new LabelAtom("test"), 1);
    assertThat(provisionedNodes, hasSize(1));
    KafkaCloudSlave agent = (KafkaCloudSlave) provisionedNodes.iterator().next().future.get();
    TaskListener listener = new LogTaskListener(Logger.getLogger(KafkaKubernetesCloudTest.class.getName()), Level.INFO);
    PodResource<Pod, DoneablePod> pod = k.getClient().pods().inNamespace(cloud.getNamespace()).withName(agent.getNodeName());

    assertNull(pod.get());

    // Poll for pod creation
    j.jenkins.addNode(agent);
    final int TIMEOUT = 30;
    for(int i = 0; i < TIMEOUT; i++) {
        if (pod.get() != null) break;
        TimeUnit.SECONDS.sleep(1);
    }
    assertNotNull(pod.get());

    agent._terminate(listener);
    assertNull(pod.get());
}
 

@Override
public synchronized Collection<NodeProvisioner.PlannedNode> provision(Label label, int excessWorkload) {

    LOGGER.log(Level.INFO, "Asked to provision {0} agent(s) for: {1}", new Object[]{excessWorkload, label});

    List<NodeProvisioner.PlannedNode> result = new ArrayList<>();
    final ECSTaskTemplate template = getTemplate(label);
    if (template != null) {
        String parentLabel = template.getInheritFrom();
        final ECSTaskTemplate merged = template.merge(getTemplate(parentLabel));

        for (int i = 1; i <= excessWorkload; i++) {
            String agentName = name + "-" + label.getName() + "-" + RandomStringUtils.random(5, "bcdfghjklmnpqrstvwxz0123456789");
            LOGGER.log(Level.INFO, "Will provision {0}, for label: {1}", new Object[]{agentName, label} );
            result.add(
                    new NodeProvisioner.PlannedNode(
                            agentName,
                            Computer.threadPoolForRemoting.submit(
                                    new ProvisioningCallback(merged, agentName)
                            ),
                            1
                    )
            );
        }
    }
    return result.isEmpty() ? Collections.emptyList() : result;

}
 

@Test
public void testContainerCap() {
    KubernetesCloud cloud = new KubernetesCloud("name") {
        @Override
        public KubernetesClient connect()  {
            KubernetesClient mockClient =  Mockito.mock(KubernetesClient.class);
            Mockito.when(mockClient.getNamespace()).thenReturn("default");
            MixedOperation<Pod, PodList, DoneablePod, PodResource<Pod, DoneablePod>> operation = Mockito.mock(MixedOperation.class);
            Mockito.when(operation.inNamespace(Mockito.anyString())).thenReturn(operation);
            Mockito.when(operation.withLabels(Mockito.anyMap())).thenReturn(operation);
            PodList podList = Mockito.mock(PodList.class);
            Mockito.when(podList.getItems()).thenReturn(new ArrayList<>());
            Mockito.when(operation.list()).thenReturn(podList);
            Mockito.when(mockClient.pods()).thenReturn(operation);
            return mockClient;
        }
    };

    PodTemplate podTemplate = new PodTemplate();
    podTemplate.setName("test");
    podTemplate.setLabel("test");

    cloud.addTemplate(podTemplate);

    Label test = Label.get("test");
    assertTrue(cloud.canProvision(test));

    Collection<NodeProvisioner.PlannedNode> plannedNodes = cloud.provision(test, 200);
    assertEquals(200, plannedNodes.size());

    cloud.setContainerCapStr("10");
    podTemplate.setInstanceCap(20);
    plannedNodes = cloud.provision(test, 200);
    assertEquals(10, plannedNodes.size());
}
 

@Test
public void provision_shouldProvisionNoMoreMaxWhenMultipleCallBeforeUpdate() {
    // given
    when(ec2Api.connect(any(String.class), any(String.class), anyString())).thenReturn(amazonEC2);

    PowerMockito.when(FleetStateStats.readClusterState(any(AmazonEC2.class), anyString(), anyString()))
            .thenReturn(new FleetStateStats("", 0, "active",
                    Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    EC2FleetCloud fleetCloud = new EC2FleetCloud(null, null, "credId", null, "region",
            "", "", "", null, null, false,
            false, 0, 0, 10, 1, false,
            false, false, 0, 0, false,
            10, false);

    fleetCloud.setStats(new FleetStateStats("", 5, "active",
            Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    // when
    Collection<NodeProvisioner.PlannedNode> r1 = fleetCloud.provision(null, 2);
    Collection<NodeProvisioner.PlannedNode> r2 = fleetCloud.provision(null, 2);
    Collection<NodeProvisioner.PlannedNode> r3 = fleetCloud.provision(null, 5);

    // then
    assertEquals(2, r1.size());
    assertEquals(2, r2.size());
    assertEquals(1, r3.size());
    assertEquals(5, fleetCloud.getToAdd());
}
 

@Override
public Collection<NodeProvisioner.PlannedNode> provision(
        final Label label, final int excessWorkload) {
    try (Meter.Shot s = provisionMeter.start()) {
        return super.provision(label, excessWorkload);
    }
}
 

@Override
public Collection<NodeProvisioner.PlannedNode> provision(
        final Label label, final int excessWorkload) {
    final Collection<NodeProvisioner.PlannedNode> r = super.provision(label, excessWorkload);
    for (NodeProvisioner.PlannedNode ignore : r) provisionTimes.add(System.currentTimeMillis());
    return r;
}
 

@Override
public NodeProvisioner.StrategyDecision apply(final NodeProvisioner.StrategyState strategyState) {
    final Label label = strategyState.getLabel();

    final LoadStatistics.LoadStatisticsSnapshot snapshot = strategyState.getSnapshot();
    final int availableCapacity =
            snapshot.getAvailableExecutors()   // live executors
                    + snapshot.getConnectingExecutors()  // executors present but not yet connected
                    + strategyState.getPlannedCapacitySnapshot()     // capacity added by previous strategies from previous rounds
                    + strategyState.getAdditionalPlannedCapacity();  // capacity added by previous strategies _this round_

    int currentDemand = snapshot.getQueueLength() - availableCapacity;
    LOGGER.log(Level.INFO, "Available capacity={0}, currentDemand={1}",
            new Object[]{availableCapacity, currentDemand});

    for (final Cloud cloud : getClouds()) {
        if (currentDemand < 1) break;

        if (!(cloud instanceof EC2FleetCloud)) continue;
        if (!cloud.canProvision(label)) continue;

        final EC2FleetCloud ec2 = (EC2FleetCloud) cloud;
        if (!ec2.isNoDelayProvision()) continue;

        final Collection<NodeProvisioner.PlannedNode> plannedNodes = cloud.provision(label, currentDemand);
        currentDemand -= plannedNodes.size();
        LOGGER.log(Level.FINE, "Planned {0} new nodes", plannedNodes.size());
        strategyState.recordPendingLaunches(plannedNodes);
        LOGGER.log(Level.FINE, "After provisioning, available capacity={0}, currentDemand={1}",
                new Object[]{availableCapacity, currentDemand});
    }

    if (currentDemand < 1) {
        LOGGER.log(Level.FINE, "Provisioning completed");
        return NodeProvisioner.StrategyDecision.PROVISIONING_COMPLETED;
    } else {
        LOGGER.log(Level.FINE, "Provisioning not complete, consulting remaining strategies");
        return NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES;
    }
}
 

@Override
public void onRollback(@Nonnull NodeProvisioner.PlannedNode plannedNode, @Nonnull Node node, @Nonnull Throwable t) {
    delegate.onRollback(plannedNode, node, t);
}
 

@Override
public synchronized Collection<NodeProvisioner.PlannedNode> provision(final Label label, final int excessWorkload) {
    info("excessWorkload %s", excessWorkload);

    if (stats == null) {
        info("No first update, skip provision");
        return Collections.emptyList();
    }

    final int cap = stats.getNumDesired() + toAdd;

    if (cap >= getMaxSize()) {
        info("max %s reached, no more provision", getMaxSize());
        return Collections.emptyList();
    }

    if (!"active".equals(stats.getState())) {
        info("fleet in %s not active state", stats.getState());
        return Collections.emptyList();
    }

    // if the planned node has 0 executors configured force it to 1 so we end up doing an unweighted check
    final int numExecutors1 = this.numExecutors == 0 ? 1 : this.numExecutors;

    // Calculate the ceiling, without having to work with doubles from Math.ceil
    // https://stackoverflow.com/a/21830188/877024
    final int weightedExcessWorkload = (excessWorkload + numExecutors1 - 1) / numExecutors1;
    int targetCapacity = Math.min(cap + weightedExcessWorkload, getMaxSize());

    int toProvision = targetCapacity - cap;
    info("to provision = %s", toProvision);

    if (toProvision < 1) return Collections.emptyList();

    toAdd += toProvision;

    final List<NodeProvisioner.PlannedNode> resultList = new ArrayList<>();
    for (int f = 0; f < toProvision; ++f) {
        // todo make name unique per fleet
        final NodeProvisioner.PlannedNode plannedNode = new NodeProvisioner.PlannedNode(
                "FleetNode-" + f, SettableFuture.<Node>create(), this.numExecutors);
        resultList.add(plannedNode);
        plannedNodesCache.add(plannedNode);
    }
    return resultList;
}
 

/**
 * Takes a provisioning decision for a single label. Determines how many ECS tasks to start based solely on
 * queue length and how many agents are in the process of connecting.
 */
@Nonnull
@Override
public NodeProvisioner.StrategyDecision apply(@Nonnull NodeProvisioner.StrategyState state) {
    LOGGER.log(Level.FINE, "Received {0}", new Object[]{state});
    LoadStatistics.LoadStatisticsSnapshot snap = state.getSnapshot();
    Label label = state.getLabel();

    int excessWorkload = snap.getQueueLength() - snap.getAvailableExecutors() - snap.getConnectingExecutors();

    CLOUD:
    for (Cloud c : Jenkins.get().clouds) {
        if (excessWorkload <= 0) {
            break;  // enough agents allocated
        }

        // Make sure this cloud actually can provision for this label.
        if (!c.canProvision(label)) {
            continue;
        }

        for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
            CauseOfBlockage causeOfBlockage = cl.canProvision(c, label, excessWorkload);
            if (causeOfBlockage != null) {
                continue CLOUD;
            }
        }

        Collection<NodeProvisioner.PlannedNode> additionalCapacities = c.provision(label, excessWorkload);

        // compat with what the default NodeProvisioner.Strategy does
        fireOnStarted(c, label, additionalCapacities);

        for (NodeProvisioner.PlannedNode ac : additionalCapacities) {
            excessWorkload -= ac.numExecutors;
            LOGGER.log(Level.FINE, "Started provisioning {0} from {1} with {2,number,integer} "
                            + "executors. Remaining excess workload: {3,number,#.###}",
                    new Object[]{ac.displayName, c.name, ac.numExecutors, excessWorkload});
        }
        state.recordPendingLaunches(additionalCapacities);
    }
    // we took action, only pass on to other strategies if our action was insufficient
    return excessWorkload > 0 ? NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES : NodeProvisioner.StrategyDecision.PROVISIONING_COMPLETED;
}
 

@Test
public void testProvisionWorkloadSize() {
    KafkaKubernetesCloud cloud = new KafkaKubernetesCloud("kafka-kubernetes");
    Collection<NodeProvisioner.PlannedNode> nodes = cloud.provision(new LabelAtom("test"), 200);
    assertThat(nodes, hasSize(200));
}
 

@Override
public Collection<NodeProvisioner.PlannedNode> provision(final Label label, final int excessWorkload) {
    return new ArrayList<>();
}
 

@Override
public void onStarted(Cloud cloud, Label label, Collection<NodeProvisioner.PlannedNode> plannedNodes) {
    delegate.onStarted(cloud, label, plannedNodes);
}
 

/**
 * Builds the {@link hudson.slaves.NodeProvisioner.PlannedNode} instance based on the given inputs.
 * @return a {@link hudson.slaves.NodeProvisioner.PlannedNode} configured from this builder.
 */
public abstract NodeProvisioner.PlannedNode build();