hudson.slaves.OfflineCause Java Examples

public ComputerOfflineEventImpl(Computer computer, OfflineCause cause, Map<String, Set<String>> tags, boolean isTemporarily) {
    super(tags);

    String nodeName = DatadogUtilities.getNodeName(computer);
    setAggregationKey(nodeName);

    String title = "Jenkins node " + nodeName + " is" + (isTemporarily? " temporarily ": " ") + "offline";
    setTitle(title);

    // TODO: Add more info about the case in the event in message.
    String text = "%%% \nJenkins node " + nodeName + " is" + (isTemporarily? " temporarily ": " ") +
            "offline \n%%%";
    setText(text);

    setPriority(Priority.NORMAL);
    setAlertType(AlertType.WARNING);
}
 

@Test
public void should_successfully_resubmit_freestyle_task() throws Exception {
    EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region",
            null, "fId", "momo", null, new LocalComputerConnector(j), false, false,
            0, 0, 10, 1, false, false,
            false, 0, 0, false,
            10, false);
    j.jenkins.clouds.add(cloud);

    List<QueueTaskFuture> rs = getQueueTaskFutures(1);

    System.out.println("check if zero nodes!");
    Assert.assertEquals(0, j.jenkins.getNodes().size());

    assertAtLeastOneNode();

    final Node node = j.jenkins.getNodes().get(0);
    assertQueueIsEmpty();

    System.out.println("disconnect node");
    node.toComputer().disconnect(new OfflineCause.ChannelTermination(new UnsupportedOperationException("Test")));

    // due to test nature job could be failed if started or aborted as we call disconnect
    // in prod code it's not matter
    assertLastBuildResult(Result.FAILURE, Result.ABORTED);

    node.toComputer().connect(true);
    assertNodeIsOnline(node);
    assertQueueAndNodesIdle(node);

    Assert.assertEquals(1, j.jenkins.getProjects().size());
    Assert.assertEquals(Result.SUCCESS, j.jenkins.getProjects().get(0).getLastBuild().getResult());
    Assert.assertEquals(2, j.jenkins.getProjects().get(0).getBuilds().size());

    cancelTasks(rs);
}
 

@Test
public void should_not_resubmit_if_disabled() throws Exception {
    EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region",
            null, "fId", "momo", null, new LocalComputerConnector(j), false, false,
            0, 0, 10, 1, false, false,
            true, 0, 0, false, 10, false);
    j.jenkins.clouds.add(cloud);

    List<QueueTaskFuture> rs = getQueueTaskFutures(1);

    System.out.println("check if zero nodes!");
    Assert.assertEquals(0, j.jenkins.getNodes().size());

    assertAtLeastOneNode();

    final Node node = j.jenkins.getNodes().get(0);
    assertQueueIsEmpty();

    System.out.println("disconnect node");
    node.toComputer().disconnect(new OfflineCause.ChannelTermination(new UnsupportedOperationException("Test")));

    assertLastBuildResult(Result.FAILURE, Result.ABORTED);

    node.toComputer().connect(true);
    assertNodeIsOnline(node);
    assertQueueAndNodesIdle(node);

    Assert.assertEquals(1, j.jenkins.getProjects().size());
    Assert.assertEquals(Result.FAILURE, j.jenkins.getProjects().get(0).getLastBuild().getResult());
    Assert.assertEquals(1, j.jenkins.getProjects().get(0).getBuilds().size());

    cancelTasks(rs);
}
 

@Test
public void afterDisconnect_should_do_nothing_if_task_finished_cause_but_still_online() {
    when(computer.isOffline()).thenReturn(false);
    when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null));
    new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher)
            .afterDisconnect(computer, taskListener);
    verifyZeroInteractions(queue);
}
 

@Test
public void taskCompleted_should_resubmit_task_if_offline_and_cause_disconnect() {
    when(computer.getExecutors()).thenReturn(Arrays.asList(executor1));
    when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null));
    new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher)
            .afterDisconnect(computer, taskListener);
    verify(queue).schedule2(eq(task1), anyInt(), eq(Collections.<Action>emptyList()));
    verifyZeroInteractions(queue);
}
 

@Test
public void taskCompleted_should_not_resubmit_task_if_offline_and_cause_disconnect_but_disabled() {
    when(cloud.isDisableTaskResubmit()).thenReturn(true);
    when(computer.getExecutors()).thenReturn(Arrays.asList(executor1));
    when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null));
    new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher)
            .afterDisconnect(computer, taskListener);
    verifyZeroInteractions(queue);
}
 

@Test
public void taskCompleted_should_resubmit_task_for_all_executors() {
    when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null));
    new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher)
            .afterDisconnect(computer, taskListener);
    verify(queue).schedule2(eq(task1), anyInt(), eq(Collections.<Action>emptyList()));
    verify(queue).schedule2(eq(task2), anyInt(), eq(Collections.<Action>emptyList()));
    verifyZeroInteractions(queue);
}
 

@Test
public void taskCompleted_should_abort_executors_during_resubmit() {
    when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null));
    new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher)
            .afterDisconnect(computer, taskListener);
    verify(executor1).interrupt(Result.ABORTED, new EC2TerminationCause("i-12"));
    verify(executor2).interrupt(Result.ABORTED, new EC2TerminationCause("i-12"));
}
 

@Test
public void taskCompleted_should_resubmit_task_with_actions() {
    when(computer.getExecutors()).thenReturn(Arrays.asList(executor1));
    when(executable1.getActions()).thenReturn(Arrays.asList(action1));
    when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null));
    new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher)
            .afterDisconnect(computer, taskListener);
    verify(queue).schedule2(eq(task1), anyInt(), eq(Arrays.asList(action1)));
    verifyZeroInteractions(queue);
}
 

public void disconnectSlave(DumbSlave slave) throws Exception {
    slave.getComputer().disconnect(new OfflineCause.ChannelTermination(new Exception("terminate")));
    long start = System.currentTimeMillis();
    while (slave.getChannel() != null) {
        if (System.currentTimeMillis() > (start + 10000)) {
            throw new IllegalStateException("Timed out waiting on DumbSlave channel to disconnect.");
        }
        Thread.sleep(200);
    }
}
 

@Override
public void onOffline(@Nonnull Computer computer, @CheckForNull OfflineCause cause) {
    try {
        final boolean emitSystemEvents = DatadogUtilities.getDatadogGlobalDescriptor().isEmitSystemEvents();
        if (!emitSystemEvents) {
            return;
        }
        logger.fine("Start DatadogComputerListener#onOffline");

        // Get Datadog Client Instance
        DatadogClient client = ClientFactory.getClient();

        // Get the list of tags to apply
        Map<String, Set<String>> tags = TagsUtil.merge(
                DatadogUtilities.getTagsFromGlobalTags(),
                DatadogUtilities.getComputerTags(computer));

        // Send event
        DatadogEvent event = new ComputerOfflineEventImpl(computer, cause, tags, false);
        client.event(event);

        // Submit counter
        String hostname = DatadogUtilities.getHostname("null");
        client.incrementCounter("jenkins.computer.offline", hostname, tags);

        logger.fine("End DatadogComputerListener#onOffline");
    } catch (Exception e) {
        logger.warning("Unexpected exception occurred - " + e.getMessage());
    }
}
 

@Override
public void onTemporarilyOffline(Computer computer, OfflineCause cause) {
    try {
        final boolean emitSystemEvents = DatadogUtilities.getDatadogGlobalDescriptor().isEmitSystemEvents();
        if (!emitSystemEvents) {
            return;
        }
        logger.fine("Start DatadogComputerListener#onTemporarilyOffline");

        // Get Datadog Client Instance
        DatadogClient client = ClientFactory.getClient();

        // Get the list of tags to apply
        Map<String, Set<String>> tags = TagsUtil.merge(
                DatadogUtilities.getTagsFromGlobalTags(),
                DatadogUtilities.getComputerTags(computer));

        // Send event
        DatadogEvent event = new ComputerOfflineEventImpl(computer, cause, tags, true);
        client.event(event);

        // Submit counter
        String hostname = DatadogUtilities.getHostname("null");
        client.incrementCounter("jenkins.computer.temporarily_offline", hostname, tags);

        logger.fine("End DatadogComputerListener#onTemporarilyOffline");
    } catch (Exception e) {
        logger.warning("Unexpected exception occurred - " + e.getMessage());
    }
}
 

@Override
public Future<?> disconnect(OfflineCause cause) {
    return CompletableFuture.completedFuture(true);
}
 

/**
 * {@link ComputerLauncher#afterDisconnect(SlaveComputer, TaskListener)}
 * <p>
 * EC2 Fleet plugin overrides this method to detect jobs which were failed because of
 * EC2 instance was terminated/stopped. It could be manual stop or because of Spot marked.
 * In all cases as soon as job aborted because of broken connection and slave is offline
 * it will try to resubmit aborted job back to the queue, so user doesn't need to do that manually
 * and another slave could take it.
 * <p>
 * Implementation details
 * <p>
 * There is no official recommendation about way how to resubmit job according to
 * https://issues.jenkins-ci.org/browse/JENKINS-49707 moreover some of Jenkins code says it impossible.
 * <p>
 * method checks {@link SlaveComputer#getOfflineCause()} for disconnect because of EC2 instance termination
 * it returns
 * <code>
 * result = {OfflineCause$ChannelTermination@13708} "Connection was broken: java.io.IOException:
 * Unexpected termination of the channel\n\tat hudson.remoting.SynchronousCommandTransport$ReaderThread...
 * cause = {IOException@13721} "java.io.IOException: Unexpected termination of the channel"
 * timestamp = 1561067177837
 * </code>
 *
 * @param computer computer
 * @param listener listener
 */
@Override
public void afterDisconnect(final SlaveComputer computer, final TaskListener listener) {
    // according to jenkins docs could be null in edge cases, check ComputerLauncher.afterDisconnect
    if (computer == null) return;

    // in some multi-thread edge cases cloud could be null for some time, just be ok with that
    final EC2FleetCloud cloud = ((EC2FleetNodeComputer) computer).getCloud();
    if (cloud == null) {
        LOGGER.warning("Edge case cloud is null for computer " + computer.getDisplayName()
                + " should be autofixed in a few minutes, if no please create issue for plugin");
        return;
    }

    final boolean unexpectedDisconnect = computer.isOffline() && computer.getOfflineCause() instanceof OfflineCause.ChannelTermination;
    if (!cloud.isDisableTaskResubmit() && unexpectedDisconnect) {
        final List<Executor> executors = computer.getExecutors();
        LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
                + " termination,  resubmit");

        for (Executor executor : executors) {
            if (executor.getCurrentExecutable() != null) {
                executor.interrupt(Result.ABORTED, new EC2TerminationCause(computer.getDisplayName()));

                final Queue.Executable executable = executor.getCurrentExecutable();
                // if executor is not idle
                if (executable != null) {
                    final SubTask subTask = executable.getParent();
                    final Queue.Task task = subTask.getOwnerTask();

                    List<Action> actions = new ArrayList<>();
                    if (executable instanceof Actionable) {
                        actions = ((Actionable) executable).getActions();
                    }

                    Queue.getInstance().schedule2(task, RESCHEDULE_QUIET_PERIOD_SEC, actions);
                    LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
                            + " termination, resubmit " + task + " with actions " + actions);
                }
            }
        }
        LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
                + " termination, resubmit finished");
    } else {
        LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName()
                + " termination but resubmit disabled, no actions, disableTaskResubmit: "
                + cloud.isDisableTaskResubmit() + ", offline: " + computer.isOffline()
                + ", offlineCause: " + (computer.getOfflineCause() != null ? computer.getOfflineCause().getClass() : "null"));
    }

    // call parent
    super.afterDisconnect(computer, listener);
}
 

@Test
public void should_successfully_resubmit_parametrized_task() throws Exception {
    EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region",
            null, "fId", "momo", null, new LocalComputerConnector(j), false, false,
            0, 0, 10, 1, false, false,
            false, 0, 0, false,
            10, false);
    j.jenkins.clouds.add(cloud);

    List<QueueTaskFuture> rs = new ArrayList<>();
    final FreeStyleProject project = j.createFreeStyleProject();
    project.setAssignedLabel(new LabelAtom("momo"));
    project.addProperty(new ParametersDefinitionProperty(new StringParameterDefinition("number", "opa")));
    /*
    example of actions for project

    actions = {CopyOnWriteArrayList@14845}  size = 2
        0 = {ParametersAction@14853}
        safeParameters = {TreeSet@14855}  size = 0
        parameters = {ArrayList@14856}  size = 1
        0 = {StringParameterValue@14862} "(StringParameterValue) number='1'"
        value = "1"
        name = "number"
        description = ""
        parameterDefinitionNames = {ArrayList@14857}  size = 1
        0 = "number"
        build = null
        run = {FreeStyleBuild@14834} "parameter #14"
     */
    project.getBuildersList().add(Functions.isWindows() ? new BatchFile("Ping -n %number% 127.0.0.1 > nul") : new Shell("sleep ${number}"));

    rs.add(project.scheduleBuild2(0, new ParametersAction(new StringParameterValue("number", "30"))));

    System.out.println("check if zero nodes!");
    Assert.assertEquals(0, j.jenkins.getNodes().size());

    assertAtLeastOneNode();

    final Node node = j.jenkins.getNodes().get(0);
    assertQueueIsEmpty();

    System.out.println("disconnect node");
    node.toComputer().disconnect(new OfflineCause.ChannelTermination(new UnsupportedOperationException("Test")));

    assertLastBuildResult(Result.FAILURE, Result.ABORTED);

    node.toComputer().connect(true);
    assertNodeIsOnline(node);
    assertQueueAndNodesIdle(node);

    Assert.assertEquals(1, j.jenkins.getProjects().size());
    Assert.assertEquals(Result.SUCCESS, j.jenkins.getProjects().get(0).getLastBuild().getResult());
    Assert.assertEquals(2, j.jenkins.getProjects().get(0).getBuilds().size());

    cancelTasks(rs);
}