com.amazonaws.services.autoscaling.model.Instance Java Examples

public boolean checkInstanceInServiceOnAllElb(Instance newInstance, List<String> loadBalancerNames) {
    if (newInstance == null) {
        throw new IllegalStateException("Unable to check null instance");
    }
    for (String elb : loadBalancerNames) {
        DescribeInstanceHealthResult result = awsElbClient.describeInstanceHealth(new DescribeInstanceHealthRequest(elb));
        Optional<InstanceState> state = result.getInstanceStates().stream().filter(s -> s.getInstanceId().equals(newInstance.getInstanceId())).findFirst();
        if (!state.isPresent()) {
            log.info("instance state for instance " + newInstance.getInstanceId() + " on elb " + elb + " is unknown");
            return false;
        }
        log.info("instance state for instance " + newInstance.getInstanceId() + " on elb " + elb + " is " + state.get().getState());
        if (!"InService".equals(state.get().getState())) {
            return false;
        }
    }
    return true;
}
 

public Map<Group, List<String>> getInstanceIdsByGroups(AmazonAutoScalingRetryClient amazonASClient, Map<String, Group> groupNameMapping) {
    DescribeAutoScalingGroupsResult result = amazonASClient
            .describeAutoScalingGroups(new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(groupNameMapping.keySet()));
    return result.getAutoScalingGroups().stream()
            .collect(Collectors.toMap(
                    ag -> groupNameMapping.get(ag.getAutoScalingGroupName()),
                    ag -> ag.getInstances().stream()
                            .filter(instance -> INSTANCE_LIFECYCLE_IN_SERVICE.equals(instance.getLifecycleState()))
                            .map(Instance::getInstanceId)
                            .collect(Collectors.toList())));
}
 

private List<Ec2Instance> checkInstances(AwsAutoScalingDeployUtils awsDeployUtils, AutoScalingGroup asGroup, List<Ec2Instance> instances) {
    List<String> removedInstances = asGroup.getInstances().stream()
            .filter(i -> i.getLifecycleState().equalsIgnoreCase(AwsState.STANDBY.name()))
            .map(Instance::getInstanceId)
            .filter(awsDeployUtils::checkEc2Instance)
            .collect(Collectors.toList());

    if (removedInstances != null && removedInstances.isEmpty()) {
        return instances.stream()
                .filter(i -> !removedInstances.contains(i.getInstanceId()))
                .collect(Collectors.toList());
    }
    return instances;
}
 

public boolean checkEc2Instance(String instanceId) {
    boolean instanceTerminated = false;
    try {
        DescribeInstancesResult result = awsEc2Client.describeInstances(new DescribeInstancesRequest().withInstanceIds(instanceId));
        List<com.amazonaws.services.ec2.model.Instance> instances = result.getReservations().stream()
                .flatMap(r -> r.getInstances().stream())
                .filter(i -> i.getInstanceId().equals(instanceId))
                .collect(Collectors.toList());
        instanceTerminated = instances.isEmpty() || instances.stream()
                .map(com.amazonaws.services.ec2.model.Instance::getState)
                .anyMatch(s -> s.getCode().equals(48));
    } catch (AmazonServiceException e) {
        log.info(e.toString(), e);
        if (e.getStatusCode() == 400) {
            instanceTerminated = true;
        }
    }

    if (instanceTerminated) {
        log.warn("Invalid instance " + instanceId + " in group " + activeConfiguration.getAutoScalingGroupId() + ". Detaching instance.");
        awsAsClient.detachInstances(new DetachInstancesRequest()
                .withAutoScalingGroupName(activeConfiguration.getAutoScalingGroupId())
                .withInstanceIds(instanceId)
                .withShouldDecrementDesiredCapacity(false));
    }
    return instanceTerminated;
}
 

public List<Ec2Instance> getInstancesForAutoScalingGroup(Log log, AutoScalingGroup autoScalingGroup) throws MojoFailureException {
    log.info("retrieving list of instanceId's for auto scaling group with id : " + activeConfiguration.getAutoScalingGroupId());

    activeConfiguration.getHosts().clear();

    log.debug("describing instances in auto scaling group");

    if (autoScalingGroup.getInstances().isEmpty()) {
        return new ArrayList<>();
    }

    Map<String, Instance> instanceMap = autoScalingGroup.getInstances().stream().collect(Collectors.toMap(Instance::getInstanceId, Function.identity()));

    try {
        DescribeInstancesResult instancesResult = awsEc2Client.describeInstances(new DescribeInstancesRequest().withInstanceIds(autoScalingGroup.getInstances().stream().map(Instance::getInstanceId).collect(Collectors.toList())));
        List<Ec2Instance> ec2Instances = instancesResult.getReservations().stream().flatMap(r -> r.getInstances().stream()).map(this::toEc2Instance).collect(Collectors.toList());
        log.debug("describing elb status");
        autoScalingGroup.getLoadBalancerNames().forEach(elb -> this.updateInstancesStateOnLoadBalancer(elb, ec2Instances));
        ec2Instances.forEach(i -> i.updateAsState(AwsState.map(instanceMap.get(i.getInstanceId()).getLifecycleState())));
        ec2Instances.sort((o1, o2) -> {

            int sComp = o1.getAsState().compareTo(o2.getAsState());

            if (sComp != 0) {
                return sComp;
            } else {
                return o1.getElbState().compareTo(o2.getElbState());
            }
        });
        if (activeConfiguration.isIgnoreInStandby()) {
            return ec2Instances.stream().filter(i -> i.getAsState() != AwsState.STANDBY).collect(Collectors.toList());
        }
        return ec2Instances;
    } catch (AmazonClientException e) {
        log.error(e.getMessage(), e);
        throw new MojoFailureException(e.getMessage());
    }

}
 

private void upscaleStack() throws Exception {
    AuthenticatedContext authenticatedContext = componentTestUtil.getAuthenticatedContext();
    CloudStack stack = componentTestUtil.getStack(InstanceStatus.CREATE_REQUESTED, InstanceStatus.STARTED);
    List<CloudResource> cloudResources = List.of(
            CloudResource.builder()
                    .name(AWS_SUBNET_ID)
                    .type(ResourceType.AWS_SUBNET)
                    .build(),
            createVolumeResource(VOLUME_ID_1, INSTANCE_ID_1, SIZE_DISK_1, FSTAB_1, CommonStatus.DETACHED),
            createVolumeResource(VOLUME_ID_2, INSTANCE_ID_2, SIZE_DISK_2, FSTAB_2, CommonStatus.DETACHED),
            createVolumeResource(VOLUME_ID_3, INSTANCE_ID_3, SIZE_DISK_2, FSTAB_2, CommonStatus.CREATED));

    InMemoryStateStore.putStack(1L, PollGroup.POLLABLE);

    when(amazonCloudFormationRetryClient.describeStackResource(any()))
            .thenReturn(new DescribeStackResourceResult()
                    .withStackResourceDetail(new StackResourceDetail().withPhysicalResourceId(AUTOSCALING_GROUP_NAME)));

    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any()))
            .thenReturn(new DescribeAutoScalingGroupsResult()
                    .withAutoScalingGroups(new AutoScalingGroup()
                            .withAutoScalingGroupName(AUTOSCALING_GROUP_NAME)
                            .withInstances(List.of(
                                    new Instance().withInstanceId(INSTANCE_ID_1).withLifecycleState(LifecycleState.InService),
                                    new Instance().withInstanceId(INSTANCE_ID_2).withLifecycleState(LifecycleState.InService)))
                    ));

    when(amazonEC2Client.describeVolumes(any()))
            .thenReturn(new DescribeVolumesResult().withVolumes(
                    new com.amazonaws.services.ec2.model.Volume().withVolumeId(VOLUME_ID_1).withState(VolumeState.Available),
                    new com.amazonaws.services.ec2.model.Volume().withVolumeId(VOLUME_ID_2).withState(VolumeState.Available),
                    new com.amazonaws.services.ec2.model.Volume().withVolumeId(VOLUME_ID_3).withState(VolumeState.InUse)
            ));

    when(amazonEC2Client.describeInstances(any())).thenReturn(
            new DescribeInstancesResult().withReservations(
                    new Reservation().withInstances(new com.amazonaws.services.ec2.model.Instance().withInstanceId("i-instance")))
    );


    AmazonEC2Waiters waiters = mock(AmazonEC2Waiters.class);
    when(amazonEC2Client.waiters()).thenReturn(waiters);
    Waiter<DescribeInstancesRequest> instanceWaiter = mock(Waiter.class);
    when(waiters.instanceRunning()).thenReturn(instanceWaiter);

    when(amazonAutoScalingClient.waiters()).thenReturn(asWaiters);
    when(asWaiters.groupInService()).thenReturn(describeAutoScalingGroupsRequestWaiter);

    underTest.upscale(authenticatedContext, stack, cloudResources);

    verify(amazonAutoScalingRetryClient).resumeProcesses(argThat(argument -> AUTOSCALING_GROUP_NAME.equals(argument.getAutoScalingGroupName())
            && argument.getScalingProcesses().contains("Launch")));
    verify(amazonAutoScalingRetryClient).updateAutoScalingGroup(argThat(argument -> {
        Group workerGroup = stack.getGroups().get(1);
        return AUTOSCALING_GROUP_NAME.equals(argument.getAutoScalingGroupName())
                && workerGroup.getInstancesSize().equals(argument.getMaxSize())
                && workerGroup.getInstancesSize().equals(argument.getDesiredCapacity());
    }));

    verify(amazonAutoScalingRetryClient, times(stack.getGroups().size()))
            .suspendProcesses(argThat(argument -> AUTOSCALING_GROUP_NAME.equals(argument.getAutoScalingGroupName())
                    && SUSPENDED_PROCESSES.equals(argument.getScalingProcesses())));

    ArgumentCaptor<CloudResource> updatedCloudResourceArgumentCaptor = ArgumentCaptor.forClass(CloudResource.class);
    verify(resourceNotifier, times(4)).notifyUpdate(updatedCloudResourceArgumentCaptor.capture(), any());

    assertVolumeResource(updatedCloudResourceArgumentCaptor.getAllValues(), INSTANCE_ID_1, SIZE_DISK_1, FSTAB_1);
    assertVolumeResource(updatedCloudResourceArgumentCaptor.getAllValues(), INSTANCE_ID_2, SIZE_DISK_2, FSTAB_2);
}
 

private void setupAutoscalingResponses() {
    DescribeScalingActivitiesResult describeScalingActivitiesResult = new DescribeScalingActivitiesResult();
    when(amazonAutoScalingRetryClient.describeScalingActivities(any())).thenReturn(describeScalingActivitiesResult);

    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult()
            .withAutoScalingGroups(
                    new AutoScalingGroup()
                            .withInstances(new Instance().withLifecycleState(LifecycleState.InService).withInstanceId(INSTANCE_ID))
                            .withAutoScalingGroupName(AUTOSCALING_GROUP_NAME)
            );
    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any())).thenReturn(describeAutoScalingGroupsResult);
}
 

public List<String> getInstanceIds(AmazonAutoScalingRetryClient amazonASClient, String asGroupName) {
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = amazonASClient
            .describeAutoScalingGroups(new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(asGroupName));
    List<String> instanceIds = new ArrayList<>();
    if (!describeAutoScalingGroupsResult.getAutoScalingGroups().isEmpty()
            && describeAutoScalingGroupsResult.getAutoScalingGroups().get(0).getInstances() != null) {
        for (Instance instance : describeAutoScalingGroupsResult.getAutoScalingGroups().get(0).getInstances()) {
            if (INSTANCE_LIFECYCLE_IN_SERVICE.equals(instance.getLifecycleState())) {
                instanceIds.add(instance.getInstanceId());
            }
        }
    }
    return instanceIds;
}
 

@Override
public Map<String, List<String>> getRacMembership(Collection<String> autoScalingGroupNames) {
    if (CollectionUtils.isEmpty(autoScalingGroupNames)) {
        return Collections.emptyMap();
    }

    AmazonAutoScaling client = null;

    try {
        client = getAutoScalingClient();
        DescribeAutoScalingGroupsRequest describeAutoScalingGroupsRequest =
                new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(autoScalingGroupNames);
        DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = client.describeAutoScalingGroups(describeAutoScalingGroupsRequest);

        Map<String, List<String>> asgs = new HashMap<>();
        for (AutoScalingGroup autoScalingGroup : describeAutoScalingGroupsResult.getAutoScalingGroups()) {
            List<String> asgInstanceIds = Lists.newArrayList();
            for (Instance asgInstance : autoScalingGroup.getInstances()) {
                if (!(asgInstance.getLifecycleState().equalsIgnoreCase("terminating") ||
                        asgInstance.getLifecycleState().equalsIgnoreCase("shutting-down") ||
                        asgInstance.getLifecycleState().equalsIgnoreCase("terminated"))) {
                    asgInstanceIds.add(asgInstance.getInstanceId());
                }
            }
            asgs.put(autoScalingGroup.getAutoScalingGroupName(), asgInstanceIds);
            logger.info("AWS returned the following instance ID's for {} ASG: {}",
                    autoScalingGroup.getAutoScalingGroupName(),
                    StringUtils.join(asgInstanceIds, ","));
        }

        return asgs;
    } finally {
        if (client != null) {
            client.shutdown();
        }
    }
}
 

@Test
void upscaleAwsASGroupFail() throws AmazonAutoscalingFailed {
    AmazonAutoScalingRetryClient amazonAutoScalingRetryClient = mock(AmazonAutoScalingRetryClient.class);
    AmazonCloudFormationRetryClient amazonCloudFormationRetryClient = mock(AmazonCloudFormationRetryClient.class);
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    List<AutoScalingGroup> autoScalingGroups = new ArrayList<>();

    AutoScalingGroup masterASGroup = new AutoScalingGroup();
    masterASGroup.setAutoScalingGroupName("masterASG");
    List<Instance> masterASGInstances = new ArrayList<>();
    masterASGInstances.add(new Instance().withInstanceId("i-master1"));
    masterASGInstances.add(new Instance().withInstanceId("i-master2"));
    masterASGroup.setInstances(masterASGInstances);

    AutoScalingGroup workerASGroup = new AutoScalingGroup();
    workerASGroup.setAutoScalingGroupName("workerASG");
    List<Instance> workerASGInstances = new ArrayList<>();
    workerASGInstances.add(new Instance().withInstanceId("i-worker1"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker2"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker3"));
    workerASGroup.setInstances(workerASGInstances);

    autoScalingGroups.add(masterASGroup);
    autoScalingGroups.add(workerASGroup);

    describeAutoScalingGroupsResult.setAutoScalingGroups(autoScalingGroups);
    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any(DescribeAutoScalingGroupsRequest.class)))
            .thenReturn(describeAutoScalingGroupsResult);
    when(awsClient.createAutoScalingRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingRetryClient);
    when(awsClient.createCloudFormationRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationRetryClient);

    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("worker")))
            .thenReturn("workerASG");
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("master")))
            .thenReturn("masterASG");

    AuthenticatedContext authenticatedContext = new AuthenticatedContext(new CloudContext(1L, "teststack", "AWS", "AWS",
            Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a")), "1", "1"), new CloudCredential());

    List<CloudResource> allInstances = new ArrayList<>();
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker1").group("worker").instanceId("i-worker1").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker2").group("worker").instanceId("i-worker2").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker3").group("worker").instanceId("i-worker3").build());
    CloudResource workerInstance4 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker4").group("worker").instanceId("i-worker4").build();
    allInstances.add(workerInstance4);
    CloudResource workerInstance5 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker5").group("worker").instanceId("i-worker5").build();
    allInstances.add(workerInstance5);
    when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationRetryClient), eq(amazonAutoScalingRetryClient), anyList()))
            .thenReturn(allInstances);

    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();

    Group master = getMasterGroup(instanceAuthentication);
    groups.add(master);

    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);

    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), emptyMap(), null,
            instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null);

    List<CloudResource> cloudResourceList = Collections.emptyList();

    AutoScalingGroup newWorkerASGroup = new AutoScalingGroup();
    newWorkerASGroup.setAutoScalingGroupName("workerASG");
    List<Instance> newWorkerASGInstances = new ArrayList<>();
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker1"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker2"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker3"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker4"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker5"));
    newWorkerASGroup.setInstances(newWorkerASGInstances);

    when(awsAutoScalingService.getAutoscalingGroups(eq(amazonAutoScalingRetryClient), any()))
            .thenReturn(Collections.singletonList(newWorkerASGroup));

    doThrow(new AmazonAutoscalingFailed("autoscaling failed"))
            .when(awsAutoScalingService).scheduleStatusChecks(eq(List.of(worker)),
            eq(authenticatedContext), eq(amazonCloudFormationRetryClient), any(Date.class));

    assertThrows(CloudConnectorException.class, () -> awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList),
            "Autoscaling group update failed: 'autoscaling failed' Original autoscaling group state has been recovered.");
    verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingRetryClient.class), eq("workerASG"), eq(5));
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext),  eq(amazonCloudFormationRetryClient), any());
    verify(awsComputeResourceService, times(0)).buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack),
            anyList(), anyList(), any(), any());
    verify(awsAutoScalingService, times(1)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingRetryClient), eq("i-worker4"));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingRetryClient), eq("i-worker5"));
    Map<String, Integer> desiredGroups = new HashMap<>();
    desiredGroups.put("workerASG", 3);
    desiredGroups.put("masterASG", 2);
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(desiredGroups), eq(authenticatedContext), any(Date.class));
}
 

private void setupDescribeInstancesResponse() {
    when(amazonEC2Client.describeInstances(any())).thenReturn(
            new DescribeInstancesResult().withReservations(
                    new Reservation().withInstances(new com.amazonaws.services.ec2.model.Instance().withInstanceId("i-instance")))
    );
}
 

@Test
void upscaleTest() throws AmazonAutoscalingFailed {
    AmazonAutoScalingRetryClient amazonAutoScalingRetryClient = mock(AmazonAutoScalingRetryClient.class);
    AmazonCloudFormationRetryClient amazonCloudFormationRetryClient = mock(AmazonCloudFormationRetryClient.class);
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    List<AutoScalingGroup> autoScalingGroups = new ArrayList<>();

    AutoScalingGroup masterASGroup = new AutoScalingGroup();
    masterASGroup.setAutoScalingGroupName("masterASG");
    List<Instance> masterASGInstances = new ArrayList<>();
    masterASGInstances.add(new Instance().withInstanceId("i-master1"));
    masterASGInstances.add(new Instance().withInstanceId("i-master2"));
    masterASGroup.setInstances(masterASGInstances);

    AutoScalingGroup workerASGroup = new AutoScalingGroup();
    workerASGroup.setAutoScalingGroupName("workerASG");
    List<Instance> workerASGInstances = new ArrayList<>();
    workerASGInstances.add(new Instance().withInstanceId("i-worker1"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker2"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker3"));
    workerASGroup.setInstances(workerASGInstances);

    autoScalingGroups.add(masterASGroup);
    autoScalingGroups.add(workerASGroup);

    describeAutoScalingGroupsResult.setAutoScalingGroups(autoScalingGroups);
    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any(DescribeAutoScalingGroupsRequest.class)))
            .thenReturn(describeAutoScalingGroupsResult);
    when(awsClient.createAutoScalingRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingRetryClient);
    when(awsClient.createCloudFormationRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationRetryClient);
    when(awsClient.createAccess(any(), any())).thenReturn(new AmazonEC2Client());

    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("worker")))
            .thenReturn("workerASG");
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("master")))
            .thenReturn("masterASG");

    AuthenticatedContext authenticatedContext = new AuthenticatedContext(new CloudContext(1L, "teststack", "AWS", "AWS",
            Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a")), "1", "1"), new CloudCredential());

    ArrayList<CloudResource> allInstances = new ArrayList<>();
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker1").group("worker").instanceId("i-worker1").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker2").group("worker").instanceId("i-worker2").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker3").group("worker").instanceId("i-worker3").build());
    CloudResource workerInstance4 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker4").group("worker").instanceId("i-worker4").build();
    allInstances.add(workerInstance4);
    CloudResource workerInstance5 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker5").group("worker").instanceId("i-worker5").build();
    allInstances.add(workerInstance5);
    when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationRetryClient), eq(amazonAutoScalingRetryClient), anyList()))
            .thenReturn(allInstances);

    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();

    Group master = getMasterGroup(instanceAuthentication);
    groups.add(master);

    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);

    Map<String, String> tags = new HashMap<>();
    tags.put("owner", "cbuser");
    tags.put("created", "yesterday");
    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), tags, null,
            instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null);

    List<CloudResource> cloudResourceList = Collections.emptyList();
    awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList);
    verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingRetryClient.class), eq("workerASG"), eq(5));
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext),  eq(amazonCloudFormationRetryClient), any());
    verify(awsAutoScalingService, times(1)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack));
    ArgumentCaptor<List<CloudResource>> captor = ArgumentCaptor.forClass(List.class);
    verify(awsComputeResourceService, times(1))
            .buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack), anyList(), captor.capture(), any(), any());
    verify(awsTaggingService, times(1)).tagRootVolumes(eq(authenticatedContext), any(AmazonEC2Client.class), eq(allInstances), eq(tags));
    verify(awsCloudWatchService, times(1)).addCloudWatchAlarmsForSystemFailures(any(), eq(cloudStack), eq("eu-west-1"),
            any(AwsCredentialView.class));
    List<CloudResource> newInstances = captor.getValue();
    assertEquals("Two new instances should be created", 2, newInstances.size());
    assertThat(newInstances, hasItem(workerInstance4));
    assertThat(newInstances, hasItem(workerInstance5));
}
 

private List<Instance> getInstancesInAutoscalingGroups(AmazonAutoScalingRetryClient amazonASClient, Set<String> autoscalingGroupNames) {
    List<AutoScalingGroup> autoscalingGroups = awsAutoScalingService.getAutoscalingGroups(amazonASClient, autoscalingGroupNames);
    return autoscalingGroups.stream()
            .flatMap(autoScalingGroup -> autoScalingGroup.getInstances().stream()).collect(Collectors.toList());
}
 

private List<String> getUnknownInstancesFromASGs(List<Instance> instancesFromASGs, List<String> knownInstanceIdsByCloudbreak) {
    return instancesFromASGs.stream()
                    .filter(instance -> !knownInstanceIdsByCloudbreak.contains(instance.getInstanceId()))
                    .map(Instance::getInstanceId)
                    .collect(Collectors.toList());
}
 

private Ec2Instance toEc2Instance(com.amazonaws.services.ec2.model.Instance instance) {
    return new Ec2Instance.Builder().withInstanceId(instance.getInstanceId()).withPrivateIp(instance.getPrivateIpAddress()).withPublicIp(instance.getPublicIpAddress()).build();
}
 

private List<CloudResourceStatus> recoverOriginalState(AuthenticatedContext ac,
        CloudStack stack,
        AmazonAutoScalingRetryClient amazonASClient,
        Map<String, Group> desiredAutoscalingGroupsByName,
        Map<String, Integer> originalAutoScalingGroupsBySize,
        AmazonAutoscalingFailed amazonAutoscalingFailedException) {
    LOGGER.info("Amazon autoscaling group update failed, suspend autoscaling", amazonAutoscalingFailedException);
    try {
        awsAutoScalingService.suspendAutoScaling(ac, stack);
        List<Instance> instancesFromASGs = getInstancesInAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName.keySet());
        LOGGER.info("Instances from ASGs: {}", instancesFromASGs);
        List<String> knownInstanceIdsByCloudbreak = getKnownInstancesByCloudbreak(stack);
        LOGGER.info("Known instance ids by cloudbreak: {}", knownInstanceIdsByCloudbreak);
        List<String> unknownInstancesByCloudbreak = getUnknownInstancesFromASGs(instancesFromASGs, knownInstanceIdsByCloudbreak);
        LOGGER.info("Unknown instances by cloudbreak: {}", unknownInstancesByCloudbreak);
        if (!unknownInstancesByCloudbreak.isEmpty()) {
            for (String unknownInstance : unknownInstancesByCloudbreak) {
                LOGGER.info("Terminate unknown instance: {}", unknownInstance);
                awsAutoScalingService.terminateInstance(amazonASClient, unknownInstance);
            }
        }
        Date timeBeforeASUpdate = new Date();
        for (Entry<String, Group> desiredAutoscalingGroup : desiredAutoscalingGroupsByName.entrySet()) {
            String autoscalingGroupName = desiredAutoscalingGroup.getKey();
            Integer originalInstanceSizeForTheGroup = originalAutoScalingGroupsBySize.get(autoscalingGroupName);
            LOGGER.info("Original instance size for the group: {}", originalInstanceSizeForTheGroup);
            Integer desiredInstanceSize = desiredAutoscalingGroup.getValue().getInstancesSize();
            if (originalInstanceSizeForTheGroup < desiredInstanceSize) {
                LOGGER.info("Restore original instance size [{}] for autoscaling group: {}", originalInstanceSizeForTheGroup, autoscalingGroupName);
                awsAutoScalingService.updateAutoscalingGroup(amazonASClient, autoscalingGroupName, originalInstanceSizeForTheGroup);
            }
        }
        awsAutoScalingService.scheduleStatusChecks(originalAutoScalingGroupsBySize, ac, timeBeforeASUpdate);
    } catch (Exception autoscalingFailed) {
        autoscalingFailed.addSuppressed(amazonAutoscalingFailedException);
        LOGGER.info("AWS Autoscaling group update failed, original autoscaling group state recover is failed", autoscalingFailed);
        throw new CloudConnectorException("AWS Autoscaling Group update failed: '" + amazonAutoscalingFailedException.getMessage() +
                "'. We tried to recover to the original state, but it failed also: '" + autoscalingFailed.getMessage() +
                "' Please check your autoscaling groups on AWS.", autoscalingFailed);
    }
    LOGGER.info("AWS Autoscaling group update failed, original autoscaling group state has been recovered");
    throw new CloudConnectorException("Autoscaling group update failed: '" + amazonAutoscalingFailedException.getMessage() +
            "' Original autoscaling group state has been recovered.", amazonAutoscalingFailedException);
}
 

private Instance findNewInstance(AutoScalingGroup originalGroup, AutoScalingGroup updatedGroup) {
    updatedGroup.getInstances().removeAll(originalGroup.getInstances());
    return updatedGroup.getInstances().isEmpty() ? null : updatedGroup.getInstances().get(0);
}
 

public void executeRequest(final AutoScalingGroup autoScalingGroup, AwsAutoScalingDeployUtils awsDeployUtils, InstanceStatus instanceStatus) {
    final AtomicInteger waitFor = new AtomicInteger(1);
    final AtomicBoolean inService = new AtomicBoolean(false);
    final AtomicBoolean found = new AtomicBoolean(false);


    log.info("Waiting for new instance in asGroup to come in service...");
    ScheduledExecutorService exec = Executors.newSingleThreadScheduledExecutor();

    exec.scheduleAtFixedRate(() -> {
        log.debug("existing instances : " + Arrays.toString(autoScalingGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));
        AutoScalingGroup updatedGroup = awsDeployUtils.getAutoScalingGroup();
        log.debug("Updated instances : " + Arrays.toString(updatedGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));

        if (updatedGroup.getInstances().equals(autoScalingGroup.getInstances())) {
            log.info("no new instance found in auto scaling group.");
        }

        if (newInstance == null) {
            newInstance = findNewInstance(autoScalingGroup, updatedGroup);
            if (newInstance != null && !found.get()) {
                found.set(true);
                log.info("Found new instance with id " + newInstance.getInstanceId());
            }
        }
        if (newInstance != null && instanceStatus.inService(newInstance)) {
            waitFor.decrementAndGet();
        }
    }, 30, 30, TimeUnit.SECONDS);

    try {
        while (waitFor.intValue() > 0 && System.currentTimeMillis() <= timeout) {
            Thread.sleep(3000);
        }
        log.info("Shutting down executor");
        exec.shutdown();
        log.info("awaiting termination of executor");
        exec.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        log.error(e.getMessage());
        Thread.currentThread().interrupt();
        inService.get();
    } catch (Exception t) {
        log.error("Throwable: ", t);
    }
}
 

Boolean inService(Instance newInstance);