Java Code Examples for com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsResult#setAutoScalingGroups()

/**
 * Fetch launch configurations test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchLaunchConfigurationsTest() throws Exception {
    
    mockStatic(AmazonAutoScalingClientBuilder.class);
    AmazonAutoScaling asgClient = PowerMockito.mock(AmazonAutoScaling.class);
    AmazonAutoScalingClientBuilder amazonAutoScalingClientBuilder = PowerMockito.mock(AmazonAutoScalingClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonAutoScalingClientBuilder.standard()).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.withCredentials(anyObject())).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.withRegion(anyString())).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.build()).thenReturn(asgClient);
    
    DescribeAutoScalingGroupsResult autoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    List<AutoScalingGroup> asgList = new ArrayList<>();
    asgList.add(new AutoScalingGroup());
    autoScalingGroupsResult.setAutoScalingGroups(asgList);
    when(asgClient.describeAutoScalingGroups(anyObject())).thenReturn(autoScalingGroupsResult);
    assertThat(inventoryUtil.fetchAsg(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
    
}
 

@Test
public void testTerminateWhenResourcesHasNoCfButStackNotExist() {
    CloudResource cf = new Builder().name("cfn-87654321").type(ResourceType.CLOUDFORMATION_STACK).build();
    CloudResource lc = new Builder().name("lc-87654321").type(ResourceType.AWS_LAUNCHCONFIGURATION).build();
    Group group = new Group("alma", InstanceGroupType.GATEWAY, List.of(), null, null, null, null, "", 0, Optional.empty());
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    describeAutoScalingGroupsResult.setAutoScalingGroups(List.of());

    when(cloudStack.getGroups()).thenReturn(List.of(group));
    when(cfStackUtil.getCloudFormationStackResource(any())).thenReturn(cf);
    when(retryService.testWith2SecDelayMax15Times(any())).thenThrow(new Retry.ActionFailedException("Fail no more"));
    when(awsClient.createAutoScalingClient(any(), any())).thenReturn(amazonAutoScalingClient);

    List<CloudResourceStatus> result = underTest.terminate(authenticatedContext(), cloudStack, List.of(
            new Builder().name("ami-87654321").type(ResourceType.AWS_ENCRYPTED_AMI).build(), cf, lc
    ));
    verify(awsResourceConnector, times(1)).check(any(), any());
    verify(awsComputeResourceService, times(1)).deleteComputeResources(any(), any(), any());
    verify(encryptedImageCopyService, times(1)).deleteResources(any(), any(), any());
    verify(cloudFormationRetryClient, never()).deleteStack(any());
    verify(amazonAutoScalingRetryClient, never()).describeAutoScalingGroups(any());
    Assertions.assertEquals(0, result.size(), "Resources result should be empty");

}
 

@Test
public void testTerminateWhenResourcesHasNoCfButStackNotExist() {
    CloudResource cf = new Builder().name("cfn-87654321").type(ResourceType.CLOUDFORMATION_STACK).build();
    CloudResource lc = new Builder().name("lc-87654321").type(ResourceType.AWS_LAUNCHCONFIGURATION).build();
    Group group = new Group("alma", InstanceGroupType.GATEWAY, List.of(), null, null, null, null, "", 0, Optional.empty());
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    describeAutoScalingGroupsResult.setAutoScalingGroups(List.of());

    when(cloudStack.getGroups()).thenReturn(List.of(group));
    when(cfStackUtil.getCloudFormationStackResource(any())).thenReturn(cf);
    when(retryService.testWith2SecDelayMax15Times(any())).thenThrow(new Retry.ActionFailedException("Fail no more"));
    when(awsClient.createAutoScalingClient(any(), any())).thenReturn(amazonAutoScalingClient);

    List<CloudResourceStatus> result = underTest.terminate(authenticatedContext(), cloudStack, List.of(
            new Builder().name("ami-87654321").type(ResourceType.AWS_ENCRYPTED_AMI).build(), cf, lc
    ));
    verify(awsResourceConnector, times(1)).check(any(), any());
    verify(awsComputeResourceService, times(1)).deleteComputeResources(any(), any(), any());
    verify(encryptedImageCopyService, times(1)).deleteResources(any(), any(), any());
    verify(cloudFormationRetryClient, never()).deleteStack(any());
    verify(amazonAutoScalingRetryClient, never()).describeAutoScalingGroups(any());
    Assertions.assertEquals(0, result.size(), "Resources result should be empty");

}
 

/**
 * Fetch launch configurations test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchLaunchConfigurationsTest() throws Exception {
    
    mockStatic(AmazonAutoScalingClientBuilder.class);
    AmazonAutoScaling asgClient = PowerMockito.mock(AmazonAutoScaling.class);
    AmazonAutoScalingClientBuilder amazonAutoScalingClientBuilder = PowerMockito.mock(AmazonAutoScalingClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonAutoScalingClientBuilder.standard()).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.withCredentials(anyObject())).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.withRegion(anyString())).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.build()).thenReturn(asgClient);
    
    DescribeAutoScalingGroupsResult autoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    List<AutoScalingGroup> asgList = new ArrayList<>();
    AutoScalingGroup autoScalingGroup = new AutoScalingGroup();
    autoScalingGroup.setLaunchConfigurationName("launchConfigurationName");
    asgList.add(autoScalingGroup);
    autoScalingGroupsResult.setAutoScalingGroups(asgList);
    when(asgClient.describeAutoScalingGroups(anyObject())).thenReturn(autoScalingGroupsResult);
    
    DescribeLaunchConfigurationsResult launchConfigurationsResult = new DescribeLaunchConfigurationsResult();
    List<LaunchConfiguration> launchConfigurations = new ArrayList<>();
    launchConfigurations.add(new LaunchConfiguration());
    
    launchConfigurationsResult.setLaunchConfigurations(launchConfigurations);
    when(asgClient.describeLaunchConfigurations(anyObject())).thenReturn(launchConfigurationsResult);
    assertThat(asgInventoryUtil.fetchLaunchConfigurations(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
    
}
 

/**
 * Fetch scaling policies test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchScalingPoliciesTest() throws Exception {
    
    mockStatic(AmazonAutoScalingClientBuilder.class);
    AmazonAutoScaling asgClient = PowerMockito.mock(AmazonAutoScaling.class);
    AmazonAutoScalingClientBuilder amazonAutoScalingClientBuilder = PowerMockito.mock(AmazonAutoScalingClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonAutoScalingClientBuilder.standard()).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.withCredentials(anyObject())).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.withRegion(anyString())).thenReturn(amazonAutoScalingClientBuilder);
    when(amazonAutoScalingClientBuilder.build()).thenReturn(asgClient);
    
    DescribeAutoScalingGroupsResult autoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    List<AutoScalingGroup> asgList = new ArrayList<>();
    AutoScalingGroup autoScalingGroup = new AutoScalingGroup();
    autoScalingGroup.setAutoScalingGroupName("autoScalingGrpName");
    asgList.add(autoScalingGroup);
    autoScalingGroupsResult.setAutoScalingGroups(asgList);
    
    when(asgClient.describeAutoScalingGroups(anyObject())).thenReturn(autoScalingGroupsResult);
    
    DescribePoliciesResult policiesResult = new DescribePoliciesResult();
    List<ScalingPolicy> scalingPolicies = new ArrayList<>();
    scalingPolicies.add(new ScalingPolicy());
    policiesResult.setScalingPolicies(scalingPolicies);
    when(asgClient.describePolicies(anyObject())).thenReturn(policiesResult);
    assertThat(asgInventoryUtil.fetchScalingPolicies(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
}
 

@Test
public void testTerminateWhenResourcesHasCf() {
    CloudResource cf = new Builder().name("cfn-87654321").type(ResourceType.CLOUDFORMATION_STACK).build();
    CloudResource lc = new Builder().name("lc-87654321").type(ResourceType.AWS_LAUNCHCONFIGURATION).build();
    Group group = new Group("alma", InstanceGroupType.GATEWAY, List.of(), null, null, null, null, "", 0, Optional.empty());
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    describeAutoScalingGroupsResult.setAutoScalingGroups(List.of());

    when(cloudStack.getGroups()).thenReturn(List.of(group));
    when(cfStackUtil.getCloudFormationStackResource(any())).thenReturn(cf);
    when(cfStackUtil.getAutoscalingGroupName(any(), anyString(), anyString())).thenReturn("alma");
    when(awsClient.createCloudFormationRetryClient(any())).thenReturn(cloudFormationRetryClient);
    when(awsClient.createCloudFormationClient(any(), any())).thenReturn(cloudFormationClient);
    when(awsClient.createAutoScalingClient(any(), any())).thenReturn(amazonAutoScalingClient);
    when(awsClient.createAutoScalingRetryClient(any(), any())).thenReturn(amazonAutoScalingRetryClient);
    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any())).thenReturn(describeAutoScalingGroupsResult);
    when(cloudFormationClient.waiters()).thenReturn(cfWaiters);
    when(cfWaiters.stackDeleteComplete()).thenReturn(deletionWaiter);

    List<CloudResourceStatus> result = underTest.terminate(authenticatedContext(), cloudStack, List.of(
            new Builder().name("ami-87654321").type(ResourceType.AWS_ENCRYPTED_AMI).build(), cf, lc
    ));

    verify(awsResourceConnector, times(1)).check(any(), any());
    verify(awsComputeResourceService, times(1)).deleteComputeResources(any(), any(), any());
    verify(encryptedImageCopyService, times(1)).deleteResources(any(), any(), any());
    verify(cloudFormationRetryClient, times(1)).deleteStack(any());
    verify(amazonAutoScalingRetryClient, times(1)).describeAutoScalingGroups(any());
    Assertions.assertEquals(0, result.size(), "Resources result should have one size list");
}
 

@Test
public void testTerminateWhenResourcesHasCf() {
    CloudResource cf = new Builder().name("cfn-87654321").type(ResourceType.CLOUDFORMATION_STACK).build();
    CloudResource lc = new Builder().name("lc-87654321").type(ResourceType.AWS_LAUNCHCONFIGURATION).build();
    Group group = new Group("alma", InstanceGroupType.GATEWAY, List.of(), null, null, null, null, "", 0, Optional.empty());
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    describeAutoScalingGroupsResult.setAutoScalingGroups(List.of());

    when(cloudStack.getGroups()).thenReturn(List.of(group));
    when(cfStackUtil.getCloudFormationStackResource(any())).thenReturn(cf);
    when(cfStackUtil.getAutoscalingGroupName(any(), anyString(), anyString())).thenReturn("alma");
    when(awsClient.createCloudFormationRetryClient(any())).thenReturn(cloudFormationRetryClient);
    when(awsClient.createCloudFormationClient(any(), any())).thenReturn(cloudFormationClient);
    when(awsClient.createAutoScalingClient(any(), any())).thenReturn(amazonAutoScalingClient);
    when(awsClient.createAutoScalingRetryClient(any(), any())).thenReturn(amazonAutoScalingRetryClient);
    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any())).thenReturn(describeAutoScalingGroupsResult);
    when(cloudFormationClient.waiters()).thenReturn(cfWaiters);
    when(cfWaiters.stackDeleteComplete()).thenReturn(deletionWaiter);

    List<CloudResourceStatus> result = underTest.terminate(authenticatedContext(), cloudStack, List.of(
            new Builder().name("ami-87654321").type(ResourceType.AWS_ENCRYPTED_AMI).build(), cf, lc
    ));

    verify(awsResourceConnector, times(1)).check(any(), any());
    verify(awsComputeResourceService, times(1)).deleteComputeResources(any(), any(), any());
    verify(encryptedImageCopyService, times(1)).deleteResources(any(), any(), any());
    verify(cloudFormationRetryClient, times(1)).deleteStack(any());
    verify(amazonAutoScalingRetryClient, times(1)).describeAutoScalingGroups(any());
    Assertions.assertEquals(0, result.size(), "Resources result should have one size list");
}
 

@Test
void upscaleTest() throws AmazonAutoscalingFailed {
    AmazonAutoScalingRetryClient amazonAutoScalingRetryClient = mock(AmazonAutoScalingRetryClient.class);
    AmazonCloudFormationRetryClient amazonCloudFormationRetryClient = mock(AmazonCloudFormationRetryClient.class);
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    List<AutoScalingGroup> autoScalingGroups = new ArrayList<>();

    AutoScalingGroup masterASGroup = new AutoScalingGroup();
    masterASGroup.setAutoScalingGroupName("masterASG");
    List<Instance> masterASGInstances = new ArrayList<>();
    masterASGInstances.add(new Instance().withInstanceId("i-master1"));
    masterASGInstances.add(new Instance().withInstanceId("i-master2"));
    masterASGroup.setInstances(masterASGInstances);

    AutoScalingGroup workerASGroup = new AutoScalingGroup();
    workerASGroup.setAutoScalingGroupName("workerASG");
    List<Instance> workerASGInstances = new ArrayList<>();
    workerASGInstances.add(new Instance().withInstanceId("i-worker1"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker2"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker3"));
    workerASGroup.setInstances(workerASGInstances);

    autoScalingGroups.add(masterASGroup);
    autoScalingGroups.add(workerASGroup);

    describeAutoScalingGroupsResult.setAutoScalingGroups(autoScalingGroups);
    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any(DescribeAutoScalingGroupsRequest.class)))
            .thenReturn(describeAutoScalingGroupsResult);
    when(awsClient.createAutoScalingRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingRetryClient);
    when(awsClient.createCloudFormationRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationRetryClient);
    when(awsClient.createAccess(any(), any())).thenReturn(new AmazonEC2Client());

    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("worker")))
            .thenReturn("workerASG");
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("master")))
            .thenReturn("masterASG");

    AuthenticatedContext authenticatedContext = new AuthenticatedContext(new CloudContext(1L, "teststack", "AWS", "AWS",
            Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a")), "1", "1"), new CloudCredential());

    ArrayList<CloudResource> allInstances = new ArrayList<>();
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker1").group("worker").instanceId("i-worker1").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker2").group("worker").instanceId("i-worker2").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker3").group("worker").instanceId("i-worker3").build());
    CloudResource workerInstance4 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker4").group("worker").instanceId("i-worker4").build();
    allInstances.add(workerInstance4);
    CloudResource workerInstance5 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker5").group("worker").instanceId("i-worker5").build();
    allInstances.add(workerInstance5);
    when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationRetryClient), eq(amazonAutoScalingRetryClient), anyList()))
            .thenReturn(allInstances);

    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();

    Group master = getMasterGroup(instanceAuthentication);
    groups.add(master);

    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);

    Map<String, String> tags = new HashMap<>();
    tags.put("owner", "cbuser");
    tags.put("created", "yesterday");
    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), tags, null,
            instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null);

    List<CloudResource> cloudResourceList = Collections.emptyList();
    awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList);
    verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingRetryClient.class), eq("workerASG"), eq(5));
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext),  eq(amazonCloudFormationRetryClient), any());
    verify(awsAutoScalingService, times(1)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack));
    ArgumentCaptor<List<CloudResource>> captor = ArgumentCaptor.forClass(List.class);
    verify(awsComputeResourceService, times(1))
            .buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack), anyList(), captor.capture(), any(), any());
    verify(awsTaggingService, times(1)).tagRootVolumes(eq(authenticatedContext), any(AmazonEC2Client.class), eq(allInstances), eq(tags));
    verify(awsCloudWatchService, times(1)).addCloudWatchAlarmsForSystemFailures(any(), eq(cloudStack), eq("eu-west-1"),
            any(AwsCredentialView.class));
    List<CloudResource> newInstances = captor.getValue();
    assertEquals("Two new instances should be created", 2, newInstances.size());
    assertThat(newInstances, hasItem(workerInstance4));
    assertThat(newInstances, hasItem(workerInstance5));
}
 

@Test
void upscaleAwsASGroupFail() throws AmazonAutoscalingFailed {
    AmazonAutoScalingRetryClient amazonAutoScalingRetryClient = mock(AmazonAutoScalingRetryClient.class);
    AmazonCloudFormationRetryClient amazonCloudFormationRetryClient = mock(AmazonCloudFormationRetryClient.class);
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult();
    List<AutoScalingGroup> autoScalingGroups = new ArrayList<>();

    AutoScalingGroup masterASGroup = new AutoScalingGroup();
    masterASGroup.setAutoScalingGroupName("masterASG");
    List<Instance> masterASGInstances = new ArrayList<>();
    masterASGInstances.add(new Instance().withInstanceId("i-master1"));
    masterASGInstances.add(new Instance().withInstanceId("i-master2"));
    masterASGroup.setInstances(masterASGInstances);

    AutoScalingGroup workerASGroup = new AutoScalingGroup();
    workerASGroup.setAutoScalingGroupName("workerASG");
    List<Instance> workerASGInstances = new ArrayList<>();
    workerASGInstances.add(new Instance().withInstanceId("i-worker1"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker2"));
    workerASGInstances.add(new Instance().withInstanceId("i-worker3"));
    workerASGroup.setInstances(workerASGInstances);

    autoScalingGroups.add(masterASGroup);
    autoScalingGroups.add(workerASGroup);

    describeAutoScalingGroupsResult.setAutoScalingGroups(autoScalingGroups);
    when(amazonAutoScalingRetryClient.describeAutoScalingGroups(any(DescribeAutoScalingGroupsRequest.class)))
            .thenReturn(describeAutoScalingGroupsResult);
    when(awsClient.createAutoScalingRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingRetryClient);
    when(awsClient.createCloudFormationRetryClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationRetryClient);

    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("worker")))
            .thenReturn("workerASG");
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationRetryClient.class), eq("master")))
            .thenReturn("masterASG");

    AuthenticatedContext authenticatedContext = new AuthenticatedContext(new CloudContext(1L, "teststack", "AWS", "AWS",
            Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a")), "1", "1"), new CloudCredential());

    List<CloudResource> allInstances = new ArrayList<>();
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker1").group("worker").instanceId("i-worker1").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker2").group("worker").instanceId("i-worker2").build());
    allInstances.add(CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker3").group("worker").instanceId("i-worker3").build());
    CloudResource workerInstance4 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker4").group("worker").instanceId("i-worker4").build();
    allInstances.add(workerInstance4);
    CloudResource workerInstance5 = CloudResource.builder().type(ResourceType.AWS_INSTANCE).status(CommonStatus.CREATED)
            .name("worker5").group("worker").instanceId("i-worker5").build();
    allInstances.add(workerInstance5);
    when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationRetryClient), eq(amazonAutoScalingRetryClient), anyList()))
            .thenReturn(allInstances);

    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();

    Group master = getMasterGroup(instanceAuthentication);
    groups.add(master);

    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);

    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), emptyMap(), null,
            instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null);

    List<CloudResource> cloudResourceList = Collections.emptyList();

    AutoScalingGroup newWorkerASGroup = new AutoScalingGroup();
    newWorkerASGroup.setAutoScalingGroupName("workerASG");
    List<Instance> newWorkerASGInstances = new ArrayList<>();
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker1"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker2"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker3"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker4"));
    newWorkerASGInstances.add(new Instance().withInstanceId("i-worker5"));
    newWorkerASGroup.setInstances(newWorkerASGInstances);

    when(awsAutoScalingService.getAutoscalingGroups(eq(amazonAutoScalingRetryClient), any()))
            .thenReturn(Collections.singletonList(newWorkerASGroup));

    doThrow(new AmazonAutoscalingFailed("autoscaling failed"))
            .when(awsAutoScalingService).scheduleStatusChecks(eq(List.of(worker)),
            eq(authenticatedContext), eq(amazonCloudFormationRetryClient), any(Date.class));

    assertThrows(CloudConnectorException.class, () -> awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList),
            "Autoscaling group update failed: 'autoscaling failed' Original autoscaling group state has been recovered.");
    verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingRetryClient.class), eq("workerASG"), eq(5));
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext),  eq(amazonCloudFormationRetryClient), any());
    verify(awsComputeResourceService, times(0)).buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack),
            anyList(), anyList(), any(), any());
    verify(awsAutoScalingService, times(1)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingRetryClient), eq("i-worker4"));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingRetryClient), eq("i-worker5"));
    Map<String, Integer> desiredGroups = new HashMap<>();
    desiredGroups.put("workerASG", 3);
    desiredGroups.put("masterASG", 2);
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(desiredGroups), eq(authenticatedContext), any(Date.class));
}