com.amazonaws.services.ec2.model.InstanceStateName Java Examples

/**
 * Return the DNS name of one Amazon EC2 instance with the provided filter name and value.
 * 
 * @param ec2Client
 *        an Amazon EC2 instance
 * @param filterName
 *        the name of the filter
 * @param filterValue
 *        the value of the filter
 * @return the public DNS name of an instance with the filter name and value. Null if none exist.
 */
public static String getEndpointForFirstActiveInstanceWithTag(AmazonEC2 ec2Client,
        String filterName,
        String filterValue) {
    DescribeInstancesRequest describeInstancesRequest =
            new DescribeInstancesRequest().withFilters(new Filter().withName(filterName).withValues(filterValue));
    DescribeInstancesResult describeInstancesResult = ec2Client.describeInstances(describeInstancesRequest);

    List<Reservation> reservations = describeInstancesResult.getReservations();
    for (Reservation reservation : reservations) {
        List<Instance> ec2Instances = reservation.getInstances();
        for (Instance ec2Instance : ec2Instances) {
            if (InstanceStateName.Running.toString().equals(ec2Instance.getState().getName())) {
                return ec2Instance.getPublicDnsName();
            }
        }
    }
    return null;
}
 

@Test
public void should_not_keep_planned_node_if_configured_so_jenkins_will_overprovision() throws Exception {
    ComputerLauncher computerLauncher = mock(ComputerLauncher.class);
    ComputerConnector computerConnector = mock(ComputerConnector.class);
    when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher);

    final EC2FleetCloud cloud = spy(new EC2FleetCloud(null, null, "credId", null, "region",
            null, "fId", "momo", null, computerConnector, false, false,
            0, 0, 10, 1, false, false,
            false, 0, 0, false,
            10, false));
    j.jenkins.clouds.add(cloud);

    mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Running);

    getQueueTaskFutures(1);

    tryUntil(new Runnable() {
        @Override
        public void run() {
            j.jenkins.getLabelAtom("momo").nodeProvisioner.suggestReviewNow();
            verify(cloud, atLeast(2)).provision(any(Label.class), anyInt());
        }
    });
}
 

public Instance waitInstance(AwsProcessClient awsProcessClient, String instanceId) {
    // インスタンスの処理待ち
    Instance instance;
    while (true) {
        try {
            Thread.sleep(1000L * awsProcessClient.getDescribeInterval());
        } catch (InterruptedException ignore) {
        }

        instance = describeInstance(awsProcessClient, instanceId);
        InstanceStateName state;
        try {
            state = InstanceStateName.fromValue(instance.getState().getName());
        } catch (IllegalArgumentException e) {
            // 予期しないステータス
            AutoException exception = new AutoException("EPROCESS-000104", instanceId,
                    instance.getState().getName());
            exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(instance));
            throw exception;
        }

        // 安定状態のステータスになったら終了
        if (state == InstanceStateName.Running || state == InstanceStateName.Terminated
                || state == InstanceStateName.Stopped) {
            break;
        }
    }

    return instance;
}
 

@Test
public void testGetEnvironmentFromInstance() throws Exception {
    final Date launchTime = new Date();
    final Instance instance = new Instance().withInstanceId("instance").withInstanceType(InstanceType.C1Xlarge).withTags(new Tag(EC2Connector.DEFAULT_INSTANCE_NAME_TAG, "unknown"))
            .withState(new InstanceState().withName(InstanceStateName.Running)).withLaunchTime(launchTime).withPublicIpAddress("127.0.0.1");
    ServerEnvironment serverEnv = Whitebox.<ServerEnvironment> invokeMethod(env, "getEnvironmentFromInstance", instance);
    assertThat(serverEnv, notNullValue());
    assertThat(serverEnv.getInstanceId(), is("instance"));
    assertThat(serverEnv.getInstanceType(), is(InstanceType.C1Xlarge.toString()));
    assertThat(serverEnv.getType(), is(ENVIRONMENT_TYPES.TEST));
    assertThat(serverEnv.getEnvironmentTag(), is("unknown"));
    assertThat(serverEnv.getState().getName(), is(InstanceStateName.Running.toString()));
    assertThat(serverEnv.getLaunchTime(), is(launchTime));
    assertThat(serverEnv.getPublicIpAddress(), is("127.0.0.1"));
}
 

@Test
public void should_not_convert_planned_to_node_if_state_is_not_running_and_check_state_enabled() throws Exception {
    ComputerLauncher computerLauncher = mock(ComputerLauncher.class);
    ComputerConnector computerConnector = mock(ComputerConnector.class);
    when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher);

    EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region",
            null, "fId", "momo", null, computerConnector, false, false,
            0, 0, 10, 1, true, false,
            false, 0, 0, false,
            2, false);
    j.jenkins.clouds.add(cloud);

    mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Pending);

    List<QueueTaskFuture> rs = getQueueTaskFutures(1);

    triggerSuggestReviewNow("momo");

    Assert.assertEquals(0, j.jenkins.getNodes().size());

    tryUntil(new Runnable() {
        @Override
        public void run() {
            Assert.assertEquals(ImmutableSet.of("master", "momo"), labelsToNames(j.jenkins.getLabels()));
            Assert.assertEquals(1, j.jenkins.getLabelAtom("momo").nodeProvisioner.getPendingLaunches().size());
            Assert.assertEquals(0, j.jenkins.getNodes().size());
        }
    });

    cancelTasks(rs);
}
 

@Test
public void should_keep_planned_node_until_node_will_not_be_online_so_jenkins_will_not_request_overprovision() throws Exception {
    ComputerLauncher computerLauncher = mock(ComputerLauncher.class);
    ComputerConnector computerConnector = mock(ComputerConnector.class);
    when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher);

    EC2FleetCloud cloud = spy(new EC2FleetCloud(null, null, "credId", null, "region",
            null, "fId", "momo", null, computerConnector, false, false,
            0, 0, 10, 1, false, false,
            false, 300, 15, false,
            2, false));

    // provide init state
    cloud.setStats(new FleetStateStats("", 0, "active",
            Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

    j.jenkins.clouds.add(cloud);

    mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Running);

    List<QueueTaskFuture> rs = getQueueTaskFutures(1);

    final String labelString = "momo";
    triggerSuggestReviewNow(labelString);

    Thread.sleep(TimeUnit.MINUTES.toMillis(2));

    verify(cloud, times(1)).provision(any(Label.class), anyInt());

    cancelTasks(rs);
}
 

@Before
public void before() {
    EC2Api ec2Api = spy(EC2Api.class);
    Registry.setEc2Api(ec2Api);

    AmazonEC2 amazonEC2 = mock(AmazonEC2.class);
    when(ec2Api.connect(anyString(), anyString(), Mockito.nullable(String.class))).thenReturn(amazonEC2);

    final Instance instance = new Instance()
            .withState(new InstanceState().withName(InstanceStateName.Running))
            .withPublicIpAddress("public-io")
            .withInstanceId("i-1");

    when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))).thenReturn(
            new DescribeInstancesResult().withReservations(
                    new Reservation().withInstances(
                            instance
                    )));

    when(amazonEC2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class)))
            .thenReturn(new DescribeSpotFleetInstancesResult()
                    .withActiveInstances(new ActiveInstance().withInstanceId("i-1")));

    DescribeSpotFleetRequestsResult describeSpotFleetRequestsResult = new DescribeSpotFleetRequestsResult();
    describeSpotFleetRequestsResult.setSpotFleetRequestConfigs(Arrays.asList(
            new SpotFleetRequestConfig()
                    .withSpotFleetRequestState("active")
                    .withSpotFleetRequestConfig(
                            new SpotFleetRequestConfigData().withTargetCapacity(1))));
    when(amazonEC2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class)))
            .thenReturn(describeSpotFleetRequestsResult);
}
 

public void waitRun(AwsProcessClient awsProcessClient, Long instanceNo) {
    AwsInstance awsInstance = awsInstanceDao.read(instanceNo);
    String instanceId = awsInstance.getInstanceId();

    // インスタンスの作成待ち
    com.amazonaws.services.ec2.model.Instance instance = awsCommonProcess.waitInstance(awsProcessClient,
            instanceId);

    if (!instance.getState().getName().equals(InstanceStateName.Running.toString())) {
        // インスタンス作成失敗時
        AutoException exception = new AutoException("EPROCESS-000106", instanceId, instance.getState().getName());
        exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(instance));
        throw exception;
    }

    // ログ出力
    if (log.isInfoEnabled()) {
        log.info(MessageUtils.getMessage("IPROCESS-100116", instanceId));
    }

    // イベントログ出力
    Instance instance2 = instanceDao.read(instanceNo);
    processLogger.debug(null, instance2, "AwsInstanceCreateFinish",
            new Object[] { awsProcessClient.getPlatform().getPlatformName(), instanceId });

    // データベース更新
    awsInstance.setAvailabilityZone(instance.getPlacement().getAvailabilityZone());
    awsInstance.setStatus(instance.getState().getName());
    awsInstance.setDnsName(instance.getPublicDnsName());
    awsInstance.setPrivateDnsName(instance.getPrivateDnsName());
    awsInstance.setIpAddress(instance.getPublicIpAddress());
    awsInstance.setPrivateIpAddress(instance.getPrivateIpAddress());
    awsInstanceDao.update(awsInstance);
}
 

/**
 * NotFound exception example data
 * <p>
 * <code>
 * Single instance
 * requestId = "0fd56c54-e11a-4928-843c-9a80a24bedd1"
 * errorCode = "InvalidInstanceID.NotFound"
 * errorType = {AmazonServiceException$ErrorType@11247} "Unknown"
 * errorMessage = "The instance ID 'i-1233f' does not exist"
 * </code>
 * <p>
 * Multiple instances
 * <code>
 * ex = {AmazonEC2Exception@11233} "com.amazonaws.services.ec2.model.AmazonEC2Exception: The instance IDs 'i-1233f, i-ffffff' do not exist (Service: AmazonEC2; Status Code: 400; Error Code: InvalidInstanceID.NotFound; Request ID:)"
 * requestId = "1a353313-ef52-4626-b87b-fd828db6343f"
 * errorCode = "InvalidInstanceID.NotFound"
 * errorType = {AmazonServiceException$ErrorType@11251} "Unknown"
 * errorMessage = "The instance IDs 'i-1233f, i-ffffff' do not exist"
 * </code>
 */
@Test
public void describeInstances_shouldHandleAmazonEc2NotFoundErrorAsTerminatedInstancesAndRetry() {
    // given
    Set<String> instanceIds = new HashSet<>();
    instanceIds.add("i-1");
    instanceIds.add("i-f");
    instanceIds.add("i-3");

    AmazonEC2Exception notFoundException = new AmazonEC2Exception(
            "The instance IDs 'i-1, i-f' do not exist");
    notFoundException.setErrorCode("InvalidInstanceID.NotFound");

    final Instance instance3 = new Instance().withInstanceId("i-3")
            .withState(new InstanceState().withName(InstanceStateName.Running));
    DescribeInstancesResult describeInstancesResult2 = new DescribeInstancesResult()
            .withReservations(new Reservation().withInstances(
                    instance3));

    when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class)))
            .thenThrow(notFoundException)
            .thenReturn(describeInstancesResult2);

    // when
    final Map<String, Instance> described = new EC2Api().describeInstances(amazonEC2, instanceIds);

    // then
    Assert.assertEquals(ImmutableMap.of("i-3", instance3), described);
    verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i-1", "i-3", "i-f")));
    verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i-3")));
    verifyNoMoreInteractions(amazonEC2);
}
 

@Test
public void describeInstances_shouldSendInOneCallNoMoreThenBatchSizeOfInstance() {
    // given
    Set<String> instanceIds = new HashSet<>();
    instanceIds.add("i1");
    instanceIds.add("i2");
    instanceIds.add("i3");

    DescribeInstancesResult describeInstancesResult1 =
            new DescribeInstancesResult()
                    .withReservations(
                            new Reservation().withInstances(new Instance()
                                            .withInstanceId("stopped")
                                            .withState(new InstanceState().withName(InstanceStateName.Running)),
                                    new Instance()
                                            .withInstanceId("stopping")
                                            .withState(new InstanceState().withName(InstanceStateName.Running))
                            ));

    DescribeInstancesResult describeInstancesResult2 = new DescribeInstancesResult();

    when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class)))
            .thenReturn(describeInstancesResult1)
            .thenReturn(describeInstancesResult2);

    // when
    new EC2Api().describeInstances(amazonEC2, instanceIds, 2);

    // then
    verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i1", "i2")));
    verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i3")));
    verifyNoMoreInteractions(amazonEC2);
}
 

@Test
public void describeInstances_shouldNotDescribeMissedInResultInstanceOrTerminatedOrStoppedOrStoppingOrShuttingDownAs() {
    // given
    Set<String> instanceIds = new HashSet<>();
    instanceIds.add("missed");
    instanceIds.add("stopped");
    instanceIds.add("terminated");
    instanceIds.add("stopping");
    instanceIds.add("shutting-down");

    DescribeInstancesResult describeInstancesResult1 =
            new DescribeInstancesResult()
                    .withReservations(
                            new Reservation().withInstances(new Instance()
                                            .withInstanceId("stopped")
                                            .withState(new InstanceState().withName(InstanceStateName.Stopped)),
                                    new Instance()
                                            .withInstanceId("stopping")
                                            .withState(new InstanceState().withName(InstanceStateName.Stopping)),
                                    new Instance()
                                            .withInstanceId("shutting-down")
                                            .withState(new InstanceState().withName(InstanceStateName.ShuttingDown)),
                                    new Instance()
                                            .withInstanceId("terminated")
                                            .withState(new InstanceState().withName(InstanceStateName.Terminated))
                            ));


    when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class)))
            .thenReturn(describeInstancesResult1);

    // when
    Map<String, Instance> described = new EC2Api().describeInstances(amazonEC2, instanceIds);

    // then
    Assert.assertEquals(Collections.<String, Instance>emptyMap(), described);
    verify(amazonEC2, times(1))
            .describeInstances(any(DescribeInstancesRequest.class));
}
 

public void waitStart(AwsProcessClient awsProcessClient, Long instanceNo) {
    AwsInstance awsInstance = awsInstanceDao.read(instanceNo);
    String instanceId = awsInstance.getInstanceId();

    // インスタンスの起動待ち
    com.amazonaws.services.ec2.model.Instance instance = awsCommonProcess.waitInstance(awsProcessClient,
            instanceId);

    if (!instance.getState().getName().equals(InstanceStateName.Running.toString())) {
        // インスタンス起動失敗時
        AutoException exception = new AutoException("EPROCESS-000126", instanceId, instance.getState().getName());
        exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(instance));
        throw exception;
    }

    // ログ出力
    if (log.isInfoEnabled()) {
        log.info(MessageUtils.getMessage("IPROCESS-100112", instanceId));
    }

    // イベントログ出力
    Instance instance2 = instanceDao.read(instanceNo);
    processLogger.debug(null, instance2, "AwsInstanceStartFinish",
            new Object[] { awsProcessClient.getPlatform().getPlatformName(), instanceId });

    // データベース更新
    awsInstance.setAvailabilityZone(instance.getPlacement().getAvailabilityZone());
    awsInstance.setStatus(instance.getState().getName());
    awsInstance.setDnsName(instance.getPublicDnsName());
    awsInstance.setPrivateDnsName(instance.getPrivateDnsName());
    awsInstance.setIpAddress(instance.getPublicIpAddress());
    awsInstance.setPrivateIpAddress(instance.getPrivateIpAddress());
    awsInstanceDao.update(awsInstance);
}
 

@Test
public void describeInstances_shouldReturnAllInstancesIfStillActive() {
    // given
    Set<String> instanceIds = new HashSet<>();
    instanceIds.add("i-1");
    instanceIds.add("i-2");

    DescribeInstancesResult describeInstancesResult = new DescribeInstancesResult();
    Reservation reservation = new Reservation();
    Instance instance1 = new Instance()
            .withInstanceId("i-1")
            .withState(new InstanceState().withName(InstanceStateName.Running));
    Instance instance2 = new Instance()
            .withInstanceId("i-2")
            .withState(new InstanceState().withName(InstanceStateName.Running));
    reservation.setInstances(Arrays.asList(instance1, instance2));
    describeInstancesResult.setReservations(Arrays.asList(reservation));

    when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))).thenReturn(describeInstancesResult);

    // when
    Map<String, Instance> described = new EC2Api().describeInstances(amazonEC2, instanceIds);

    // then
    Assert.assertEquals(ImmutableMap.of("i-1", instance1, "i-2", instance2), described);
    verify(amazonEC2, times(1))
            .describeInstances(any(DescribeInstancesRequest.class));
}
 

public void waitStop(AwsProcessClient awsProcessClient, Long instanceNo) {
    AwsInstance awsInstance = awsInstanceDao.read(instanceNo);
    String instanceId = awsInstance.getInstanceId();

    // インスタンスの停止待ち
    com.amazonaws.services.ec2.model.Instance instance = awsCommonProcess.waitInstance(awsProcessClient,
            instanceId);

    if (!instance.getState().getName().equals(InstanceStateName.Stopped.toString())) {
        // インスタンス停止失敗時
        AutoException exception = new AutoException("EPROCESS-000129", instanceId, instance.getState().getName());
        exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(instance));
        throw exception;
    }

    // ログ出力
    if (log.isInfoEnabled()) {
        log.info(MessageUtils.getMessage("IPROCESS-100114", instanceId));
    }

    // イベントログ出力
    Instance instance2 = instanceDao.read(instanceNo);
    processLogger.debug(null, instance2, "AwsInstanceStopFinish",
            new Object[] { awsProcessClient.getPlatform().getPlatformName(), instanceId });

    // データベース更新
    awsInstance.setAvailabilityZone(instance.getPlacement().getAvailabilityZone());
    awsInstance.setStatus(instance.getState().getName());
    awsInstance.setDnsName(instance.getPublicDnsName());
    awsInstance.setPrivateDnsName(instance.getPrivateDnsName());
    awsInstance.setIpAddress(instance.getPublicIpAddress());
    awsInstance.setPrivateIpAddress(instance.getPrivateIpAddress());
    awsInstanceDao.update(awsInstance);
}
 

/**
 * TODO: メソッドコメント
 * 
 * @param awsProcessClient
 * @param instanceNo
 */
public void startInstance(AwsProcessClient awsProcessClient, Long instanceNo) {
    AwsInstance awsInstance = awsInstanceDao.read(instanceNo);
    Instance instance = instanceDao.read(instanceNo);
    Image image = imageDao.read(instance.getImageNo());
    ImageAws imageAws = imageAwsDao.read(instance.getImageNo());

    // インスタンスストアイメージの場合や、EBSイメージで初回起動の場合
    if (BooleanUtils.isNotTrue(imageAws.getEbsImage()) || StringUtils.isEmpty(awsInstance.getInstanceId())) {
        // インスタンスIDがある場合はスキップ
        if (!StringUtils.isEmpty(awsInstance.getInstanceId())) {
            return;
        }

        // インスタンスの作成
        run(awsProcessClient, instanceNo);

        // インスタンスの作成待ち
        waitRun(awsProcessClient, instanceNo);

        // インスタンスにタグをつける
        createTag(awsProcessClient, instanceNo);

        // Windowsの場合、パスワードデータを取得できるようになるまで待つ
        if (StringUtils.startsWithIgnoreCase(image.getOs(), "windows")) {
            waitGetPasswordData(awsProcessClient, instanceNo);
        }
    }
    // EBSイメージで2回目以降の起動の場合
    else {
        // インスタンスが停止中でない場合はスキップ
        if (!StringUtils.equals(awsInstance.getStatus(), InstanceStateName.Stopped.toString())) {
            return;
        }

        // インスタンスの設定変更
        modify(awsProcessClient, instanceNo);

        // インスタンスの起動
        start(awsProcessClient, instanceNo);

        // インスタンスの起動待ち
        waitStart(awsProcessClient, instanceNo);
    }
}
 

/**
 * TODO: メソッドコメント
 * 
 * @param instanceNo
 */
public void start(Long instanceNo) {
    Instance instance = instanceDao.read(instanceNo);
    Farm farm = farmDao.read(instance.getFarmNo());

    if (log.isInfoEnabled()) {
        log.info(MessageUtils.getMessage("IPROCESS-100101", instanceNo, instance.getInstanceName()));
    }

    // AwsProcessClientの作成
    AwsProcessClient awsProcessClient = awsProcessClientFactory.createAwsProcessClient(farm.getUserNo(),
            instance.getPlatformNo());

    // インスタンスに関する処理
    int retryCount = 1; // インスタンス起動処理の最大リトライ回数
    for (int i = 0; i <= retryCount; i++) {
        try {
            awsInstanceProcess.startInstance(awsProcessClient, instanceNo);
            break;

        } catch (AutoException e) {
            if (i < retryCount) {
                // RunInstancesが正常に動いたが、予期せぬ異常によりインスタンスが起動しなかった場合、リトライする
                // TODO: エラーコード、インスタンスIDの取り方を改善すべきか（例外スロー元の実装に依存しているため）
                if ("EPROCESS-000106".equals(e.getCode())) {
                    String instanceId = (String) e.getAdditions()[0];

                    try {
                        // インスタンス情報の参照
                        com.amazonaws.services.ec2.model.Instance instance2 = awsCommonProcess
                                .describeInstance(awsProcessClient, instanceId);

                        if (InstanceStateName.Terminated.toString().equals(instance2.getState().getName())
                                && "Server.InternalError".equals(instance2.getStateTransitionReason())) {
                            // 警告ログ出力した後にリトライする
                            log.warn(e.getMessage());

                            // リトライ用にインスタンスIDを削除
                            AwsInstance awsInstance = awsInstanceDao.read(instanceNo);
                            awsInstance.setInstanceId(null);
                            awsInstanceDao.update(awsInstance);

                            continue;
                        }
                    } catch (Exception ignore) {
                    }
                }
            }

            throw e;
        }
    }

    // ボリュームに関する処理
    List<AwsVolume> awsVolumes = awsVolumeDao.readByInstanceNo(instanceNo);
    for (AwsVolume awsVolume : awsVolumes) {
        if (awsVolume.getComponentNo() != null) {
            // コンポーネント番号がある場合はスキップ
            continue;
        }
        awsVolumeProcess.startVolume(awsProcessClient, instanceNo, awsVolume.getVolumeNo());
    }

    // アドレスに関する処理
    awsAddressProcess.startAddress(awsProcessClient, instanceNo);

    // DNSに関する処理
    awsDnsProcess.startDns(instanceNo);

    if (log.isInfoEnabled()) {
        log.info(MessageUtils.getMessage("IPROCESS-100102", instanceNo, instance.getInstanceName()));
    }
}
 

public void waitTerminate(AwsProcessClient awsProcessClient, Long instanceNo) {
    AwsInstance awsInstance = awsInstanceDao.read(instanceNo);
    String instanceId = awsInstance.getInstanceId();

    // インスタンスの削除待ち
    com.amazonaws.services.ec2.model.Instance instance;
    try {
        instance = awsCommonProcess.waitInstance(awsProcessClient, instanceId);

        if (!StringUtils.equals(instance.getState().getName(), InstanceStateName.Terminated.toString())) {
            // インスタンス削除失敗時
            AutoException exception = new AutoException("EPROCESS-000109", instanceId,
                    instance.getState().getName());
            exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(instance));
            throw exception;
        }
    } catch (AutoException e) {
        // インスタンス情報を参照できない場合、既に削除されたものとして例外を無視する
        if ("EPROCESS-000101".equals(e.getCode())) {
            instance = null;
        } else {
            throw e;
        }
    }

    // ログ出力
    if (log.isInfoEnabled()) {
        log.info(MessageUtils.getMessage("IPROCESS-100118", instanceId));
    }

    // イベントログ出力
    Instance instance2 = instanceDao.read(instanceNo);
    processLogger.debug(null, instance2, "AwsInstanceDeleteFinish",
            new Object[] { awsProcessClient.getPlatform().getPlatformName(), instanceId });

    String status = null;
    if (instance != null) {
        status = instance.getState().getName();
    }

    // データベース更新
    awsInstance.setInstanceId(null);
    awsInstance.setStatus(status);
    awsInstance.setDnsName(null);
    awsInstance.setPrivateDnsName(null);
    awsInstance.setIpAddress(null);
    awsInstance.setPrivateIpAddress(null);
    awsInstanceDao.update(awsInstance);
}
 

@Test
    public void should_not_allow_jenkins_to_provision_if_address_not_available() throws Exception {
        ComputerLauncher computerLauncher = mock(ComputerLauncher.class);
        ComputerConnector computerConnector = mock(ComputerConnector.class);
        when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher);

        EC2FleetCloud cloud = spy(new EC2FleetCloud(null, null, "credId", null, "region",
                null, "fId", "momo", null, computerConnector, false, false,
                0, 0, 10, 1, false, false,
                false, 0, 0, false,
                10, false));

        cloud.setStats(new FleetStateStats("", 0, "active",
                Collections.<String>emptySet(), Collections.<String, Double>emptyMap()));

        j.jenkins.clouds.add(cloud);

        EC2Api ec2Api = spy(EC2Api.class);
        Registry.setEc2Api(ec2Api);

        AmazonEC2 amazonEC2 = mock(AmazonEC2.class);
        when(ec2Api.connect(anyString(), anyString(), Mockito.nullable(String.class))).thenReturn(amazonEC2);

        when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))).thenReturn(
                new DescribeInstancesResult().withReservations(
                        new Reservation().withInstances(
                                new Instance()
                                        .withState(new InstanceState().withName(InstanceStateName.Running))
//                                        .withPublicIpAddress("public-io")
                                        .withInstanceId("i-1")
                        )));

        when(amazonEC2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class))).thenReturn(
                new DescribeSpotFleetInstancesResult().withActiveInstances(new ActiveInstance().withInstanceId("i-1")));

        DescribeSpotFleetRequestsResult describeSpotFleetRequestsResult = new DescribeSpotFleetRequestsResult();
        describeSpotFleetRequestsResult.setSpotFleetRequestConfigs(Arrays.asList(
                new SpotFleetRequestConfig()
                        .withSpotFleetRequestState("active")
                        .withSpotFleetRequestConfig(
                                new SpotFleetRequestConfigData().withTargetCapacity(1))));
        when(amazonEC2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class)))
                .thenReturn(describeSpotFleetRequestsResult);

        List<QueueTaskFuture> rs = getQueueTaskFutures(1);

        j.jenkins.getLabelAtom("momo").nodeProvisioner.suggestReviewNow();

        Assert.assertEquals(0, j.jenkins.getNodes().size());

        Thread.sleep(TimeUnit.MINUTES.toMillis(2));

        cancelTasks(rs);

        verify(cloud, times(1)).provision(any(Label.class), anyInt());
    }
 

@Test
    public void should_continue_update_after_termination() throws IOException {
        mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Running, 5);

        final ComputerConnector computerConnector = new LocalComputerConnector(j);
        final EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region",
                null, "fId", "momo", null, computerConnector, false, false,
                1, 0, 5, 1, true, false,
                false, 0, 0, false,
                10, false);
        j.jenkins.clouds.add(cloud);

        // wait while all nodes will be ok
//        tryUntil(new Runnable() {
//            @Override
//            public void run() {
//                for (Node node : j.jenkins.getNodes()) {
//                    final Computer computer = node.toComputer();
//                    Assert.assertNotNull(computer);
//                    Assert.assertTrue(computer.isOnline());
//                }
//            }
//        });

        final List<QueueTaskFuture<FreeStyleBuild>> tasks = new ArrayList<>();
        tasks.addAll((List) getQueueTaskFutures(5));
        System.out.println("tasks submitted");

        // wait full execution
        for (final QueueTaskFuture<FreeStyleBuild> task : tasks) {
            try {
                Assert.assertEquals(task.get().getResult(), Result.SUCCESS);
            } catch (InterruptedException | ExecutionException e) {
                throw new RuntimeException(e);
            }
        }

        // wait until downscale happens
        tryUntil(new Runnable() {
            @Override
            public void run() {
                // defect in termination logic, that why 1
                Assert.assertThat(j.jenkins.getLabel("momo").getNodes().size(), Matchers.lessThanOrEqualTo(1));
            }
        }, TimeUnit.MINUTES.toMillis(3));

        final FleetStateStats oldStats = cloud.getStats();
        tryUntil(new Runnable() {
            @Override
            public void run() {
                System.out.println("stats should be updated");
                Assert.assertNotSame(oldStats, cloud.getStats());
            }
        });
    }
 

/**
 * https://github.com/jenkinsci/ec2-plugin/blob/master/src/main/java/hudson/plugins/ec2/EC2Cloud.java#L640
 *
 * @param ec2      ec2 client
 * @param instance instance
 */
private void addNewSlave(final AmazonEC2 ec2, final Instance instance, FleetStateStats stats) throws Exception {
    final String instanceId = instance.getInstanceId();

    // instance state check enabled and not running, skip adding
    if (addNodeOnlyIfRunning && InstanceStateName.Running != InstanceStateName.fromValue(instance.getState().getName()))
        return;

    final String address = privateIpUsed ? instance.getPrivateIpAddress() : instance.getPublicIpAddress();
    // Check if we have the address to use. Nodes don't get it immediately.
    if (address == null) {
        if (!privateIpUsed) {
            info("%s instance public IP address not assigned, it could take some time or" +
                    " Spot Request is not configured to assign public IPs", instance.getInstanceId());
        }
        return; // wait more time, probably IP address not yet assigned
    }

    // Generate a random FS root if one isn't specified
    final String effectiveFsRoot;
    if (StringUtils.isBlank(fsRoot)) {
        effectiveFsRoot = "/tmp/jenkins-" + UUID.randomUUID().toString().substring(0, 8);
    } else {
        effectiveFsRoot = fsRoot;
    }

    final Double instanceTypeWeight = stats.getInstanceTypeWeights().get(instance.getInstanceType());
    final int effectiveNumExecutors;
    if (scaleExecutorsByWeight && instanceTypeWeight != null) {
        effectiveNumExecutors = (int) Math.max(Math.round(numExecutors * instanceTypeWeight), 1);
    } else {
        effectiveNumExecutors = numExecutors;
    }

    final EC2FleetAutoResubmitComputerLauncher computerLauncher = new EC2FleetAutoResubmitComputerLauncher(
            computerConnector.launch(address, TaskListener.NULL));
    final Node.Mode nodeMode = restrictUsage ? Node.Mode.EXCLUSIVE : Node.Mode.NORMAL;
    final EC2FleetNode node = new EC2FleetNode(instanceId, "Fleet slave for " + instanceId,
            effectiveFsRoot, effectiveNumExecutors, nodeMode, labelString, new ArrayList<NodeProperty<?>>(),
            this, computerLauncher);

    // Initialize our retention strategy
    node.setRetentionStrategy(new IdleRetentionStrategy());

    final Jenkins jenkins = Jenkins.getInstance();
    // jenkins automatically remove old node with same name if any
    jenkins.addNode(node);

    final SettableFuture<Node> future;
    if (plannedNodesCache.isEmpty()) {
        future = SettableFuture.create();
    } else {
        final NodeProvisioner.PlannedNode plannedNode = plannedNodesCache.iterator().next();
        plannedNodesCache.remove(plannedNode);
        future = ((SettableFuture<Node>) plannedNode.future);
    }

    // use getters for timeout and interval as they provide default value
    // when user just install new version and did't recreate fleet
    EC2FleetOnlineChecker.start(node, future,
            TimeUnit.SECONDS.toMillis(getInitOnlineTimeoutSec()),
            TimeUnit.SECONDS.toMillis(getInitOnlineCheckIntervalSec()));
}
 

private void test(int workers, int maxTasks) throws IOException, InterruptedException {
    mockEc2ApiToDescribeInstancesWhenModifiedWithDelay(InstanceStateName.Running, 500);

    final ComputerConnector computerConnector = new LocalComputerConnector(j);
    final EC2FleetCloudWithMeter cloud = new EC2FleetCloudWithMeter(null, null, "credId", null, "region",
            null, "fId", "momo", null, computerConnector, false, false,
            1, 0, workers, 1, true, false,
            false, 0, 0, false,
            2, false);
    j.jenkins.clouds.add(cloud);

    // updated plugin requires some init time to get first update
    // so wait this event to be really correct with perf comparison as old version is not require init time
    tryUntil(new Runnable() {
        @Override
        public void run() {
            Assert.assertNotNull(cloud.getStats());
        }
    });

    System.out.println("start test");
    final long start = System.currentTimeMillis();

    final List<QueueTaskFuture<FreeStyleBuild>> tasks = new ArrayList<>();

    final int taskBatch = 5;

    while (tasks.size() < maxTasks) {
        tasks.addAll((List) getQueueTaskFutures(taskBatch));
        triggerSuggestReviewNow("momo");
        System.out.println(taskBatch + " added into queue, " + (maxTasks - tasks.size()) + " remain");
    }

    for (final QueueTaskFuture<FreeStyleBuild> task : tasks) {
        try {
            Assert.assertEquals(task.get().getResult(), Result.SUCCESS);
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }
    }

    System.out.println("downscale");
    final long finish = System.currentTimeMillis();

    // wait until downscale happens
    tryUntil(new Runnable() {
        @Override
        public void run() {
            // defect in termination logic, that why 1
            Assert.assertThat(j.jenkins.getLabel("momo").getNodes().size(), Matchers.lessThanOrEqualTo(1));
        }
    }, TimeUnit.MINUTES.toMillis(3));

    final long upTime = TimeUnit.MILLISECONDS.toSeconds(finish - start);
    final long downTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - finish);
    final long totalTime = upTime + downTime;
    final long ideaUpTime = (maxTasks / workers) * JOB_SLEEP_TIME;
    final int idealDownTime = 60;
    final long ideaTime = ideaUpTime + idealDownTime;

    System.out.println(maxTasks + " up in " + upTime + " sec, ideal time is " + ideaUpTime + " sec, overhead is " + (upTime - ideaUpTime) + " sec");
    System.out.println(maxTasks + " down in " + downTime + " sec, ideal time is " + idealDownTime + " sec, overhead is " + (downTime - idealDownTime) + " sec");
    System.out.println(maxTasks + " completed in " + totalTime + " sec, ideal time is " + ideaTime + " sec, overhead is " + (totalTime - ideaTime) + " sec");
    System.out.println(cloud.provisionMeter);
    System.out.println(cloud.removeMeter);
    System.out.println(cloud.updateMeter);
}
 

@Test
public void describeInstances_shouldProcessAllPagesUntilNextTokenIsAvailable() {
    // given
    Set<String> instanceIds = new HashSet<>();
    instanceIds.add("i-1");
    instanceIds.add("i-2");
    instanceIds.add("i-3");

    final Instance instance1 = new Instance()
            .withInstanceId("i-1")
            .withState(new InstanceState().withName(InstanceStateName.Running));
    DescribeInstancesResult describeInstancesResult1 =
            new DescribeInstancesResult()
                    .withReservations(
                            new Reservation().withInstances(instance1))
                    .withNextToken("a");

    final Instance instance2 = new Instance()
            .withInstanceId("i-2")
            .withState(new InstanceState().withName(InstanceStateName.Running));
    DescribeInstancesResult describeInstancesResult2 =
            new DescribeInstancesResult()
                    .withReservations(new Reservation().withInstances(
                            instance2,
                            new Instance()
                                    .withInstanceId("i-3")
                                    .withState(new InstanceState().withName(InstanceStateName.Terminated))
                    ));

    when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class)))
            .thenReturn(describeInstancesResult1)
            .thenReturn(describeInstancesResult2);

    // when
    Map<String, Instance> described = new EC2Api().describeInstances(amazonEC2, instanceIds);

    // then
    Assert.assertEquals(ImmutableMap.of("i-1", instance1, "i-2", instance2), described);
    verify(amazonEC2, times(2))
            .describeInstances(any(DescribeInstancesRequest.class));
}
 

protected void mockEc2ApiToDescribeInstancesWhenModifiedWithDelay(final InstanceStateName instanceStateName, final long delayMillis) {
    mockEc2ApiToDescribeInstancesWhenModifiedWithDelay(instanceStateName, 0, delayMillis);
}
 

protected void mockEc2ApiToDescribeInstancesWhenModified(final InstanceStateName instanceStateName, final int initialTargetCapacity) {
    mockEc2ApiToDescribeInstancesWhenModifiedWithDelay(instanceStateName, initialTargetCapacity, 0);
}
 

protected void mockEc2ApiToDescribeInstancesWhenModified(final InstanceStateName instanceStateName) {
    mockEc2ApiToDescribeInstancesWhenModifiedWithDelay(instanceStateName, 0, 0);
}
 

/**
 * @param stack     <code>ICoordinatedStack</code> object containing the <code>cluster</code>
 * @param cluster
 * @return          Cluster instances which are in <code>Running</code> state
 */
@Override
public Collection<Instance> getClusterInstances( ICoordinatedStack stack, ICoordinatedCluster cluster ) {

    String name = getInstanceName( stack, cluster );

    if( stack.getDataCenter() != null && ! stack.getDataCenter().isEmpty() ) {
        client.setEndpoint( AmazonUtils.getEndpoint( stack.getDataCenter() ) );
    }

    return toInstances( getEC2Instances( name, InstanceStateName.Running ) );

}
 

/**
 * @param stack
 * @return      Runner instances which belong to <code>stack</code> and in <code>Running</code> state
 */
@Override
public Collection<Instance> getRunnerInstances( ICoordinatedStack stack ) {

    String name = getRunnerName( stack );

    if( stack.getDataCenter() != null && ! stack.getDataCenter().isEmpty() ) {
        client.setEndpoint( AmazonUtils.getEndpoint( stack.getDataCenter() ) );
    }

    return toInstances( getEC2Instances( name, InstanceStateName.Running ) );

}