com.amazonaws.services.ec2.model.Volume Java Examples

/**
 * Create Volume.
 *
 * @param availabilityZone the availability zone
 * @param iops the iops
 * @param size the size
 * @param snapshotId the snapshot id
 * @param volumeType the volume type
 * @return Volume
 */
protected final Volume createVolume(final String availabilityZone, final Integer iops, final Integer size, final String snapshotId, final String volumeType) {
    Volume volume = null;

    CreateVolumeRequest req = new CreateVolumeRequest();
    req.setAvailabilityZone(availabilityZone);
    req.setIops(iops);
    req.setSize(size);
    req.setSnapshotId(snapshotId);
    req.setVolumeType(volumeType);
    
    CreateVolumeResult result = amazonEC2Client.createVolume(req);
    if (result != null) {
        volume = result.getVolume();
    }

    return volume;
}
 

@Override
protected void setUpRead()
{
    when(mockEc2.describeVolumes(any(DescribeVolumesRequest.class))).thenAnswer((InvocationOnMock invocation) -> {
        DescribeVolumesRequest request = (DescribeVolumesRequest) invocation.getArguments()[0];

        assertEquals(getIdValue(), request.getVolumeIds().get(0));
        DescribeVolumesResult mockResult = mock(DescribeVolumesResult.class);
        List<Volume> values = new ArrayList<>();
        values.add(makeVolume(getIdValue()));
        values.add(makeVolume(getIdValue()));
        values.add(makeVolume("fake-id"));
        when(mockResult.getVolumes()).thenReturn(values);
        return mockResult;
    });
}
 

private Volume makeVolume(String id)
{
    Volume volume = new Volume();
    volume.withVolumeId(id)
            .withVolumeType("type")
            .withAttachments(new VolumeAttachment()
                    .withInstanceId("target")
                    .withDevice("attached_device")
                    .withState("attachment_state")
                    .withAttachTime(new Date(100_000)))
            .withAvailabilityZone("availability_zone")
            .withCreateTime(new Date(100_000))
            .withEncrypted(true)
            .withKmsKeyId("kms_key_id")
            .withSize(100)
            .withIops(100)
            .withSnapshotId("snapshot_id")
            .withState("state")
            .withTags(new Tag("key", "value"));

    return volume;
}
 

/**
 * Describe All Volume.
 *
 * @return Collection Volume
 */
protected final List<Volume> getVolumes() {
    List<Volume> volumes = null;

    DescribeVolumesRequest req = new DescribeVolumesRequest();
    req.setMaxResults(20);
    DescribeVolumesResult result = amazonEC2Client.describeVolumes(req);
    if (result != null && !result.getVolumes().isEmpty()) {
        volumes = result.getVolumes();
        log.info("Page Size : " + volumes.size());
    }

    while(result.getNextToken() != null) { 
        req.setNextToken(result.getNextToken());
        result = amazonEC2Client.describeVolumes(req);
        if (result != null && !result.getVolumes().isEmpty()) {
             volumes = result.getVolumes();
             log.info("Page Size : " + volumes.size());
        }
    }
    
    return volumes;
}
 

@SuppressWarnings("static-access")
@Test
public void collectAllVolumesTest() throws Exception {
    
    Volume vol = new Volume();
    vol.setVolumeId("123");
    Collection<Volume> volumes = new ArrayList<>();
    volumes.add(vol);
   
    
    DescribeVolumesResult result = new DescribeVolumesResult();
    result.setVolumes(volumes);
    
    when(ec2ServiceClient.describeVolumes(anyObject())).thenReturn(result);
    assertThat(pacmanEc2Utils.collectAllVolumes(ec2ServiceClient,describeVolumesRequest),is(notNullValue()));
}
 

/**
 * Generatefetch volume files.
 *
 * @param volumeMap the volume map
 * @throws IOException Signals that an I/O exception has occurred.
 */
public static void generatefetchVolumeFiles(Map<String, List<Volume>> volumeMap) throws IOException {
	String fieldNames;
	String keys;
	fieldNames = "VolumeId`VolumeType`AvailabilityZone`CreateTime`Encrypted`Iops`KmsKeyId`Size`SnapshotId`State";
	keys = "discoverydate`accountid`accountname`region`volumeid`volumetype`availabilityzone`createtime`encrypted`iops`kmskeyid`size`snapshotid`state";
	FileGenerator.generateJson(volumeMap, fieldNames, "aws-volume.data",keys);

	fieldNames = "VolumeId`attachments.InstanceId`attachments.AttachTime`attachments.DeleteOnTermination`attachments.Device`attachments.State";
	keys = "discoverydate`accountid`accountname`region`volumeid`instanceid`attachtime`deleteontermination`device`state";
	FileGenerator.generateJson(volumeMap, fieldNames, "aws-volume-attachments.data",keys);

	fieldNames = "VolumeId`tags.key`tags.value";
	keys = "discoverydate`accountid`accountname`region`volumeid`key`value";
	FileGenerator.generateJson(volumeMap, fieldNames, "aws-volume-tags.data",keys);
}
 

/**
 * Fetch volumet info.
 *
 * @param temporaryCredentials the temporary credentials
 * @param skipRegions the skip regions
 * @param accountId the accountId
 * @param accountName the account name
 * @return the map
 */
public static Map<String,List<Volume>> fetchVolumetInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName) {
	Map<String,List<Volume>> volumeList = new LinkedHashMap<>();
	AmazonEC2 ec2Client ;
	String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource in specific region\" ,\"type\": \"Volume\" , \"region\":\"" ;
	for(Region region : RegionUtils.getRegions()){
		try{
			if(!skipRegions.contains(region.getName())){
				ec2Client = AmazonEC2ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
				DescribeVolumesResult  rslt = ec2Client.describeVolumes(); // No need to paginate as all volumes will be returned.
				List<Volume> volumeListTemp = rslt.getVolumes();

				if( !volumeListTemp.isEmpty() ) {
					log.debug(InventoryConstants.ACCOUNT + accountId +" Type : Volume "+region.getName() + " >> "+volumeListTemp.size());
					volumeList.put(accountId+delimiter+accountName+delimiter+region.getName(),volumeListTemp);
				}
			}

		}catch(Exception e){
			log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
			ErrorManageUtil.uploadError(accountId,region.getName(),"volume",e.getMessage());
		}
	}
	return volumeList;
}
 

/**
 * Fetch volumet info test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchVolumetInfoTest() throws Exception {
    
    mockStatic(AmazonEC2ClientBuilder.class);
    AmazonEC2 ec2Client = PowerMockito.mock(AmazonEC2.class);
    AmazonEC2ClientBuilder amazonEC2ClientBuilder = PowerMockito.mock(AmazonEC2ClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonEC2ClientBuilder.standard()).thenReturn(amazonEC2ClientBuilder);
    when(amazonEC2ClientBuilder.withCredentials(anyObject())).thenReturn(amazonEC2ClientBuilder);
    when(amazonEC2ClientBuilder.withRegion(anyString())).thenReturn(amazonEC2ClientBuilder);
    when(amazonEC2ClientBuilder.build()).thenReturn(ec2Client);
    
    DescribeVolumesResult describeVolumesResult = new DescribeVolumesResult();
    List<Volume> volumeList = new ArrayList<>();
    volumeList.add(new Volume());
    describeVolumesResult.setVolumes(volumeList);
    when(ec2Client.describeVolumes()).thenReturn(describeVolumesResult);
    assertThat(inventoryUtil.fetchVolumetInfo(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
}
 

@Override
protected Volume convertObject(VolumeInfo from) {
    Volume to = new Volume();

    to.setVolumeId(from.getVolumeId());

    Integer size = null;
    if (from.getSize() != null && from.getSize().length() != 0) {
        size = Integer.valueOf(from.getSize());
    }
    to.setSize(size);

    to.setSnapshotId(from.getSnapshotId());
    to.setAvailabilityZone(from.getZone());
    to.setState(from.getStatus());
    to.setCreateTime(from.getCreateTime().getTime());
    to.setAttachments(new VolumeAttachmentConverter().convert(from.getAttachmentInfo()));

    // 未実装
    to.setTags(null);

    return to;
}
 

public Volume describeVolume(AwsProcessClient awsProcessClient, String volumeId) {
    // 単一ボリュームの参照
    DescribeVolumesRequest request = new DescribeVolumesRequest();
    request.withVolumeIds(volumeId);
    DescribeVolumesResult result = awsProcessClient.getEc2Client().describeVolumes(request);
    List<Volume> volumes = result.getVolumes();

    // API実行結果チェック
    if (volumes.size() == 0) {
        // ボリュームが存在しない場合
        throw new AutoException("EPROCESS-000110", volumeId);

    } else if (volumes.size() > 1) {
        // ボリュームを複数参照できた場合
        AutoException exception = new AutoException("EPROCESS-000111", volumeId);
        exception.addDetailInfo("result=" + volumes);
        throw exception;
    }

    return volumes.get(0);
}
 

/**
 * Maps an EBS Volume into a row in our Apache Arrow response block(s).
 *
 * @param volume The EBS Volume to map.
 * @param spiller The BlockSpiller to use when we want to write a matching row to the response.
 * @note The current implementation is rather naive in how it maps fields. It leverages a static
 * list of fields that we'd like to provide and then explicitly filters and converts each field.
 */
private void instanceToRow(Volume volume,
        BlockSpiller spiller)
{
    spiller.writeRows((Block block, int row) -> {
        boolean matched = true;

        matched &= block.offerValue("id", row, volume.getVolumeId());
        matched &= block.offerValue("type", row, volume.getVolumeType());
        matched &= block.offerValue("availability_zone", row, volume.getAvailabilityZone());
        matched &= block.offerValue("created_time", row, volume.getCreateTime());
        matched &= block.offerValue("is_encrypted", row, volume.getEncrypted());
        matched &= block.offerValue("kms_key_id", row, volume.getKmsKeyId());
        matched &= block.offerValue("size", row, volume.getSize());
        matched &= block.offerValue("iops", row, volume.getIops());
        matched &= block.offerValue("snapshot_id", row, volume.getSnapshotId());
        matched &= block.offerValue("state", row, volume.getState());

        if (volume.getAttachments().size() == 1) {
            matched &= block.offerValue("target", row, volume.getAttachments().get(0).getInstanceId());
            matched &= block.offerValue("attached_device", row, volume.getAttachments().get(0).getDevice());
            matched &= block.offerValue("attachment_state", row, volume.getAttachments().get(0).getState());
            matched &= block.offerValue("attachment_time", row, volume.getAttachments().get(0).getAttachTime());
        }

        List<String> tags = volume.getTags().stream()
                .map(next -> next.getKey() + ":" + next.getValue()).collect(Collectors.toList());
        matched &= block.offerComplexValue("tags", row, FieldResolver.DEFAULT, tags);

        return matched ? 1 : 0;
    });
}
 

/**
 * Describe Volume.
 *
 * @return Volume
 */
protected final Volume getVolume() {
    Volume volume = null;

    DescribeVolumesRequest req = new DescribeVolumesRequest();
    DescribeVolumesResult result = amazonEC2Client.describeVolumes(req);
    if (result != null && !result.getVolumes().isEmpty()) {
        volume = result.getVolumes().get(0);
    }

    return volume;
}
 

/**
 * Test delete Volumes.
 */
@Test(timeout = TIMEOUT_LEVEL1)
public final void deleteVolumesTest() {
    log.info("Start describing volume test");
    
    Volume volume = createVolume(MOCK_EC2_PLACEMENT, 1000, 32, "snap-12312313", "gp2");

    Assert.assertNotNull("volume should not be null", volume);
    Assert.assertNotNull("volume id should not be null", volume.getVolumeId());
    
    Assert.assertTrue("volume should be deleted",deleteVolume(volume.getVolumeId()));
}
 

/**
 * Test create Volumes.
 */
@Test(timeout = TIMEOUT_LEVEL1)
public final void createVolumesTest() {
    log.info("Start create volume test");
    
    Volume volume = createVolume(MOCK_EC2_PLACEMENT, 1000, 32, "snap-12312313", "gp2");

    Assert.assertNotNull("volume should not be null", volume);
    Assert.assertNotNull("volume id should not be null", volume.getVolumeId());
}
 

/**
 * Test describing Volumes.
 */
@Test(timeout = TIMEOUT_LEVEL1)
public final void describeVolumesAllTest() {
    log.info("Start describing volume test");
    List<Volume>  volumes = getVolumes();
    Assert.assertNotNull("volume should not be null", volumes);
    Assert.assertNotNull("volume size", volumes.size());
    log.info("Sizes " + volumes.size());
}
 

/**
 * Test describing Volumes.
 */
@Test(timeout = TIMEOUT_LEVEL1)
public final void describeVolumesTest() {
    log.info("Start describing volume test");
    createVolumesTest();
    Volume volume = getVolume();

    Assert.assertNotNull("volume should not be null", volume);
    Assert.assertNotNull("volume id should not be null", volume.getVolumeId());
    Assert.assertTrue("volume should be deleted", deleteVolume(volume.getVolumeId()));
}
 

public Volume waitVolume(AwsProcessClient awsProcessClient, String volumeId) {
    // ボリュームの処理待ち
    Volume volume;
    while (true) {
        try {
            Thread.sleep(1000L * awsProcessClient.getDescribeInterval());
        } catch (InterruptedException ignore) {
        }

        volume = describeVolume(awsProcessClient, volumeId);
        VolumeState state;
        try {
            state = VolumeState.fromValue(volume.getState());
        } catch (IllegalArgumentException e) {
            // 予期しないステータス
            AutoException exception = new AutoException("EPROCESS-000112", volume, volume.getState());
            exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(volume));
            throw exception;
        }

        // 安定状態のステータスになったら終了
        if (state == VolumeState.Available || state == VolumeState.InUse || state == VolumeState.Deleted
                || state == VolumeState.Error) {
            break;
        }
    }

    return volume;
}
 

public void createVolume(AwsProcessClient awsProcessClient, Long instanceNo, Long volumeNo) {
    AwsVolume awsVolume = awsVolumeDao.read(volumeNo);
    AwsInstance awsInstance = awsInstanceDao.read(instanceNo);

    // ボリュームの作成
    CreateVolumeRequest request = new CreateVolumeRequest();
    request.withSize(awsVolume.getSize());
    request.withSnapshotId(awsVolume.getSnapshotId());
    request.withAvailabilityZone(awsInstance.getAvailabilityZone());

    String volumeType = Config.getProperty("aws.volumeType");
    if (StringUtils.isNotEmpty(volumeType)) {
        request.withVolumeType(volumeType);
    }

    CreateVolumeResult result = awsProcessClient.getEc2Client().createVolume(request);
    Volume volume = result.getVolume();

    String volumeId = volume.getVolumeId();

    // ログ出力
    if (log.isInfoEnabled()) {
        log.info(MessageUtils.getMessage("IPROCESS-100121", volumeId));
    }

    //イベントログ出力
    Component component = null;
    if (awsVolume.getComponentNo() != null) {
        component = componentDao.read(awsVolume.getComponentNo());
    }
    Instance instance = instanceDao.read(instanceNo);
    processLogger.debug(component, instance, "AwsEbsCreate",
            new Object[] { awsProcessClient.getPlatform().getPlatformName() });

    // データベース更新
    awsVolume.setVolumeId(volume.getVolumeId());
    awsVolume.setStatus(volume.getState());
    awsVolumeDao.update(awsVolume);
}
 

/**
 * 
 * @param ec2ServiceClient
 * @param request
 * @return
 */
public static List<Volume> collectAllVolumes(AmazonEC2 ec2ServiceClient,
        DescribeVolumesRequest request) {
    DescribeVolumesResult result;
    String nextToken;
    List<Volume> volumes = new ArrayList<>();
    do {
        result = ec2ServiceClient.describeVolumes(request);
        volumes.addAll(result.getVolumes());
        nextToken = result.getNextToken();
        request.setNextToken(nextToken);
    } while (null != nextToken);
    return volumes;
}
 

/**
 * Collect all volumes.
 *
 * @param ec2ServiceClient the ec 2 service client
 * @param region the region
 * @return the list
 */
public static List<Volume> collectAllVolumes(AmazonEC2 ec2ServiceClient,Region region){
	DescribeVolumesRequest request = new DescribeVolumesRequest();
	DescribeVolumesResult result;
	String nextToken;
	List<Volume> volumes=new ArrayList<Volume>();
	do{
		result = ec2ServiceClient.describeVolumes(request);
		volumes.addAll(result.getVolumes());
		nextToken = result.getNextToken();
	    request.setNextToken(nextToken);
	}while(null!=nextToken);
	return volumes;
}
 

/**
 * Calls DescribeVolumes on the AWS EC2 Client returning all volumes that match the supplied predicate and attempting
 * to push down certain predicates (namely queries for specific volumes) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    boolean done = false;
    DescribeVolumesRequest request = new DescribeVolumesRequest();

    ValueSet idConstraint = recordsRequest.getConstraints().getSummary().get("id");
    if (idConstraint != null && idConstraint.isSingleValue()) {
        request.setVolumeIds(Collections.singletonList(idConstraint.getSingleValue().toString()));
    }

    while (!done) {
        DescribeVolumesResult response = ec2.describeVolumes(request);

        for (Volume volume : response.getVolumes()) {
            logger.info("readWithConstraint: {}", response);
            instanceToRow(volume, spiller);
        }

        request.setNextToken(response.getNextToken());

        if (response.getNextToken() == null || !queryStatusChecker.isQueryRunning()) {
            done = true;
        }
    }
}
 

public void waitDetachVolume(AwsProcessClient awsProcessClient, Long instanceNo, Long volumeNo) {
    AwsVolume awsVolume = awsVolumeDao.read(volumeNo);
    String volumeId = awsVolume.getVolumeId();
    String instanceId = awsVolume.getInstanceId();

    Volume volume = null;
    try {
        // TODO: アタッチ情報がすぐに更新されない問題に暫定的に対応
        int retry = 6;
        for (int i = 0; i < retry; i++) {
            volume = awsCommonProcess.waitVolume(awsProcessClient, volumeId);
            if (StringUtils.equals(volume.getState(), VolumeState.Available.toString())) {
                break;
            }
        }

        if (!StringUtils.equals(volume.getState(), VolumeState.Available.toString())) {
            // デタッチに失敗した場合
            AutoException exception = new AutoException("EPROCESS-000116", instanceId, volumeId, volume.getState());
            exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(volume));
            throw exception;
        }

        // ログ出力
        if (log.isInfoEnabled()) {
            log.info(MessageUtils.getMessage("IPROCESS-100126", volumeId, instanceId));
        }
    } catch (AutoException e) {
        // デタッチに失敗した場合
        awsVolume = awsVolumeDao.read(volumeNo);
        awsVolume.setStatus(VolumeState.Error.toString());
        awsVolume.setInstanceId(null);
        awsVolumeDao.update(awsVolume);

        throw e;
    }

    //イベントログ出力
    Component component = null;
    if (awsVolume.getComponentNo() != null) {
        component = componentDao.read(awsVolume.getComponentNo());
    }
    Instance instance = instanceDao.read(instanceNo);
    processLogger.debug(component, instance, "AwsEbsDetachFinish",
            new Object[] { instance.getInstanceName(), awsVolume.getVolumeId(), awsVolume.getDevice() });

    // データベースの更新
    awsVolume = awsVolumeDao.read(volumeNo);
    awsVolume.setStatus(volume.getState());
    awsVolume.setInstanceId(null);
    awsVolumeDao.update(awsVolume);
}
 

public void waitAttachVolume(AwsProcessClient awsProcessClient, Long instanceNo, Long volumeNo) {
    AwsVolume awsVolume = awsVolumeDao.read(volumeNo);
    String volumeId = awsVolume.getVolumeId();

    Volume volume = null;
    try {
        // volume = awsProcessClient.waitAttachVolume(volumeId, instanceId);
        // TODO: アタッチ情報がすぐに更新されない問題に暫定的に対応
        int retry = 6;
        for (int i = 0; i < retry; i++) {
            volume = awsCommonProcess.waitVolume(awsProcessClient, volumeId);
            if (StringUtils.equals(volume.getState(), VolumeState.InUse.toString())) {
                break;
            }
        }

        if (!StringUtils.equals(volume.getState(), VolumeState.InUse.toString())) {
            // アタッチに失敗した場合
            AutoException exception = new AutoException("EPROCESS-000115", awsVolume.getInstanceId(), volumeId,
                    volume.getState());
            exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(volume));
            throw exception;
        }

        // ログ出力
        if (log.isInfoEnabled()) {
            log.info(MessageUtils.getMessage("IPROCESS-100124", volumeId, awsVolume.getInstanceId()));
        }
    } catch (AutoException e) {
        // アタッチに失敗した場合
        awsVolume = awsVolumeDao.read(volumeNo);
        awsVolume.setStatus(VolumeState.Error.toString());
        awsVolume.setInstanceId(null);
        awsVolumeDao.update(awsVolume);

        throw e;
    }

    //イベントログ出力
    Component component = null;
    if (awsVolume.getComponentNo() != null) {
        component = componentDao.read(awsVolume.getComponentNo());
    }
    Instance instance = instanceDao.read(instanceNo);
    processLogger.debug(component, instance, "AwsEbsAttachFinish",
            new Object[] { instance.getInstanceName(), awsVolume.getVolumeId(), awsVolume.getDevice() });

    // データベースの更新
    awsVolume = awsVolumeDao.read(volumeNo);
    awsVolume.setStatus(volume.getState());
    awsVolumeDao.update(awsVolume);
}
 

public void waitCreateVolume(AwsProcessClient awsProcessClient, Long instanceNo, Long volumeNo) {
    AwsVolume awsVolume = awsVolumeDao.read(volumeNo);
    String volumeId = awsVolume.getVolumeId();

    // ボリュームの作成待ち
    Volume volume;
    try {
        volume = awsCommonProcess.waitVolume(awsProcessClient, volumeId);

        if (!StringUtils.equals(volume.getState(), VolumeState.Available.toString())) {
            // ボリューム作成失敗時
            AutoException exception = new AutoException("EPROCESS-000113", volumeId, volume.getState());
            exception.addDetailInfo("result=" + ReflectionToStringBuilder.toString(volume));
            throw exception;
        }

        // ログ出力
        if (log.isInfoEnabled()) {
            log.info(MessageUtils.getMessage("IPROCESS-100122", volumeId));
        }
    } catch (AutoException e) {
        // ボリューム作成失敗時
        awsVolume = awsVolumeDao.read(volumeNo);
        awsVolume.setVolumeId(null);
        awsVolume.setStatus(null);
        awsVolumeDao.update(awsVolume);

        throw e;
    }

    //イベントログ出力
    Component component = null;
    if (awsVolume.getComponentNo() != null) {
        component = componentDao.read(awsVolume.getComponentNo());
    }
    Instance instance = instanceDao.read(instanceNo);
    processLogger.debug(component, instance, "AwsEbsCreateFinish", new Object[] {
            awsProcessClient.getPlatform().getPlatformName(), awsVolume.getVolumeId(), awsVolume.getSize() });

    // データベース更新
    awsVolume = awsVolumeDao.read(volumeNo);
    awsVolume.setStatus(volume.getState());
    awsVolumeDao.update(awsVolume);
}