software.amazon.awssdk.services.kinesis.KinesisClient Java Examples

public static void addShards(KinesisClient kinesisClient, String name , int goalShards) {

        try {
             UpdateShardCountRequest request = UpdateShardCountRequest.builder()
                .scalingType("UNIFORM_SCALING")
                .streamName(name)
                .targetShardCount(goalShards)
                .build();

            UpdateShardCountResponse response = kinesisClient.updateShardCount(request);
            System.out.println(response.streamName() + " has updated shard count to " + response.currentShardCount());
           
        } catch (KinesisException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
        System.out.println("Done");
    }
 

/**
 * Wait for a Stream to become available or transition to the indicated status
 *
 * @param streamName
 * @param status
 * @throws Exception
 */
public static void waitForStreamStatus(KinesisClient kinesisClient, String streamName, String status)
		throws Exception {
	boolean ok = false;
	String streamStatus;
	// stream mutation takes around 30 seconds, so we'll start with 20 as
	// a timeout
	int waitTimeout = 20000;
	do {
		streamStatus = getStreamStatus(kinesisClient, streamName);
		if (!streamStatus.equals(status)) {
			Thread.sleep(waitTimeout);
			// reduce the wait timeout from the initial wait time
			waitTimeout = 1000;
		} else {
			ok = true;
		}
	} while (!ok);
}
 

private static void validateStream(KinesisClient kinesisClient, String streamName) {
    try {
        DescribeStreamRequest describeStreamRequest = DescribeStreamRequest.builder()
                .streamName(streamName)
                .build();

        DescribeStreamResponse describeStreamResponse = kinesisClient.describeStream(describeStreamRequest);

        if(!describeStreamResponse.streamDescription().streamStatus().toString().equals("ACTIVE")) {
            System.err.println("Stream " + streamName + " is not active. Please wait a few moments and try again.");
            System.exit(1);
        }
    }catch (KinesisException e) {
        System.err.println("Error found while describing the stream " + streamName);
        System.err.println(e);
        System.exit(1);
    }
    // snippet-end:[kinesis.java2.putrecord.main]
}
 

private static void sendStockTrade(StockTrade trade, KinesisClient kinesisClient,
                                   String streamName) {
    byte[] bytes = trade.toJsonAsBytes();
    // The bytes could be null if there is an issue with the JSON serialization by the Jackson JSON library.
    if (bytes == null) {
        System.out.println("Could not get JSON bytes for stock trade");
        return;
    }

    System.out.println("Putting trade: " + trade.toString());
    PutRecordRequest request = PutRecordRequest.builder()
            .partitionKey(trade.getTickerSymbol()) // We use the ticker symbol as the partition key, explained in the Supplemental Information section below.
            .streamName(streamName)
            .data(SdkBytes.fromByteArray(bytes))
            .build();
    try {
        kinesisClient.putRecord(request);
    } catch (KinesisException e) {
        e.getMessage();
    }
}
 

public static void setStockData( KinesisClient kinesisClient, String streamName) {

            try {
            // Repeatedly send stock trades with a 100 milliseconds wait in between
            StockTradeGenerator stockTradeGenerator = new StockTradeGenerator();

            // Put in 50 records for this example
            int index = 50;
            for (int x=0; x<index; x++){
                StockTrade trade = stockTradeGenerator.getRandomTrade();
                sendStockTrade(trade, kinesisClient, streamName);
                Thread.sleep(100);
             }

        } catch (KinesisException | InterruptedException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
        System.out.println("Done");
    }
 

public static void main(String[] args) {

       final String USAGE = "\n" +
                "Usage:\n" +
                "    StockTradesWriter <streamName>\n\n" +
                "Where:\n" +
                "    streamName - The Kinesis data stream to which records are written (i.e., StockTradeStream)\n\n" +
                "Example:\n" +
                "    StockTradesWriter streamName\n";

        if (args.length < 1) {
            System.out.println(USAGE);
            System.exit(1);
        }

        String streamName = args[0];

        Region region = Region.US_EAST_1;
        KinesisClient kinesisClient = KinesisClient.builder()
                    .region(region)
                    .build();

        // Ensure that the Kinesis stream is valid
        validateStream(kinesisClient, streamName);
        setStockData( kinesisClient, streamName);
    }
 

public static void mergeShards(final KinesisClient kinesisClient, final String streamName,
		final ShardHashInfo lowerShard, final ShardHashInfo higherShard, final boolean waitForActive)
		throws Exception {
	LOG.debug(String.format("Merging Shard %s and %s", lowerShard, higherShard));

	KinesisOperation merge = new KinesisOperation() {
		public Object run(KinesisClient client) {
			final MergeShardsRequest req = MergeShardsRequest.builder().streamName(streamName)
					.shardToMerge(lowerShard.getShardId()).adjacentShardToMerge(higherShard.getShardId()).build();
			client.mergeShards(req);

			return null;
		}
	};
	doOperation(kinesisClient, merge, streamName, MODIFY_RETRIES, waitForActive);
}
 

public static void main(String[] args) {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    GetRecords <streamName>\n\n" +
                "Where:\n" +
                "    streamName - The Kinesis data stream to read from (i.e., StockTradeStream)\n\n" +
                "Example:\n" +
                "    GetRecords streamName\n";

        if (args.length < 1) {
            System.out.println(USAGE);
            System.exit(1);
        }

        String streamName = args[0];

        Region region = Region.US_EAST_1;
        KinesisClient kinesisClient = KinesisClient.builder()
                .region(region)
                .build();

        getStockTrades(kinesisClient,streamName);
    }
 

public static void main(String[] args) {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    CreateDataStream <streamName>\n\n" +
                "Where:\n" +
                "    CreateDataStream - The Kinesis data stream (i.e., StockTradeStream)\n\n" +
                "Example:\n" +
                "    CreateDataStream StockTradeStream\n";

        if (args.length < 1) {
            System.out.println(USAGE);
            System.exit(1);
        }
        String streamName = args[0];

        Region region = Region.US_EAST_1;
        KinesisClient kinesisClient = KinesisClient.builder()
                .region(region)
                .build();

        createStream(kinesisClient, streamName);
    }
 

/**
 * Merge these two Shards and return the result Shard
 * 
 * @param kinesisClient
 * @return
 * @throws Exception
 */
protected ShardHashInfo doMerge(KinesisClient kinesisClient, String currentHighestShardId) throws Exception {
	StreamScalingUtils.mergeShards(kinesisClient, streamName, this.lowerShard, this.higherShard, true);

	Map<String, ShardHashInfo> openShards = StreamScalingUtils.getOpenShards(kinesisClient, streamName,
			currentHighestShardId);

	for (ShardHashInfo info : openShards.values()) {
		if (lowerShard.getShardId().equals(info.getShard().parentShardId())
				&& higherShard.getShardId().equals(info.getShard().adjacentParentShardId())) {
			return new ShardHashInfo(streamName, info.getShard());
		}
	}

	throw new Exception(String.format("Unable resolve new created Shard for parents %s and %s",
			lowerShard.getShardId(), higherShard.getShardId()));
}
 

public static void deleteStream(KinesisClient kinesisClient, String streamName) {

           try {

                DeleteStreamRequest delStream = DeleteStreamRequest.builder()
                        .streamName(streamName)
                        .build();

                kinesisClient.deleteStream(delStream);

            } catch (KinesisException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
        System.out.println("Done");
    }
 

public static void main(String[] args) {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    DeleteDataStream <streamName>\n\n" +
                "Where:\n" +
                "    streamName - The Kinesis data stream (i.e., StockTradeStream)\n\n" +
                "Example:\n" +
                "    DeleteDataStream StockTradeStream\n";

        if (args.length < 1) {
            System.out.println(USAGE);
            System.exit(1);
        }

        String streamName = args[0];

        Region region = Region.US_EAST_1;
        KinesisClient kinesisClient = KinesisClient.builder()
                .region(region)
                .build();

        deleteStream(kinesisClient, streamName);
    }
 

public static void listKinShards(KinesisClient kinesisClient, String name) {

        try {
        ListShardsRequest request = ListShardsRequest.builder()
                .streamName(name)
                .build();

            ListShardsResponse response = kinesisClient.listShards(request);
            System.out.println(request.streamName() + " has " + response.shards());

        } catch (KinesisException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
        System.out.println("Done");
    }
 

public static void main(String[] args) {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    ListShards <streamName>\n\n" +
                "Where:\n" +
                "    streamName - The Kinesis data stream (i.e., StockTradeStream)\n\n" +
                "Example:\n" +
                "    ListShards StockTradeStream\n";

        if (args.length < 1) {
            System.out.println(USAGE);
            System.exit(1);
        }

        String name = args[0];

        // snippet-start:[kinesis.java2.ListShards.client]
        Region region = Region.US_EAST_1;
        KinesisClient kinesisClient = KinesisClient.builder()
                    .region(region)
                    .build();
           // snippet-end:[kinesis.java2.ListShards.client]

        listKinShards(kinesisClient, name);
        }
 

public static void describeKinLimits(KinesisClient kinesisClient) {

        // snippet-end:[kinesis.java2.DescribeLimits.client]
        try {
        // snippet-start:[kinesis.java2.DescribeLimits.main]
        DescribeLimitsRequest request = DescribeLimitsRequest.builder()
                 .build();

        DescribeLimitsResponse response = kinesisClient.describeLimits(request);

        System.out.println("Number of open shards: " + response.openShardCount());
        System.out.println("Maximum shards allowed: " + response.shardLimit());
        } catch (KinesisException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
        System.out.println("Done");
        // snippet-end:[kinesis.java2.DescribeLimits.main]
    }
 

public StreamScaler(Region region) throws Exception {
	pctFormat.setMaximumFractionDigits(1);
	
	// use the default provider chain plus support for classpath
	// properties files
	KinesisClientBuilder builder = KinesisClient.builder()
			.credentialsProvider(DefaultCredentialsProvider.builder().build()).region(region);

	String kinesisEndpoint = System.getProperty("kinesisEndpoint");
	if (kinesisEndpoint != null) {
		builder.endpointOverride(new URI(kinesisEndpoint));
	}

	this.kinesisClient = builder.build();

	if (kinesisClient.serviceName() == null) {
		throw new Exception("Unable to reach Kinesis Service");
	}
}
 

protected static void createKinesisStream() {
    kinesis = KinesisClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN).region(Region.US_WEST_2).build();

    kinesis.createStream(CreateStreamRequest.builder().streamName(KINESIS_STREAM_NAME).shardCount(1).build());

    StreamDescription description = kinesis.describeStream(DescribeStreamRequest.builder().streamName(KINESIS_STREAM_NAME).build())
            .streamDescription();
    streamArn = description.streamARN();

    // Wait till stream is active (less than a minute)
    Instant start = Instant.now();
    while (StreamStatus.ACTIVE != description.streamStatus()) {
        if (Duration.between(start, Instant.now()).toMillis() > MAX_WAIT_TIME.toMillis()) {
            throw new RuntimeException("Timed out waiting for stream to become active");
        }
        try {
            Thread.sleep(5000);
        } catch (InterruptedException ignored) {
            // Ignored or expected.
        }

        description = kinesis.describeStream(DescribeStreamRequest.builder().streamName(KINESIS_STREAM_NAME).build())
                .streamDescription();
    }
}
 

public static Shard getShard(final KinesisClient kinesisClient, final String streamName, final String shardIdStart)
		throws Exception {
	LOG.debug(String.format("Getting Shard %s for Stream %s", shardIdStart, streamName));

	KinesisOperation describe = new KinesisOperation() {
		public Object run(KinesisClient client) {
			// reduce the shardIdStart by 1 as the API uses it as an exclusive start key not
			// a filter
			String shardIdToQuery = new BigDecimal(shardIdStart).subtract(new BigDecimal("1")).toString();
			ListShardsRequest req = ListShardsRequest.builder().streamName(streamName)
					.exclusiveStartShardId(shardIdToQuery).build();
			ListShardsResponse result = client.listShards(req);

			return result.shards().get(0);
		}
	};
	return (Shard) doOperation(kinesisClient, describe, streamName, DESCRIBE_RETRIES, false);
}
 

public StreamMetricManager(String streamName, int cloudWatchPeriod, List<KinesisOperationType> types,
		CloudWatchClient cloudWatchClient, KinesisClient kinesisClient) {
	this.streamName = streamName;
	this.trackedOperations.addAll(types);
	this.cloudWatchClient = cloudWatchClient;
	this.kinesisClient = kinesisClient;
	this.cloudWatchPeriod = cloudWatchPeriod;

	for (KinesisOperationType op : this.trackedOperations) {
		// create CloudWatch request templates for the information we have
		// at this point
		for (String metricName : op.getMetricsToFetch()) {
			GetMetricStatisticsRequest.Builder cwRequestBuilder = GetMetricStatisticsRequest.builder();

			cwRequestBuilder.namespace(CW_NAMESPACE)
					.dimensions(Dimension.builder().name("StreamName").value(this.streamName).build())
					.period(cloudWatchPeriod).statistics(Statistic.SUM).metricName(metricName);

			if (!this.cloudwatchRequestTemplates.containsKey(op)) {
				this.cloudwatchRequestTemplates.put(op, new ArrayList<GetMetricStatisticsRequest.Builder>() {
					{
						add(cwRequestBuilder);
					}
				});
			} else {
				this.cloudwatchRequestTemplates.get(op).add(cwRequestBuilder);
			}
		}
	}
}
 

public static void splitShard(final KinesisClient kinesisClient, final String streamName, final String shardId,
		final BigInteger targetHash, final boolean waitForActive) throws Exception {
	LOG.debug(String.format("Splitting Shard %s at %s", shardId, targetHash.toString()));

	KinesisOperation split = new KinesisOperation() {
		public Object run(KinesisClient client) {
			final SplitShardRequest req = SplitShardRequest.builder().streamName(streamName).shardToSplit(shardId)
					.newStartingHashKey(targetHash.toString()).build();
			client.splitShard(req);

			return null;
		}
	};
	doOperation(kinesisClient, split, streamName, MODIFY_RETRIES, waitForActive);
}
 

private static Object doOperation(KinesisClient kinesisClient, KinesisOperation operation, String streamName,
		int retries, boolean waitForActive) throws Exception {
	boolean done = false;
	int attempts = 0;
	Object result = null;
	do {
		attempts++;
		try {
			result = operation.run(kinesisClient);

			if (waitForActive) {
				waitForStreamStatus(kinesisClient, streamName, "ACTIVE");
			}
			done = true;
		} catch (ResourceInUseException e) {
			// thrown when the Shard is mutating - wait until we are able to
			// do the modification or ResourceNotFoundException is thrown
			Thread.sleep(1000);
		} catch (LimitExceededException lee) {
			// API Throttling
			LOG.warn(String.format("LimitExceededException for Stream %s", streamName));

			Thread.sleep(getTimeoutDuration(attempts));
		}
	} while (!done && attempts < retries);

	if (!done) {
		throw new Exception(String.format("Unable to Complete Kinesis Operation after %s Retries", retries));
	} else {
		return result;
	}
}
 

public static ShardHashInfo getOpenShard(KinesisClient kinesisClient, String streamName, String shardId)
		throws Exception {
	Shard s = getShard(kinesisClient, streamName, shardId);

	if (!s.shardId().equals(shardId)) {
		throw new Exception(String.format("Shard %s not found in Stream %s", shardId, streamName));
	} else {
		return new ShardHashInfo(streamName, s);
	}
}
 

public StreamMonitor(AutoscalingConfiguration config) throws Exception {
	this.config = config;
	Region setRegion = Region.of(this.config.getRegion());

	// setup credential refresh
	this.credentials = DefaultCredentialsProvider.builder().asyncCredentialUpdateEnabled(true).build();

	// create scaler class and clients
	this.cloudWatchClient = CloudWatchClient.builder().credentialsProvider(this.credentials).region(setRegion).build();
	this.kinesisClient = KinesisClient.builder().credentialsProvider(this.credentials).region(setRegion).build();
	this.snsClient = SnsClient.builder().credentialsProvider(this.credentials).region(setRegion).build();
	
	this.scaler = new StreamScaler(this.kinesisClient);
}
 

@BeforeAll
static void setUpOnce() throws IOException {
  discoverAssets(
      KinesisClient.class,
      AwsKinesisScanner::new,
      api -> {
        when(api.listStreams())
            .thenReturn(
                ListStreamsResponse.builder()
                    .streamNames("stream-encrypted", "stream-not-encrypted")
                    .build());

        when(api.describeStream(
                DescribeStreamRequest.builder().streamName("stream-encrypted").build()))
            .thenReturn(
                DescribeStreamResponse.builder()
                    .streamDescription(
                        StreamDescription.builder()
                            .streamARN("arn:aws:kinesis:us-east-1:111122223333:encrypted")
                            .encryptionType(EncryptionType.KMS)
                            .build())
                    .build());

        when(api.describeStream(
                DescribeStreamRequest.builder().streamName("stream-not-encrypted").build()))
            .thenReturn(
                DescribeStreamResponse.builder()
                    .streamDescription(
                        StreamDescription.builder()
                            .streamARN("arn:aws:kinesis:us-east-1:111122223333:unencrypted")
                            .encryptionType(EncryptionType.NONE)
                            .build())
                    .build());
      });
}
 

/**
 * Split the contained Shard at the indicated target percentage of keyspace
 * 
 * @param kinesisClient
 * @param targetPct
 * @return
 * @throws Exception
 */
public AdjacentShards doSplit(KinesisClient kinesisClient, double targetPct, String currentHighestShardId)
		throws Exception {
	BigInteger targetHash = getHashAtPctOffset(targetPct);

	// split the shard
	StreamScalingUtils.splitShard(kinesisClient, this.streamName, this.getShardId(), targetHash, true);

	ShardHashInfo lowerShard = null;
	ShardHashInfo higherShard = null;

	// resolve the newly created shards from this one
	Map<String, ShardHashInfo> openShards = StreamScalingUtils.getOpenShards(kinesisClient, streamName,
			currentHighestShardId);

	for (ShardHashInfo info : openShards.values()) {
		if (!info.getShard().shardId().equals(this.shard.shardId())) {
			if (info.getShard().hashKeyRange().startingHashKey().equals(targetHash.toString())) {
				higherShard = new ShardHashInfo(this.streamName, info.getShard());
				break;
			} else {
				lowerShard = new ShardHashInfo(this.streamName, info.getShard());
			}
		}
	}

	if (lowerShard == null || higherShard == null) {
		throw new Exception(String.format("Unable to resolve high/low shard mapping for Target Hash Value %s",
				targetHash.toString()));
	}

	return new AdjacentShards(streamName, lowerShard, higherShard);
}
 

public static List<Shard> listShards(final KinesisClient kinesisClient, final String streamName,
		final String shardIdStart) throws Exception {
	LOG.debug(String.format("Listing Stream %s from Shard %s", streamName, shardIdStart));

	KinesisOperation describe = new KinesisOperation() {
		public Object run(KinesisClient client) {
			ListShardsRequest.Builder builder = ListShardsRequest.builder().streamName(streamName);
			ListShardsRequest req = null;
			boolean hasMoreResults = true;
			List<Shard> shards = new ArrayList<>();

			while (hasMoreResults) {
				if (shardIdStart != null && (req != null && req.nextToken() == null)) {
					builder.exclusiveStartShardId(shardIdStart);
				}
				ListShardsResponse result = client.listShards(builder.build());
				shards.addAll(result.shards());

				if (result.nextToken() == null) {
					hasMoreResults = false;
				} else {
					req = ListShardsRequest.builder().nextToken(result.nextToken()).build();
				}

			}
			return shards;
		}
	};
	return (List<Shard>) doOperation(kinesisClient, describe, streamName, DESCRIBE_RETRIES, false);
}
 

public static StreamDescriptionSummary describeStream(final KinesisClient kinesisClient, final String streamName)
		throws Exception {
	KinesisOperation describe = new KinesisOperation() {
		public Object run(KinesisClient client) {
			DescribeStreamSummaryResponse result = client
					.describeStreamSummary(DescribeStreamSummaryRequest.builder().streamName(streamName).build());

			return result.streamDescriptionSummary();
		}
	};
	return (StreamDescriptionSummary) doOperation(kinesisClient, describe, streamName, DESCRIBE_RETRIES, false);
}
 

@BeforeAll
public static void setUp() throws IOException {

    Region region = Region.US_EAST_1;
    kinesisClient = KinesisClient.builder()
            .region(region)
            .build();
    try (InputStream input = KinesisServiceIntegrationTest.class.getClassLoader().getResourceAsStream("config.properties")) {

        Properties prop = new Properties();

        if (input == null) {
            System.out.println("Sorry, unable to find config.properties");
            return;
        }

        //load a properties file from class path, inside static method
        prop.load(input);

        // Populate the data members required for all tests
        streamName = prop.getProperty("streamName");
        existingDataStream = prop.getProperty("existingDataStream");


    } catch (IOException ex) {
        ex.printStackTrace();
    }
}
 

public static void createStream(KinesisClient kinesisClient, String streamName) {

        try {
            CreateStreamRequest streamReq = CreateStreamRequest.builder()
                    .streamName(streamName)
                    .shardCount(1)
                    .build();

            kinesisClient.createStream(streamReq);
        } catch (KinesisException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
        System.out.println("Done");
    }
 

public static void main(String[] args) {

        final String USAGE = "\n" +
                "Usage:\n" +
                "    AddDataShards <streamName>\n\n" +
                "Where:\n" +
                "    streamName - The Kinesis data stream (i.e., StockTradeStream)\n\n" +
                "Example:\n" +
                "    AddDataShards StockTradeStream\n";

        if (args.length < 1) {
            System.out.println(USAGE);
            System.exit(1);
        }

        String name = args[0];
        String inputShards = "2";

        int goalShards = Integer.parseInt(inputShards);

        // snippet-start:[kinesis.java2.AddDataShards.client]
        Region region = Region.US_EAST_1;
        KinesisClient kinesisClient = KinesisClient.builder()
                .region(region)
                .build();
        // snippet-end:[kinesis.java2.AddDataShards.client]

        addShards(kinesisClient, name, goalShards);
    }