org.springframework.cloud.stream.provisioning.ProvisioningException Java Examples

@Override
public ProducerDestination provisionProducerDestination(String name,
		ExtendedProducerProperties<KinesisProducerProperties> properties)
		throws ProvisioningException {

	if (logger.isInfoEnabled()) {
		logger.info("Using Kinesis stream for outbound: " + name);
	}

	if (properties.getHeaderMode() == null) {
		properties.setHeaderMode(HeaderMode.embeddedHeaders);
	}

	return new KinesisProducerDestination(name,
			createOrUpdate(name, properties.getPartitionCount()));
}
 

@Override
public ConsumerDestination provisionConsumerDestination(String name, String group,
		ExtendedConsumerProperties<KinesisConsumerProperties> properties)
		throws ProvisioningException {

	if (properties.getExtension().isDynamoDbStreams()) {
		if (logger.isInfoEnabled()) {
			logger.info("Using DynamoDB table in DynamoDB Streams support for inbound: " + name);
		}
		return new KinesisConsumerDestination(name, Collections.emptyList());
	}

	if (logger.isInfoEnabled()) {
		logger.info("Using Kinesis stream for inbound: " + name);
	}

	if (properties.getHeaderMode() == null) {
		properties.setHeaderMode(HeaderMode.embeddedHeaders);
	}

	int shardCount = properties.getInstanceCount() * properties.getConcurrency();

	return new KinesisConsumerDestination(name, createOrUpdate(name, shardCount));
}
 

private Exchange buildExchange(RabbitCommonProperties properties,
		String exchangeName) {
	try {
		ExchangeBuilder builder = new ExchangeBuilder(exchangeName,
				properties.getExchangeType());
		builder.durable(properties.isExchangeDurable());
		if (properties.isExchangeAutoDelete()) {
			builder.autoDelete();
		}
		if (properties.isDelayedExchange()) {
			builder.delayed();
		}
		return builder.build();
	}
	catch (Exception e) {
		throw new ProvisioningException("Failed to create exchange object", e);
	}
}
 

private void createTopic(AdminClient adminClient, String name, int partitionCount,
		boolean tolerateLowerPartitionsOnBroker, KafkaTopicProperties properties) {
	try {
		createTopicIfNecessary(adminClient, name, partitionCount,
				tolerateLowerPartitionsOnBroker, properties);
	}
	// TODO: Remove catching Throwable. See this thread:
	// TODO:
	// https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940
	catch (Throwable throwable) {
		if (throwable instanceof Error) {
			throw (Error) throwable;
		}
		else {
			// TODO:
			// https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940
			throw new ProvisioningException("Provisioning exception", throwable);
		}
	}
}
 

private Topic makeSureTopicExists(String topicName, boolean autoCreate) {
	Topic topic = this.pubSubAdmin.getTopic(topicName);
	if (topic == null) {
		if (autoCreate) {
			try {
				topic = this.pubSubAdmin.createTopic(topicName);
			}
			catch (AlreadyExistsException alreadyExistsException) {
				// Ignore concurrent topic creation - we're good as long as topic was created and exists
				LOGGER.info("Failed to auto-create topic '" + topicName + "' because it already exists.");
			}
		}
		else {
			throw new ProvisioningException("Non-existing '" + topicName + "' topic.");
		}
	}

	return topic;
}
 

@Test
public void testProvisionConsumerDestination_noTopicException() {
	this.expectedEx.expect(ProvisioningException.class);
	this.expectedEx.expectMessage("Non-existing 'topic_A' topic.");

	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(false);
	when(this.pubSubAdminMock.getTopic("topic_A")).thenReturn(null);

	PubSubConsumerDestination result = (PubSubConsumerDestination) this.pubSubChannelProvisioner
			.provisionConsumerDestination("topic_A", "group_A", this.properties);
}
 

/**
 * Will provision consumer destination as SI {@link DirectChannel}.
 */
@Override
public ConsumerDestination provisionConsumerDestination(String name, String group,
		ConsumerProperties properties) throws ProvisioningException {
	SubscribableChannel destination = this.provisionDestination(name, false);
	if (this.source != null) {
		this.source.setChannel(destination);
	}
	return new SpringIntegrationConsumerDestination(name, destination);
}
 

/**
 * Will provision producer destination as an SI {@link PublishSubscribeChannel}. <br>
 * This provides convenience of registering additional subscriber (handler in the test
 * method) along side of being able to call {@link OutputDestination#receive()} to get
 * a {@link Message} for additional assertions.
 */
@Override
public ProducerDestination provisionProducerDestination(String name,
		ProducerProperties properties) throws ProvisioningException {
	SubscribableChannel destination = this.provisionDestination(name, true);
	this.target.setChannel(destination);
	return new SpringIntegrationProducerDestination(name, destination);
}
 

@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledFailsIfTopicUnderPartitionedAndAutoRebalanceDisabled()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 1, 1);
	configurationProperties.setAutoAddPartitions(false);
	Binder binder = getBinder(configurationProperties);
	GenericApplicationContext context = new GenericApplicationContext();
	context.refresh();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	DirectChannel output = createBindableChannel("output",
			createConsumerBindingProperties(consumerProperties));
	// this consumer must consume from partition 2
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(2);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	expectedProvisioningException.expect(ProvisioningException.class);
	expectedProvisioningException.expectMessage(
			"The number of expected partitions was: 3, but 1 has been found instead");
	Binding binding = binder.bindConsumer(testTopicName, "test", output,
			consumerProperties);
	if (binding != null) {
		binding.unbind();
	}
}
 

private ConsumerDestination createDlqIfNeedBe(AdminClient adminClient, String name,
		String group, ExtendedConsumerProperties<KafkaConsumerProperties> properties,
		boolean anonymous, int partitions) {

	if (properties.getExtension().isEnableDlq() && !anonymous) {
		String dlqTopic = StringUtils.hasText(properties.getExtension().getDlqName())
				? properties.getExtension().getDlqName()
				: "error." + name + "." + group;
		int dlqPartitions = properties.getExtension().getDlqPartitions() == null
				? partitions
				: properties.getExtension().getDlqPartitions();
		try {
			createTopicAndPartitions(adminClient, dlqTopic, dlqPartitions,
					properties.getExtension().isAutoRebalanceEnabled(),
					properties.getExtension().getTopic());
		}
		catch (Throwable throwable) {
			if (throwable instanceof Error) {
				throw (Error) throwable;
			}
			else {
				throw new ProvisioningException("provisioning exception", throwable);
			}
		}
		return new KafkaConsumerDestination(name, partitions, dlqTopic);
	}
	return null;
}
 

@Override
public ProducerDestination provisionProducerDestination(final String name,
		ExtendedProducerProperties<KafkaProducerProperties> properties) {

	if (logger.isInfoEnabled()) {
		logger.info("Using kafka topic for outbound: " + name);
	}
	KafkaTopicUtils.validateTopicName(name);
	try (AdminClient adminClient = AdminClient.create(this.adminClientProperties)) {
		createTopic(adminClient, name, properties.getPartitionCount(), false,
				properties.getExtension().getTopic());
		int partitions = 0;
		Map<String, TopicDescription> topicDescriptions = new HashMap<>();
		if (this.configurationProperties.isAutoCreateTopics()) {
			this.metadataRetryOperations.execute(context -> {
				try {
					if (logger.isDebugEnabled()) {
						logger.debug("Attempting to retrieve the description for the topic: " + name);
					}
					DescribeTopicsResult describeTopicsResult = adminClient
							.describeTopics(Collections.singletonList(name));
					KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult
							.all();
					topicDescriptions.putAll(all.get(this.operationTimeout, TimeUnit.SECONDS));
				}
				catch (Exception ex) {
					throw new ProvisioningException("Problems encountered with partitions finding", ex);
				}
				return null;
			});
		}
		TopicDescription topicDescription = topicDescriptions.get(name);
		if (topicDescription != null) {
			partitions = topicDescription.partitions().size();
		}
		return new KafkaProducerDestination(name, partitions);
	}
}
 

@Test
public void testProvisionConsumerDestination_wrongTopicException() {
	this.expectedEx.expect(ProvisioningException.class);
	this.expectedEx.expectMessage("Existing 'topic_A.group_A' subscription is for a different topic 'topic_B'.");

	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(false);
	when(this.pubSubAdminMock.getSubscription("topic_A.group_A")).thenReturn(Subscription.newBuilder().setTopic("topic_B").build());

	PubSubConsumerDestination result = (PubSubConsumerDestination) this.pubSubChannelProvisioner
			.provisionConsumerDestination("topic_A", "group_A", this.properties);
}
 

@Test
public void testProvisionConsumerDestination_noSubscriptionException() {
	this.expectedEx.expect(ProvisioningException.class);
	this.expectedEx.expectMessage("Non-existing 'topic_A.group_A' subscription.");

	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(false);

	PubSubConsumerDestination result = (PubSubConsumerDestination) this.pubSubChannelProvisioner
			.provisionConsumerDestination("topic_A", "group_A", this.properties);
}
 

@Override
public ProducerDestination provisionProducerDestination(String name,
		ExtendedProducerProperties<RocketMQProducerProperties> properties)
		throws ProvisioningException {
	checkTopic(name);
	return new RocketProducerDestination(name);
}
 

@Override
public ConsumerDestination provisionConsumerDestination(String topicName, String group,
		ExtendedConsumerProperties<PubSubConsumerProperties> properties)
		throws ProvisioningException {

	Topic topic = makeSureTopicExists(topicName, properties.getExtension().isAutoCreateResources());

	String subscriptionName;
	Subscription subscription;
	if (StringUtils.hasText(group)) {
		// Use <topicName>.<group> as subscription name
		subscriptionName = topicName + "." + group;
		subscription = this.pubSubAdmin.getSubscription(subscriptionName);
	}
	else {
		// Generate anonymous random group since one wasn't provided
		subscriptionName = "anonymous." + topicName + "." + UUID.randomUUID().toString();
		subscription = this.pubSubAdmin.createSubscription(subscriptionName, topicName);
		this.anonymousGroupSubscriptionNames.add(subscriptionName);
	}

	// make sure subscription exists
	if (subscription == null) {
		if (properties.getExtension().isAutoCreateResources()) {
			this.pubSubAdmin.createSubscription(subscriptionName, topicName);
		}
		else {
			throw new ProvisioningException("Non-existing '" + subscriptionName + "' subscription.");
		}
	}
	else if (!subscription.getTopic().equals(topic.getName())) {
		throw new ProvisioningException(
				"Existing '" + subscriptionName + "' subscription is for a different topic '"
						+ subscription.getTopic() + "'.");
	}
	return new PubSubConsumerDestination(subscriptionName);
}
 

@Override
public ProducerDestination provisionProducerDestination(String topic,
		ExtendedProducerProperties<PubSubProducerProperties> properties)
		throws ProvisioningException {
	makeSureTopicExists(topic, properties.getExtension().isAutoCreateResources());

	return new PubSubProducerDestination(topic);
}
 

@Test
void testProvisionConsumerResourceNotFoundException() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setAutoCreateStream(false);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("Stream not found"));

	assertThatThrownBy(() -> provisioner.provisionConsumerDestination(name, group,
			extendedConsumerProperties))
			.isInstanceOf(ProvisioningException.class)
			.hasMessageContaining(
					"The stream [test-stream] was not found and auto creation is disabled.")
			.hasCauseInstanceOf(ResourceNotFoundException.class);

	verify(amazonKinesisMock, times(1))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock, never()).createStream(name,
			instanceCount * concurrency);
}
 

@Override
protected MessageProducer createConsumerEndpoint(ConsumerDestination destination,
		String group,
		ExtendedConsumerProperties<KinesisConsumerProperties> properties) {

	ConsumerDestination destinationToUse = destination;

	if (properties.getExtension().isDynamoDbStreams()) {
		DescribeTableResult describeTableResult = this.dynamoDBClient.describeTable(destinationToUse.getName());
		String latestStreamArn = describeTableResult.getTable().getLatestStreamArn();
		if (StringUtils.hasText(latestStreamArn)) {
			destinationToUse = new KinesisConsumerDestination(latestStreamArn, Collections.emptyList());
		}
		else {
			throw new ProvisioningException("The DynamoDB table ["
					+ destinationToUse.getName()
					+ "] doesn't have Streams enabled.");
		}
	}
	else {
		this.streamsInUse.add(destinationToUse.getName());
	}

	MessageProducer adapter;
	if (this.configurationProperties.isKplKclEnabled()) {
		adapter = createKclConsumerEndpoint(destinationToUse, group, properties);
	}
	else {
		adapter = createKinesisConsumerEndpoint(destinationToUse, group, properties);
	}

	return adapter;
}
 

private List<Shard> createOrUpdate(String stream, int shards) {
	List<Shard> shardList = new ArrayList<>();
	try {
		shardList = getShardList(stream);
	}
	catch (ResourceNotFoundException ex) {
		if (!this.configurationProperties.isAutoCreateStream()) {
			throw new ProvisioningException(
					"The stream [" + stream + "] was not found and auto creation is disabled.", ex);
		}
		if (logger.isInfoEnabled()) {
			logger.info("Stream '" + stream + "' not found. Create one...");
		}

		this.amazonKinesis.createStream(stream, Math.max(this.configurationProperties.getMinShardCount(), shards));

	}

	int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards);

	if ((shardList.size() < effectiveShardCount)
			&& this.configurationProperties.isAutoAddShards()) {
		return updateShardCount(stream, shardList.size(), effectiveShardCount);
	}

	return shardList;
}
 

private List<Shard> getShardList(String stream, int retryCount) {
	List<Shard> shardList = new ArrayList<>();

	if (retryCount > configurationProperties.getDescribeStreamRetries()) {
		ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
				"The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
		resourceNotFoundException.setServiceName("Kinesis");

		throw new ProvisioningException(
				"Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
				resourceNotFoundException);
	}

	ListShardsRequest listShardsRequest = new ListShardsRequest().withStreamName(stream);

	try {
		ListShardsResult listShardsResult = amazonKinesis.listShards(listShardsRequest);

		shardList.addAll(listShardsResult.getShards());

	}
	catch (LimitExceededException limitExceededException) {
		logger.info("Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for ["
				+ this.configurationProperties.getDescribeStreamBackoff() + "] millis.");

		try {
			Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
			getShardList(stream, retryCount++);
		}
		catch (InterruptedException ex) {
			Thread.currentThread().interrupt();
			throw new ProvisioningException(
					"The [describeStream] thread for the stream [" + stream + "] has been interrupted.", ex);
		}
	}

	return shardList;
}
 

@Override
public ConsumerDestination provisionConsumerDestination(String name, String group,
		ExtendedConsumerProperties<RocketMQConsumerProperties> properties)
		throws ProvisioningException {
	checkTopic(name);
	return new RocketConsumerDestination(name);
}
 

private ConsumerDestination doProvisionConsumerDestination(final String name,
		final String group,
		ExtendedConsumerProperties<KafkaConsumerProperties> properties) {

	if (properties.getExtension().isDestinationIsPattern()) {
		Assert.isTrue(!properties.getExtension().isEnableDlq(),
				"enableDLQ is not allowed when listening to topic patterns");
		if (logger.isDebugEnabled()) {
			logger.debug("Listening to a topic pattern - " + name
					+ " - no provisioning performed");
		}
		return new KafkaConsumerDestination(name);
	}
	KafkaTopicUtils.validateTopicName(name);
	boolean anonymous = !StringUtils.hasText(group);
	Assert.isTrue(!anonymous || !properties.getExtension().isEnableDlq(),
			"DLQ support is not available for anonymous subscriptions");
	if (properties.getInstanceCount() == 0) {
		throw new IllegalArgumentException("Instance count cannot be zero");
	}
	int partitionCount = properties.getInstanceCount() * properties.getConcurrency();
	ConsumerDestination consumerDestination = new KafkaConsumerDestination(name);
	try (AdminClient adminClient = createAdminClient()) {
		createTopic(adminClient, name, partitionCount,
				properties.getExtension().isAutoRebalanceEnabled(),
				properties.getExtension().getTopic());
		if (this.configurationProperties.isAutoCreateTopics()) {
			DescribeTopicsResult describeTopicsResult = adminClient
					.describeTopics(Collections.singletonList(name));
			KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult
					.all();
			try {
				Map<String, TopicDescription> topicDescriptions = all
						.get(this.operationTimeout, TimeUnit.SECONDS);
				TopicDescription topicDescription = topicDescriptions.get(name);
				int partitions = topicDescription.partitions().size();
				consumerDestination = createDlqIfNeedBe(adminClient, name, group,
						properties, anonymous, partitions);
				if (consumerDestination == null) {
					consumerDestination = new KafkaConsumerDestination(name,
							partitions);
				}
			}
			catch (Exception ex) {
				throw new ProvisioningException("provisioning exception", ex);
			}
		}
	}
	return consumerDestination;
}