Java Code Examples for com.google.pubsub.v1.ProjectSubscriptionName#format()

public List<ReceivedMessage> pullMessages(String projectId, String subscriptionId, int maxNumberOfMessages) throws Exception {
	SubscriberStubSettings subscriberStubSettings =
		SubscriberStubSettings.newBuilder()
			.setTransportChannelProvider(channelProvider)
			.setCredentialsProvider(NoCredentialsProvider.create())
			.build();
	try (SubscriberStub subscriber = GrpcSubscriberStub.create(subscriberStubSettings)) {
		String subscriptionName = ProjectSubscriptionName.format(projectId, subscriptionId);
		PullRequest pullRequest =
			PullRequest.newBuilder()
				.setMaxMessages(maxNumberOfMessages)
				.setReturnImmediately(false)
				.setSubscription(subscriptionName)
				.build();

		List<ReceivedMessage> receivedMessages = subscriber.pullCallable().call(pullRequest).getReceivedMessagesList();
		acknowledgeIds(subscriber, subscriptionName, receivedMessages);
		return receivedMessages;
	}
}
 

/**
 * Actually build the desired instance of the PubSubSourceBuilder.
 *
 * @return a brand new SourceFunction
 * @throws IOException              in case of a problem getting the credentials
 * @throws IllegalArgumentException in case required fields were not specified.
 */
public PubSubSource<OUT> build() throws IOException {
	if (credentials == null) {
		credentials = defaultCredentialsProviderBuilder().build().getCredentials();
	}

	if (pubSubSubscriberFactory == null) {
		pubSubSubscriberFactory = new DefaultPubSubSubscriberFactory(ProjectSubscriptionName.format(projectName, subscriptionName),
																	3,
																	Duration.ofSeconds(15),
																	100);
	}

	return new PubSubSource<>(deserializationSchema, pubSubSubscriberFactory, credentials, maxMessageToAcknowledge, new AcknowledgeOnCheckpointFactory());
}
 

public PubSubSubscriberFactoryForEmulator(String hostAndPort, String project, String subscription, int retries, Duration timeout, int maxMessagesPerPull) {
	this.hostAndPort = hostAndPort;
	this.retries = retries;
	this.timeout = timeout;
	this.maxMessagesPerPull = maxMessagesPerPull;
	this.projectSubscriptionName = ProjectSubscriptionName.format(project, subscription);
}
 

public List<ReceivedMessage> pullMessages(String projectId, String subscriptionId, int maxNumberOfMessages) throws Exception {
	SubscriberStubSettings subscriberStubSettings =
		SubscriberStubSettings.newBuilder()
			.setTransportChannelProvider(channelProvider)
			.setCredentialsProvider(NoCredentialsProvider.create())
			.build();
	try (SubscriberStub subscriber = GrpcSubscriberStub.create(subscriberStubSettings)) {
		// String projectId = "my-project-id";
		// String subscriptionId = "my-subscription-id";
		// int numOfMessages = 10;   // max number of messages to be pulled
		String subscriptionName = ProjectSubscriptionName.format(projectId, subscriptionId);
		PullRequest pullRequest =
			PullRequest.newBuilder()
				.setMaxMessages(maxNumberOfMessages)
				.setReturnImmediately(false) // return immediately if messages are not available
				.setSubscription(subscriptionName)
				.build();

		// use pullCallable().futureCall to asynchronously perform this operation
		PullResponse pullResponse = subscriber.pullCallable().call(pullRequest);
		List<String> ackIds = new ArrayList<>();
		for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) {
			// handle received message
			// ...
			ackIds.add(message.getAckId());
		}
		// acknowledge received messages
		AcknowledgeRequest acknowledgeRequest =
			AcknowledgeRequest.newBuilder()
				.setSubscription(subscriptionName)
				.addAllAckIds(ackIds)
				.build();
		// use acknowledgeCallable().futureCall to asynchronously perform this operation
		subscriber.acknowledgeCallable().call(acknowledgeRequest);
		return pullResponse.getReceivedMessagesList();
	}
}
 

private List<String> getMessagesFromSubscription(String subscriptionName) {
	String projectSubscriptionName = ProjectSubscriptionName.format(
			projectName, subscriptionName);

	PullRequest pullRequest = PullRequest.newBuilder()
			.setReturnImmediately(true)
			.setMaxMessages(10)
			.setSubscription(projectSubscriptionName)
			.build();

	PullResponse pullResponse = subscriptionAdminClient.getStub().pullCallable().call(pullRequest);
	return pullResponse.getReceivedMessagesList().stream()
			.map((message) -> message.getMessage().getData().toStringUtf8())
			.collect(Collectors.toList());
}
 

private void createSubscription(String subscriptionName, String topicName) {
	String url = UriComponentsBuilder.fromHttpUrl(this.appUrl + "/createSubscription")
			.queryParam("topicName", topicName)
			.queryParam("subscriptionName", subscriptionName)
			.toUriString();
	this.testRestTemplate.postForEntity(url, null, String.class);

	String projectSubscriptionName = ProjectSubscriptionName.format(projectName, subscriptionName);
	await().atMost(PUBSUB_CLIENT_TIMEOUT_SECONDS, TimeUnit.SECONDS).untilAsserted(
			() -> {
				List<String> subscriptions = getSubscriptionNamesFromProject();
				assertThat(subscriptions).contains(projectSubscriptionName);
			});
}
 

private void deleteSubscription(String subscriptionName) {
	String url = UriComponentsBuilder.fromHttpUrl(this.appUrl + "/deleteSubscription")
			.queryParam("subscription", subscriptionName)
			.toUriString();
	this.testRestTemplate.postForEntity(url, null, String.class);

	String projectSubscriptionName = ProjectSubscriptionName.format(projectName, subscriptionName);
	await().atMost(PUBSUB_CLIENT_TIMEOUT_SECONDS, TimeUnit.SECONDS).untilAsserted(
			() -> {
				List<String> subscriptions = getSubscriptionNamesFromProject();
				assertThat(subscriptions).doesNotContain(projectSubscriptionName);
			});
}
 

private static void deleteSubscriptions(String... testSubscriptions) {
	for (String testSubscription : testSubscriptions) {
		String testSubscriptionName = ProjectSubscriptionName.format(
				projectName, testSubscription);
		List<String> projectSubscriptions = getSubscriptionNamesFromProject();
		if (projectSubscriptions.contains(testSubscriptionName)) {
			subscriptionAdminClient.deleteSubscription(testSubscriptionName);
		}
	}
}
 

/**
 * Actually build the desired instance of the PubSubSourceBuilder.
 *
 * @return a brand new SourceFunction
 * @throws IOException              in case of a problem getting the credentials
 * @throws IllegalArgumentException in case required fields were not specified.
 */
public PubSubSource<OUT> build() throws IOException {
	if (credentials == null) {
		credentials = defaultCredentialsProviderBuilder().build().getCredentials();
	}

	if (pubSubSubscriberFactory == null) {
		pubSubSubscriberFactory = new DefaultPubSubSubscriberFactory(ProjectSubscriptionName.format(projectName, subscriptionName),
																	3,
																	Duration.ofSeconds(15),
																	100);
	}

	return new PubSubSource<>(deserializationSchema, pubSubSubscriberFactory, credentials, new AcknowledgeOnCheckpointFactory(), new GuavaFlinkConnectorRateLimiter(), messagePerSecondRateLimit);
}
 

public PubSubSubscriberFactoryForEmulator(String hostAndPort, String project, String subscription, int retries, Duration timeout, int maxMessagesPerPull) {
	this.hostAndPort = hostAndPort;
	this.retries = retries;
	this.timeout = timeout;
	this.maxMessagesPerPull = maxMessagesPerPull;
	this.projectSubscriptionName = ProjectSubscriptionName.format(project, subscription);
}
 

private String getLastProcessedTimestamp() {

    String timestamp = "";
    try {
      final SubscriberStubSettings subscriberStubSettings =
          SubscriberStubSettings.newBuilder()
              .setTransportChannelProvider(
                  SubscriberStubSettings.defaultGrpcTransportProviderBuilder()
                      .setMaxInboundMessageSize(20 << 20) // 20MB
                      .build())
              .build();

      try (SubscriberStub subscriber = GrpcSubscriberStub.create(subscriberStubSettings)) {
        final String subscriptionName = ProjectSubscriptionName.format(PROJECT_ID, tableName);
        final PullRequest pullRequest =
            PullRequest.newBuilder()
                .setMaxMessages(1)
                .setReturnImmediately(true)
                .setSubscription(subscriptionName)
                .build();

        final PullResponse pullResponse = subscriber.pullCallable().call(pullRequest);
        final DatumReader<GenericRecord> datumReader =
            new GenericDatumReader<GenericRecord>(avroSchema);

        for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) {
          final JsonDecoder decoder =
              DecoderFactory.get()
                  .jsonDecoder(avroSchema, message.getMessage().getData().newInput());

          final GenericRecord record = datumReader.read(null, decoder);
          timestamp = record.get("Timestamp").toString();

          log.debug("---------------- Got Timestamp: " + timestamp);
        }
      }
    } catch (IOException e) {
      log.error("Could not get last processed timestamp from pub / sub", e);

      // If we cannot find a previously processed timestamp, we will default
      // to the one present in the config file.
      return startingTimestamp;
    }

    return timestamp;
  }
 

/**
 * There is a default PubSubSubscriber factory that uses gRPC to pull in PubSub messages. This method can be used to tune this default factory.
       * Note this will not work in combination with a custom PubSubSubscriber factory.
 * @param maxMessagesPerPull the number of messages pulled per request. Default: 100
 * @param perRequestTimeout the timeout per request. Default: 15 seconds
 * @param retries the number of retries when requests fail
 * @return The current PubSubSourceBuilder instance
 */
public PubSubSourceBuilder<OUT> withPubSubSubscriberFactory(int maxMessagesPerPull, Duration perRequestTimeout, int retries) {
	this.pubSubSubscriberFactory = new DefaultPubSubSubscriberFactory(ProjectSubscriptionName.format(projectName, subscriptionName),
																	retries,
																	perRequestTimeout,
																	maxMessagesPerPull);
	return this;
}
 

/**
 * There is a default PubSubSubscriber factory that uses gRPC to pull in PubSub messages. This method can be used to tune this default factory.
       * Note this will not work in combination with a custom PubSubSubscriber factory.
 * @param maxMessagesPerPull the number of messages pulled per request. Default: 100
 * @param perRequestTimeout the timeout per request. Default: 15 seconds
 * @param retries the number of retries when requests fail
 * @return The current PubSubSourceBuilder instance
 */
public PubSubSourceBuilder<OUT> withPubSubSubscriberFactory(int maxMessagesPerPull, Duration perRequestTimeout, int retries) {
	this.pubSubSubscriberFactory = new DefaultPubSubSubscriberFactory(ProjectSubscriptionName.format(projectName, subscriptionName),
																	retries,
																	perRequestTimeout,
																	maxMessagesPerPull);
	return this;
}