com.google.pubsub.v1.ProjectSubscriptionName Java Examples

private List<String> receiveLines(int expectedMessageCount) throws Exception {
  List<String> received = new CopyOnWriteArrayList<>();
  ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(projectId,
      subscriptionId);

  MessageReceiver receiver = ((PubsubMessage message, AckReplyConsumer consumer) -> {
    try {
      String encoded = Json.asString(new org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage(
          message.getData().toByteArray(), message.getAttributesMap()));
      received.add(encoded);
    } catch (IOException e) {
      throw new UncheckedIOException(e);
    }
    consumer.ack();
  });
  Subscriber subscriber = Subscriber.newBuilder(subscriptionName, receiver).build();
  subscriber.startAsync();
  while (received.size() < expectedMessageCount) {
    Thread.sleep(100);
  }
  subscriber.stopAsync();

  return received;
}
 

/** Updates the managed Server when a Subscription is deleted. */
public final void deleteSubscription(String subscriptionName)
    throws ConfigurationNotFoundException {
  ProjectSubscriptionName projectSubscriptionName =
      ProjectSubscriptionName.parse(subscriptionName);
  ProjectName projectName = ProjectName.of(projectSubscriptionName.getProject());

  Set<com.google.pubsub.v1.Subscription> subscriptions =
      subscriptionsByProject.get(projectName.toString());
  getSubscriptionByName(subscriptionName)
      .map(subscriptions::remove)
      .orElseThrow(
          () ->
              new ConfigurationNotFoundException(
                  "Subscription " + projectSubscriptionName.toString() + " does not exist"));
  pubSubRepository.save(getPubSub());
}
 

/**
 * Create a {@link ProjectSubscriptionName} based on a subscription name within a project or the
 * fully-qualified subscription name. If the specified subscription is in the
 * {@code projects/<project_name>/subscriptions/<subscription_name>} format, then the {@code projectId} is
 * ignored}
 * @param subscription the subscription name in the project or the fully-qualified project name
 * @param projectId the project ID to use if the subscription is not a fully-qualified name
 * @return the Pub/Sub object representing the subscription name
 */
public static ProjectSubscriptionName toProjectSubscriptionName(String subscription, @Nullable String projectId) {
	Assert.notNull(subscription, "The subscription can't be null.");

	ProjectSubscriptionName projectSubscriptionName = null;

	if (ProjectSubscriptionName.isParsableFrom(subscription)) {
		// Fully-qualified subscription name in the "projects/<project_name>/subscriptions/<subscription_name>" format
		projectSubscriptionName = ProjectSubscriptionName.parse(subscription);
	}
	else {
		Assert.notNull(projectId, "The project ID can't be null when using canonical subscription name.");
		projectSubscriptionName = ProjectSubscriptionName.of(projectId, subscription);
	}

	return projectSubscriptionName;
}
 

private CPSSubscriberTask(StartRequest request, MetricsHandler metricsHandler, int workerCount) {
  this.metricsHandler = metricsHandler;
  ProjectSubscriptionName subscription =
      ProjectSubscriptionName.of(
          request.getProject(), request.getPubsubOptions().getSubscription());
  try {
    this.subscriber =
        Subscriber.newBuilder(subscription, this)
            .setParallelPullCount(workerCount)
            .setFlowControlSettings(
                FlowControlSettings.newBuilder()
                    .setMaxOutstandingElementCount(Long.MAX_VALUE)
                    .setMaxOutstandingRequestBytes(BYTES_PER_WORKER * workerCount)
                    .build())
            .build();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

public static int pubSub(String subId, long timeoutSeconds, String projectId)
    throws InterruptedException {
  // String subId = "my-occurrence-subscription";
  // long timeoutSeconds = 20;
  // String projectId = "my-project-id";
  Subscriber subscriber = null;
  MessageReceiverExample receiver = new MessageReceiverExample();

  try {
    // Subscribe to the requested Pub/Sub channel
    ProjectSubscriptionName subName = ProjectSubscriptionName.of(projectId, subId);
    subscriber = Subscriber.newBuilder(subName, receiver).build();
    subscriber.startAsync().awaitRunning();
    // Sleep to listen for messages
    TimeUnit.SECONDS.sleep(timeoutSeconds);
  } finally {
    // Stop listening to the channel
    if (subscriber != null) {
      subscriber.stopAsync();
    }
  }
  // Print and return the number of Pub/Sub messages received
  System.out.println(receiver.messageCount);
  return receiver.messageCount;
}
 

/**
 * Creates a KafkaConsumer that is manually assigned to all partitions of the test topic indicated
 * by the {@code subscription}.
 */
protected Consumer<String, ByteBuffer> getValidationConsumer(String topic, String subscription) {
  Consumer<String, ByteBuffer> consumer =
      kafkaClientFactory.createConsumer(
          ProjectSubscriptionName.of(PROJECT, subscription).toString());

  Set<TopicPartition> topicPartitions =
      consumer
          .listTopics()
          .entrySet()
          .stream()
          .filter(e -> e.getKey().equals(ProjectTopicName.of(PROJECT, topic).toString()))
          .flatMap(
              e -> e.getValue().stream().map(p -> new TopicPartition(p.topic(), p.partition())))
          .collect(Collectors.toSet());
  consumer.assign(topicPartitions);

  return consumer;
}
 

private void createSubscription(final PubSubConfig config) {
    final SubscriptionAdminClient subscriptionAdminClient = pubSubManager.subscriptionAdminClient(config);

    final ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(config.getProjectId(),
            config.getSubscription());

    try {
        subscriptionAdminClient.getSubscription(subscriptionName);
    } catch (final NotFoundException e) {
        final PushConfig pushConfig = PushConfig.newBuilder()
                .build();

        final ProjectTopicName topicName = ProjectTopicName.of(config.getProjectId(), config.getTopic());

        subscriptionAdminClient.createSubscription(subscriptionName, topicName, pushConfig, 0);
    }
}
 

public void createSubscription(String subscriptionProject, String subscription, String topicProject, String topic) throws IOException {
	ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.newBuilder()
		.setProject(subscriptionProject)
		.setSubscription(subscription)
		.build();

	deleteSubscription(subscriptionName);

	SubscriptionAdminClient adminClient = getSubscriptionAdminClient();

	ProjectTopicName topicName = ProjectTopicName.of(topicProject, topic);

	PushConfig pushConfig = PushConfig.getDefaultInstance();

	LOG.info("CreateSubscription {}", subscriptionName);
	getSubscriptionAdminClient().createSubscription(subscriptionName, topicName, pushConfig, 1);
}
 

public List<ReceivedMessage> pullMessages(String projectId, String subscriptionId, int maxNumberOfMessages) throws Exception {
	SubscriberStubSettings subscriberStubSettings =
		SubscriberStubSettings.newBuilder()
			.setTransportChannelProvider(channelProvider)
			.setCredentialsProvider(NoCredentialsProvider.create())
			.build();
	try (SubscriberStub subscriber = GrpcSubscriberStub.create(subscriberStubSettings)) {
		String subscriptionName = ProjectSubscriptionName.format(projectId, subscriptionId);
		PullRequest pullRequest =
			PullRequest.newBuilder()
				.setMaxMessages(maxNumberOfMessages)
				.setReturnImmediately(false)
				.setSubscription(subscriptionName)
				.build();

		List<ReceivedMessage> receivedMessages = subscriber.pullCallable().call(pullRequest).getReceivedMessagesList();
		acknowledgeIds(subscriber, subscriptionName, receivedMessages);
		return receivedMessages;
	}
}
 

public static Subscription createOccurrenceSubscription(String subId, String projectId) 
    throws IOException, StatusRuntimeException, InterruptedException {
  // This topic id will automatically receive messages when Occurrences are added or modified
  String topicId = "container-analysis-occurrences-v1";
  ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId);
  ProjectSubscriptionName subName = ProjectSubscriptionName.of(projectId, subId);

  SubscriptionAdminClient client = SubscriptionAdminClient.create();
  PushConfig config = PushConfig.getDefaultInstance();
  Subscription sub = client.createSubscription(subName, topicName, config, 0);
  return sub;
}
 

private String getSubscriptionName(ProcessContext context) {
    final String subscriptionName = context.getProperty(SUBSCRIPTION).evaluateAttributeExpressions().getValue();
    final String projectId = context.getProperty(PROJECT_ID).getValue();

    if (subscriptionName.contains("/")) {
        return ProjectSubscriptionName.parse(subscriptionName).toString();
    } else {
        return ProjectSubscriptionName.of(projectId, subscriptionName).toString();
    }

}
 

@Test
public void testToProjectSubscriptionName_canonical() {
	String project = "projectA";
	String subscription = "subscriptionA";
	String fqn = "projects/" + project + "/subscriptions/" + subscription;

	ProjectSubscriptionName parsedProjectSubscriptionName = PubSubSubscriptionUtils
			.toProjectSubscriptionName(subscription, project);

	assertThat(parsedProjectSubscriptionName).isEqualTo(ProjectSubscriptionName.of(project, subscription));
	assertThat(parsedProjectSubscriptionName.toString()).isEqualTo(fqn);
}
 

@Test
public void testToProjectSubscriptionName_fqn() {
	String project = "projectA";
	String subscription = "subscriptionA";
	String fqn = "projects/" + project + "/subscriptions/" + subscription;

	ProjectSubscriptionName parsedProjectSubscriptionName = PubSubSubscriptionUtils.toProjectSubscriptionName(fqn,
			project);

	assertThat(parsedProjectSubscriptionName).isEqualTo(ProjectSubscriptionName.of(project, subscription));
	assertThat(parsedProjectSubscriptionName.toString()).isEqualTo(fqn);
}
 

@Test
public void testToProjectSubscriptionName_fqn_no_project() {
	String project = "projectA";
	String subscription = "subscriptionA";
	String fqn = "projects/" + project + "/subscriptions/" + subscription;

	ProjectSubscriptionName parsedProjectSubscriptionName = PubSubSubscriptionUtils.toProjectSubscriptionName(fqn,
			null);

	assertThat(parsedProjectSubscriptionName).isEqualTo(ProjectSubscriptionName.of(project, subscription));
	assertThat(parsedProjectSubscriptionName.toString()).isEqualTo(fqn);
}
 

public void recv(PubsubRecv request, StreamObserver<EmptyResponse> responseObserver) {
  System.out.println("recv has been called");

  ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(
      "some-project", request.getSubName());
  
  Subscriber subscriber = null;
  try {
    InstantiatingExecutorProvider executorProvider =
      InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build();

    subscriber =
        Subscriber.newBuilder(subscriptionName, new SimpleReceiver())
            .setExecutorProvider(executorProvider)
            .build();
    subscriber.startAsync().awaitRunning();

    // Allow the subscriber to run indefinitely unless an unrecoverable error occurs.
    subscriber.awaitTerminated();
  } catch (IllegalStateException e) {
    System.out.println("Subscriber unexpectedly stopped: " + e);
  }

  EmptyResponse reply = EmptyResponse.newBuilder().build();
  responseObserver.onNext(reply);
  responseObserver.onCompleted();
}
 

public static void deleteSubscription(
    String projectId, String subscriptionId, String endpoint
) throws IOException {
    SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create(
        SubscriptionAdminSettings.newBuilder()
            .setTransportChannelProvider(channelProvider(endpoint))
            .setCredentialsProvider(credentialsProvider)
            .build()
    );
    ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(projectId, subscriptionId);
    subscriptionAdminClient.deleteSubscription(subscriptionName);
}
 

Connection(
    ConsumerSchema.Consumer consumer,
    ConsumerReporter reporter,
    AtomicLong errors,
    LongAdder consumed,
    String projectId,
    String topicId,
    String subscriptionId,
    int threads,
    long maxOutstandingElementCount,
    long maxOutstandingRequestBytes,
    int maxInboundMessageSize,
    long keepAlive
) {
    this.consumer = consumer;
    this.reporter = reporter;
    this.errors = errors;
    this.consumed = consumed;
    this.projectId = projectId;
    this.subscriptionId = subscriptionId;
    this.threads = threads;
    this.topicName = ProjectTopicName.of(projectId, topicId);
    this.subscriptionName = ProjectSubscriptionName.of(projectId, subscriptionId);
    this.credentialsProvider = SubscriptionAdminSettings
        .defaultCredentialsProviderBuilder()
        .build();
    this.channelProvider = SubscriptionAdminSettings
        .defaultGrpcTransportProviderBuilder()
        .setMaxInboundMessageSize(maxInboundMessageSize)
        .setKeepAliveTime(Duration.ofSeconds(keepAlive))
        .build();
    this.maxOutstandingElementCount = maxOutstandingElementCount;
    this.maxOutstandingRequestBytes = maxOutstandingRequestBytes;
}
 

public Subscriber subscribeToSubscription(String project, String subscription, MessageReceiver messageReceiver) {
	ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(project, subscription);
	Subscriber subscriber =
		Subscriber
			.newBuilder(subscriptionName, messageReceiver)
			.setChannelProvider(channelProvider)
			.setCredentialsProvider(NoCredentialsProvider.create())
			.build();
	subscriber.startAsync();
	return subscriber;
}
 

public void deleteSubscription(String subscriptionProject, String subscription) throws IOException {
	deleteSubscription(ProjectSubscriptionName
		.newBuilder()
		.setProject(subscriptionProject)
		.setSubscription(subscription)
		.build());
}
 

@AfterClass
public static void tearDownClass() {
  try {
    SubscriptionAdminClient subscriptionAdminClient = SubscriptionAdminClient.create();
    ProjectSubscriptionName subName = ProjectSubscriptionName.of(PROJECT_ID, subId);
    subscriptionAdminClient.deleteSubscription(subName);
    subscriptionAdminClient.shutdownNow();
  } catch (Exception e) {
    // these exceptions aren't relevant to the tests
    System.out.println("TearDownClass Error: " + e.toString());
  }
}
 

/**
 * Actually build the desired instance of the PubSubSourceBuilder.
 *
 * @return a brand new SourceFunction
 * @throws IOException              in case of a problem getting the credentials
 * @throws IllegalArgumentException in case required fields were not specified.
 */
public PubSubSource<OUT> build() throws IOException {
	if (credentials == null) {
		credentials = defaultCredentialsProviderBuilder().build().getCredentials();
	}

	if (pubSubSubscriberFactory == null) {
		pubSubSubscriberFactory = new DefaultPubSubSubscriberFactory(ProjectSubscriptionName.format(projectName, subscriptionName),
																	3,
																	Duration.ofSeconds(15),
																	100);
	}

	return new PubSubSource<>(deserializationSchema, pubSubSubscriberFactory, credentials, maxMessageToAcknowledge, new AcknowledgeOnCheckpointFactory());
}
 

public synchronized void start() {
  this.subscriber =
      Subscriber.newBuilder(
              ProjectSubscriptionName.of(project, subscriptionName), messageReceiver)
          .setCredentialsProvider(FixedCredentialsProvider.create(credentials))
          .build();

  subscriber.addListener(
      new GooglePubsubFailureHandler(this, formatSubscriptionName(project, subscriptionName)),
      MoreExecutors.directExecutor());
  subscriber.startAsync().awaitRunning();
  log.info(
      "Google Pubsub subscriber started for {}",
      formatSubscriptionName(project, subscriptionName));
}
 

/**
 * Actually build the desired instance of the PubSubSourceBuilder.
 *
 * @return a brand new SourceFunction
 * @throws IOException              in case of a problem getting the credentials
 * @throws IllegalArgumentException in case required fields were not specified.
 */
public PubSubSource<OUT> build() throws IOException {
	if (credentials == null) {
		credentials = defaultCredentialsProviderBuilder().build().getCredentials();
	}

	if (pubSubSubscriberFactory == null) {
		pubSubSubscriberFactory = new DefaultPubSubSubscriberFactory(ProjectSubscriptionName.format(projectName, subscriptionName),
																	3,
																	Duration.ofSeconds(15),
																	100);
	}

	return new PubSubSource<>(deserializationSchema, pubSubSubscriberFactory, credentials, new AcknowledgeOnCheckpointFactory(), new GuavaFlinkConnectorRateLimiter(), messagePerSecondRateLimit);
}
 

public void deleteSubscription(ProjectSubscriptionName subscriptionName) throws IOException {
	SubscriptionAdminClient adminClient = getSubscriptionAdminClient();
	try {
		adminClient.getSubscription(subscriptionName);
		// If it already exists we must first delete it.
		LOG.info("DeleteSubscription {}", subscriptionName);
		adminClient.deleteSubscription(subscriptionName);
	} catch (NotFoundException e) {
		// Doesn't exist. Good.
	}
}
 

public PubSubSubscriberFactoryForEmulator(String hostAndPort, String project, String subscription, int retries, Duration timeout, int maxMessagesPerPull) {
	this.hostAndPort = hostAndPort;
	this.retries = retries;
	this.timeout = timeout;
	this.maxMessagesPerPull = maxMessagesPerPull;
	this.projectSubscriptionName = ProjectSubscriptionName.format(project, subscription);
}
 

public void createSubscription(String subscriptionProject, String subscription, String topicProject, String topic) throws IOException {
	ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.newBuilder()
		.setProject(subscriptionProject)
		.setSubscription(subscription)
		.build();

	deleteSubscription(subscriptionName);

	ProjectTopicName topicName = ProjectTopicName.of(topicProject, topic);

	PushConfig pushConfig = PushConfig.getDefaultInstance();

	LOG.info("CreateSubscription {}", subscriptionName);
	getSubscriptionAdminClient().createSubscription(subscriptionName, topicName, pushConfig, 1).isInitialized();
}
 

private static void deleteSubscriptions(String... testSubscriptions) {
	for (String testSubscription : testSubscriptions) {
		String testSubscriptionName = ProjectSubscriptionName.format(
				projectName, testSubscription);
		List<String> projectSubscriptions = getSubscriptionNamesFromProject();
		if (projectSubscriptions.contains(testSubscriptionName)) {
			subscriptionAdminClient.deleteSubscription(testSubscriptionName);
		}
	}
}
 

public void deleteSubscription(String subscriptionProject, String subscription) throws IOException {
	deleteSubscription(ProjectSubscriptionName
		.newBuilder()
		.setProject(subscriptionProject)
		.setSubscription(subscription)
		.build());
}
 

public void deleteSubscription(ProjectSubscriptionName subscriptionName) throws IOException {
	SubscriptionAdminClient adminClient = getSubscriptionAdminClient();
	try {
		adminClient.getSubscription(subscriptionName);
		// If it already exists we must first delete it.
		LOG.info("DeleteSubscription {}", subscriptionName);
		adminClient.deleteSubscription(subscriptionName);
	} catch (NotFoundException e) {
		// Doesn't exist. Good.
	}
}
 

public List<ReceivedMessage> pullMessages(String projectId, String subscriptionId, int maxNumberOfMessages) throws Exception {
	SubscriberStubSettings subscriberStubSettings =
		SubscriberStubSettings.newBuilder()
			.setTransportChannelProvider(channelProvider)
			.setCredentialsProvider(NoCredentialsProvider.create())
			.build();
	try (SubscriberStub subscriber = GrpcSubscriberStub.create(subscriberStubSettings)) {
		// String projectId = "my-project-id";
		// String subscriptionId = "my-subscription-id";
		// int numOfMessages = 10;   // max number of messages to be pulled
		String subscriptionName = ProjectSubscriptionName.format(projectId, subscriptionId);
		PullRequest pullRequest =
			PullRequest.newBuilder()
				.setMaxMessages(maxNumberOfMessages)
				.setReturnImmediately(false) // return immediately if messages are not available
				.setSubscription(subscriptionName)
				.build();

		// use pullCallable().futureCall to asynchronously perform this operation
		PullResponse pullResponse = subscriber.pullCallable().call(pullRequest);
		List<String> ackIds = new ArrayList<>();
		for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) {
			// handle received message
			// ...
			ackIds.add(message.getAckId());
		}
		// acknowledge received messages
		AcknowledgeRequest acknowledgeRequest =
			AcknowledgeRequest.newBuilder()
				.setSubscription(subscriptionName)
				.addAllAckIds(ackIds)
				.build();
		// use acknowledgeCallable().futureCall to asynchronously perform this operation
		subscriber.acknowledgeCallable().call(acknowledgeRequest);
		return pullResponse.getReceivedMessagesList();
	}
}