com.google.pubsub.v1.Subscription Java Examples

@Override
public void listSubscriptions(
    ListSubscriptionsRequest request,
    StreamObserver<ListSubscriptionsResponse> responseObserver) {
  logger.atFine().log("Listing Subscriptions for %s", request);
  PaginationManager<Subscription> paginationManager =
      new PaginationManager<>(
          configurationManager.getSubscriptions(request.getProject()), Subscription::getName);
  ListSubscriptionsResponse response =
      ListSubscriptionsResponse.newBuilder()
          .addAllSubscriptions(
              paginationManager.paginate(request.getPageSize(), request.getPageToken()))
          .setNextPageToken(paginationManager.getNextToken(Subscription::getName))
          .build();
  responseObserver.onNext(response);
  responseObserver.onCompleted();
}
 

@Before
public void setup() {
	when(this.pubSubAdminMock.getSubscription(any())).thenReturn(null);
	doAnswer((invocation) ->
		Subscription.newBuilder()
				.setName("projects/test-project/subscriptions/" + invocation.getArgument(0))
				.setTopic("projects/test-project/topics/" + invocation.getArgument(1)).build()
	).when(this.pubSubAdminMock).createSubscription(any(), any());
	doAnswer((invocation) ->
			Topic.newBuilder().setName("projects/test-project/topics/" + invocation.getArgument(0)).build()
	).when(this.pubSubAdminMock).getTopic(any());
	when(this.properties.getExtension()).thenReturn(this.pubSubConsumerProperties);
	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(true);

	this.pubSubChannelProvisioner = new PubSubChannelProvisioner(this.pubSubAdminMock);
}
 

SubscriptionManager(
    Subscription subscription,
    KafkaClientFactory kafkaClientFactory,
    Clock clock,
    int consumerExecutors) {
  this.consumerExecutors = consumerExecutors;
  this.subscription = subscription;
  this.kafkaClientFactory = kafkaClientFactory;
  this.clock = clock;

  kafkaTopicName = subscription.getLabelsOrDefault(KAFKA_TOPIC, subscription.getTopic());
  committedOffsets = new HashMap<>();

  buffer = new ConcurrentLinkedQueue<>();
  outstandingMessages = new ConcurrentHashMap<>();
  nextConsumerIndex = new AtomicInteger();
  queueSizeBytes = new AtomicInteger();
  try {
    hostName = InetAddress.getLocalHost().getHostName();
  } catch (UnknownHostException e) {
    hostName = "<Unknown Host>";
  }

  kafkaConsumers = new ArrayList<>();
}
 

@Override
public List<SubscriptionPath> listSubscriptions(ProjectPath project, TopicPath topic)
    throws IOException {
  ListSubscriptionsRequest.Builder request =
      ListSubscriptionsRequest.newBuilder()
          .setProject(project.getPath())
          .setPageSize(LIST_BATCH_SIZE);
  ListSubscriptionsResponse response = subscriberStub().listSubscriptions(request.build());
  if (response.getSubscriptionsCount() == 0) {
    return ImmutableList.of();
  }
  List<SubscriptionPath> subscriptions = new ArrayList<>(response.getSubscriptionsCount());
  while (true) {
    for (Subscription subscription : response.getSubscriptionsList()) {
      if (subscription.getTopic().equals(topic.getPath())) {
        subscriptions.add(subscriptionPathFromPath(subscription.getName()));
      }
    }
    if (response.getNextPageToken().isEmpty()) {
      break;
    }
    request.setPageToken(response.getNextPageToken());
    response = subscriberStub().listSubscriptions(request.build());
  }
  return subscriptions;
}
 

void computeSubscriber(String subscription, ByteString messageData, Timestamp publishTime) {
  Optional<Subscription> subscriptionByName =
      configurationManager.getSubscriptionByName(subscription);
  if (subscriptionByName.isPresent()) {
    StatisticsInformation statisticsInformation =
        subscriberInformationByTopic.get(subscriptionByName.get().getTopic());
    if (statisticsInformation == null) {
      logger.atWarning().log(
          "Unable to record subscriber statistics error for Subscription %s (Topic %s not found)",
          subscription, subscriberInformationByTopic.get(subscriptionByName.get().getTopic()));
      return;
    }
    Instant publishTimeToInstant =
        Instant.ofEpochSecond(publishTime.getSeconds(), publishTime.getNanos());
    long subscriptionLatency = clock.millis() - publishTimeToInstant.toEpochMilli();
    statisticsInformation.compute(subscriptionLatency, messageData.toStringUtf8().length());
  } else {
    logger.atWarning().log(
        "Unable to record subscriber statistics error for Subscription %s", subscription);
  }
}
 

@Test
public void createSubscription() {
  Subscription request =
      Subscription.newBuilder()
          .setTopic(TestHelpers.PROJECT1_TOPIC1)
          .setName("projects/project-1/subscriptions/new-subscriptions")
          .build();
  Subscription builtRequest =
      request
          .toBuilder()
          .setAckDeadlineSeconds(10)
          .putLabels(KAFKA_TOPIC, "kafka-topic-1")
          .setPushConfig(PushConfig.getDefaultInstance())
          .build();
  SubscriptionManager fakeSubscriptionManager =
      new FakeSubscriptionManager(
          builtRequest, mock(SubscriptionManager.class), mockKafkaClientFactory);
  when(mockSubscriptionManagerFactory.create(builtRequest)).thenReturn(fakeSubscriptionManager);

  assertThat(blockingStub.createSubscription(request), Matchers.equalTo(builtRequest));
  assertThat(fakeSubscriptionManager.isRunning(), Matchers.is(true));
  verify(mockSubscriptionManagerFactory).create(builtRequest);
}
 

/**
 * Get the configuration of a Google Cloud Pub/Sub subscription.
 *
 * @param subscriptionName canonical subscription name, e.g., "subscriptionName", or the fully-qualified
 * subscription name in the {@code projects/<project_name>/subscriptions/<subscription_name>} format
 * @return subscription configuration or {@code null} if subscription doesn't exist
 */
public Subscription getSubscription(String subscriptionName) {
	Assert.hasText(subscriptionName, "No subscription name was specified");

	try {
		return this.subscriptionAdminClient.getSubscription(
				PubSubSubscriptionUtils.toProjectSubscriptionName(subscriptionName, this.projectId));
	}
	catch (ApiException aex) {
		if (aex.getStatusCode().getCode() == StatusCode.Code.NOT_FOUND) {
			return null;
		}

		throw aex;
	}
}
 

/**
 * Create a new subscription on Google Cloud Pub/Sub.
 *
 * @param subscriptionName canonical subscription name, e.g., "subscriptionName", or the fully-qualified
 * subscription name in the {@code projects/<project_name>/subscriptions/<subscription_name>} format
 * @param topicName canonical topic name, e.g., "topicName", or the fully-qualified topic name in the
 * {@code projects/<project_name>/topics/<topic_name>} format
 * @param ackDeadline deadline in seconds before a message is resent, must be between 10
 * and 600 seconds. If not provided, set to default of 10 seconds
 * @param pushEndpoint the URL of the service receiving the push messages. If not
 * provided, uses message pulling by default
 * @return the created subscription
 */
public Subscription createSubscription(String subscriptionName, String topicName,
		Integer ackDeadline, String pushEndpoint) {
	Assert.hasText(subscriptionName, "No subscription name was specified.");
	Assert.hasText(topicName, "No topic name was specified.");

	int finalAckDeadline = this.defaultAckDeadline;
	if (ackDeadline != null) {
		validateAckDeadline(ackDeadline);
		finalAckDeadline = ackDeadline;
	}

	PushConfig.Builder pushConfigBuilder = PushConfig.newBuilder();
	if (pushEndpoint != null) {
		pushConfigBuilder.setPushEndpoint(pushEndpoint);
	}

	return this.subscriptionAdminClient.createSubscription(
			PubSubSubscriptionUtils.toProjectSubscriptionName(subscriptionName, this.projectId),
			PubSubTopicUtils.toProjectTopicName(topicName, this.projectId),
			pushConfigBuilder.build(),
			finalAckDeadline);
}
 

@Test
public void getSubscription() {
  GetSubscriptionRequest request =
      GetSubscriptionRequest.newBuilder()
          .setSubscription(TestHelpers.PROJECT1_SUBSCRIPTION1)
          .build();
  Subscription response = blockingStub.getSubscription(request);
  assertThat(
      response,
      Matchers.equalTo(
          Subscription.newBuilder()
              .setName(TestHelpers.PROJECT1_SUBSCRIPTION1)
              .setTopic(TestHelpers.PROJECT1_TOPIC1)
              .putLabels(KAFKA_TOPIC, "kafka-topic-1")
              .setPushConfig(PushConfig.getDefaultInstance())
              .setAckDeadlineSeconds(10)
              .build()));
}
 

@Test
public void listSubscriptions() {
  ListSubscriptionsRequest request =
      ListSubscriptionsRequest.newBuilder().setProject(TestHelpers.PROJECT1).build();
  ListSubscriptionsResponse response = blockingStub.listSubscriptions(request);

  assertThat(
      response.getSubscriptionsList(),
      Matchers.contains(
          Subscription.newBuilder()
              .setName(TestHelpers.PROJECT1_SUBSCRIPTION1)
              .setTopic(TestHelpers.PROJECT1_TOPIC1)
              .putLabels(KAFKA_TOPIC, "kafka-topic-1")
              .setPushConfig(PushConfig.getDefaultInstance())
              .setAckDeadlineSeconds(10)
              .build(),
          Subscription.newBuilder()
              .setName(TestHelpers.PROJECT1_SUBSCRIPTION2)
              .setTopic(TestHelpers.PROJECT1_TOPIC1)
              .putLabels(KAFKA_TOPIC, "kafka-topic-1")
              .setPushConfig(PushConfig.getDefaultInstance())
              .setAckDeadlineSeconds(30)
              .build()));
}
 

@Override
public ConsumerDestination provisionConsumerDestination(String topicName, String group,
		ExtendedConsumerProperties<PubSubConsumerProperties> properties)
		throws ProvisioningException {

	Topic topic = makeSureTopicExists(topicName, properties.getExtension().isAutoCreateResources());

	String subscriptionName;
	Subscription subscription;
	if (StringUtils.hasText(group)) {
		// Use <topicName>.<group> as subscription name
		subscriptionName = topicName + "." + group;
		subscription = this.pubSubAdmin.getSubscription(subscriptionName);
	}
	else {
		// Generate anonymous random group since one wasn't provided
		subscriptionName = "anonymous." + topicName + "." + UUID.randomUUID().toString();
		subscription = this.pubSubAdmin.createSubscription(subscriptionName, topicName);
		this.anonymousGroupSubscriptionNames.add(subscriptionName);
	}

	// make sure subscription exists
	if (subscription == null) {
		if (properties.getExtension().isAutoCreateResources()) {
			this.pubSubAdmin.createSubscription(subscriptionName, topicName);
		}
		else {
			throw new ProvisioningException("Non-existing '" + subscriptionName + "' subscription.");
		}
	}
	else if (!subscription.getTopic().equals(topic.getName())) {
		throw new ProvisioningException(
				"Existing '" + subscriptionName + "' subscription is for a different topic '"
						+ subscription.getTopic() + "'.");
	}
	return new PubSubConsumerDestination(subscriptionName);
}
 

@Inject
SubscriberService(
    ConfigurationManager configurationManager,
    SubscriptionManagerFactory subscriptionManagerFactory,
    StatisticsManager statisticsManager) {
  this.configurationManager = configurationManager;
  this.statisticsManager = statisticsManager;
  this.subscriptionManagerFactory = subscriptionManagerFactory;

  subscriptions =
      configurationManager
          .getProjects()
          .stream()
          .flatMap(p -> configurationManager.getSubscriptions(p).stream())
          .collect(
              Collectors.toConcurrentMap(
                  Subscription::getName,
                  subscription -> {
                    SubscriptionManager sm = subscriptionManagerFactory.create(subscription);
                    logger.atFine().log(
                        "Starting Subscription Manager for %s", subscription.getName());
                    sm.startAsync().awaitRunning();
                    logger.atInfo().log(
                        "Started Subscription Manager for %s", subscription.getName());
                    return sm;
                  }));
}
 

@Test
public void testProvisionConsumerDestination_wrongTopicException() {
	this.expectedEx.expect(ProvisioningException.class);
	this.expectedEx.expectMessage("Existing 'topic_A.group_A' subscription is for a different topic 'topic_B'.");

	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(false);
	when(this.pubSubAdminMock.getSubscription("topic_A.group_A")).thenReturn(Subscription.newBuilder().setTopic("topic_B").build());

	PubSubConsumerDestination result = (PubSubConsumerDestination) this.pubSubChannelProvisioner
			.provisionConsumerDestination("topic_A", "group_A", this.properties);
}
 

@Bean
public PubSubAdmin pubSubAdmin() {
	PubSubAdmin pubSubAdminMock = Mockito.mock(PubSubAdmin.class);
	when(pubSubAdminMock.createSubscription(anyString(), anyString())).thenReturn(
			Subscription.getDefaultInstance());
	when(pubSubAdminMock.getSubscription(anyString())).thenReturn(
			Subscription.getDefaultInstance());
	when(pubSubAdminMock.getTopic(anyString())).thenReturn(
			Topic.getDefaultInstance());
	return pubSubAdminMock;
}
 

/**
 * Return every subscription in a project.
 * <p>If there are multiple pages, they will all be merged into the same result.
 * @return a list of subscriptions
 */
public List<Subscription> listSubscriptions() {
	SubscriptionAdminClient.ListSubscriptionsPagedResponse subscriptionsPage =
			this.subscriptionAdminClient.listSubscriptions(ProjectName.of(this.projectId));

	List<Subscription> subscriptions = new ArrayList<>();
	subscriptionsPage.iterateAll().forEach(subscriptions::add);
	return Collections.unmodifiableList(subscriptions);
}
 

@Override
public void createSubscription(
    TopicPath topic, SubscriptionPath subscription, int ackDeadlineSeconds) throws IOException {
  Subscription request =
      Subscription.newBuilder()
          .setTopic(topic.getPath())
          .setName(subscription.getPath())
          .setAckDeadlineSeconds(ackDeadlineSeconds)
          .build();
  subscriberStub().createSubscription(request); // ignore Subscription result.
}
 

@Override
public int ackDeadlineSeconds(SubscriptionPath subscription) throws IOException {
  GetSubscriptionRequest request =
      GetSubscriptionRequest.newBuilder().setSubscription(subscription.getPath()).build();
  Subscription response = subscriberStub().getSubscription(request);
  return response.getAckDeadlineSeconds();
}
 

public static Subscription createOccurrenceSubscription(String subId, String projectId) 
    throws IOException, StatusRuntimeException, InterruptedException {
  // This topic id will automatically receive messages when Occurrences are added or modified
  String topicId = "container-analysis-occurrences-v1";
  ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId);
  ProjectSubscriptionName subName = ProjectSubscriptionName.of(projectId, subId);

  SubscriptionAdminClient client = SubscriptionAdminClient.create();
  PushConfig config = PushConfig.getDefaultInstance();
  Subscription sub = client.createSubscription(subName, topicName, config, 0);
  return sub;
}
 

public static ProjectTopicName getPubSubTopicFromSubscription(
    String gcpProject, String subscription) throws IOException {
  setupSubscriptionClient();

  ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(gcpProject, subscription);

  LOG.info("Retrieving information about subscription {}", subscriptionName);
  Subscription subscriptionEntity = subscriptionAdminClient.getSubscription(subscriptionName);

  ProjectTopicName result = ProjectTopicName.parse(subscriptionEntity.getTopic());
  LOG.info("ProjectTopicName is {} with topic {}", result, result.getTopic());
  return result;
}
 

FakeSubscriptionManager(
    Subscription subscription,
    SubscriptionManager mockDelegate,
    KafkaClientFactory kafkaClientFactory) {
  super(subscription, kafkaClientFactory, Clock.systemUTC(), 4);
  this.mockDelegate = mockDelegate;
  this.subscription = subscription;
}
 

@Test
public void listSubscriptions_withPagination() {
  ListSubscriptionsRequest request =
      ListSubscriptionsRequest.newBuilder()
          .setProject(TestHelpers.PROJECT1)
          .setPageSize(1)
          .build();
  ListSubscriptionsResponse response = blockingStub.listSubscriptions(request);

  assertThat(
      response.getSubscriptionsList(),
      Matchers.contains(
          Subscription.newBuilder()
              .setName(TestHelpers.PROJECT1_SUBSCRIPTION1)
              .setTopic(TestHelpers.PROJECT1_TOPIC1)
              .putLabels(KAFKA_TOPIC, "kafka-topic-1")
              .setPushConfig(PushConfig.getDefaultInstance())
              .setAckDeadlineSeconds(10)
              .build()));
  assertThat(response.getNextPageToken(), Matchers.not(Matchers.isEmptyOrNullString()));

  request = request.toBuilder().setPageToken(response.getNextPageToken()).setPageSize(0).build();
  response = blockingStub.listSubscriptions(request);
  assertThat(
      response.getSubscriptionsList(),
      Matchers.contains(
          Subscription.newBuilder()
              .setName(TestHelpers.PROJECT1_SUBSCRIPTION2)
              .setTopic(TestHelpers.PROJECT1_TOPIC1)
              .putLabels(KAFKA_TOPIC, "kafka-topic-1")
              .setPushConfig(PushConfig.getDefaultInstance())
              .setAckDeadlineSeconds(30)
              .build()));
  assertThat(response.getNextPageToken(), Matchers.isEmptyOrNullString());
}
 

@Test
public void createSubscription_topicDoesNotExist() {
  expectedException.expect(StatusRuntimeException.class);
  expectedException.expectMessage(Status.NOT_FOUND.getCode().toString());

  Subscription request =
      Subscription.newBuilder()
          .setTopic("projects/project-1/topics/unknown-topic")
          .setName("projects/project-1/subscriptions/new-subscription")
          .build();
  blockingStub.createSubscription(request);
}
 

@Test
public void createSubscription_subscriptionExists() {
  expectedException.expect(StatusRuntimeException.class);
  expectedException.expectMessage(Status.ALREADY_EXISTS.getCode().toString());

  Subscription request =
      Subscription.newBuilder()
          .setTopic(TestHelpers.PROJECT1_TOPIC1)
          .setName(TestHelpers.PROJECT1_SUBSCRIPTION1)
          .build();
  blockingStub.createSubscription(request);
}
 

protected Subscription getSubscriptionByTopic(String topicName) {
  return configurationRepository
      .getSubscriptionsForTopic(topicName)
      .stream()
      .findFirst()
      .orElseThrow(() -> new IllegalArgumentException("Topic name not found"));
}
 

@Override
public void getSubscription(
    GetSubscriptionRequest request, StreamObserver<Subscription> responseObserver) {
  logger.atFine().log("Getting Subscription %s", request);
  Optional<Subscription> subscription =
      configurationManager.getSubscriptionByName(request.getSubscription());
  if (!subscription.isPresent()) {
    String message = request.getSubscription() + " is not a valid Subscription";
    logger.atWarning().log(message);
    responseObserver.onError(Status.NOT_FOUND.withDescription(message).asException());
  } else {
    responseObserver.onNext(subscription.get());
    responseObserver.onCompleted();
  }
}
 

public SubscriptionManager create(Subscription subscription) {
  return new SubscriptionManager(
      subscription,
      kafkaClientFactory,
      Clock.systemUTC(),
      configurationManager.getServer().getKafka().getConsumersPerSubscription());
}
 

private static List<String> getSubscriptionNamesFromProject() {
	ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions("projects/" + projectName);
	return StreamSupport.stream(response.iterateAll().spliterator(), false)
			.map(Subscription::getName)
			.collect(Collectors.toList());
}
 

@Override
public Subscription getSubscription() {
  return subscription;
}
 

@SuppressWarnings("unchecked")
private ConfigurationManager setupConfigurationRepository() {
  configurationManager =
      new ConfigurationManager(
          TestHelpers.SERVER_CONFIG,
          new FakePubSubRepository(
              PubSub.newBuilder()
                  .addProjects(
                      Project.newBuilder()
                          .setName("project-1")
                          .addTopics(
                              Topic.newBuilder()
                                  .setName("topic-1")
                                  .setKafkaTopic("kafka-topic-1")
                                  .addSubscriptions(
                                      com.google.cloud.partners.pubsub.kafka.config.Subscription
                                          .newBuilder()
                                          .setName("subscription-1")
                                          .setAckDeadlineSeconds(10)
                                          .build())
                                  .addSubscriptions(
                                      com.google.cloud.partners.pubsub.kafka.config.Subscription
                                          .newBuilder()
                                          .setName("subscription-2")
                                          .setAckDeadlineSeconds(30)
                                          .build())
                                  .build())
                          .build())
                  .addProjects(
                      Project.newBuilder()
                          .setName("project-2")
                          .addTopics(
                              Topic.newBuilder()
                                  .setName("topic-2")
                                  .setKafkaTopic("kafka-topic-2")
                                  .addSubscriptions(
                                      com.google.cloud.partners.pubsub.kafka.config.Subscription
                                          .newBuilder()
                                          .setName("subscription-3")
                                          .setAckDeadlineSeconds(45)
                                          .build())
                                  .build())
                          .build())
                  .build()));
  Subscription subscription1 =
      Subscription.newBuilder()
          .setName(TestHelpers.PROJECT1_SUBSCRIPTION1)
          .setTopic(TestHelpers.PROJECT1_TOPIC1)
          .putLabels(KAFKA_TOPIC, "kafka-topic-1")
          .setPushConfig(PushConfig.getDefaultInstance())
          .setAckDeadlineSeconds(10)
          .build();
  Subscription subscription2 =
      Subscription.newBuilder()
          .setName(TestHelpers.PROJECT1_SUBSCRIPTION2)
          .setTopic(TestHelpers.PROJECT1_TOPIC1)
          .putLabels(KAFKA_TOPIC, "kafka-topic-1")
          .setPushConfig(PushConfig.getDefaultInstance())
          .setAckDeadlineSeconds(30)
          .build();
  Subscription subscription3 =
      Subscription.newBuilder()
          .setName(TestHelpers.PROJECT2_SUBSCRIPTION3)
          .setTopic("projects/project-2/topics/topic-2")
          .putLabels(KAFKA_TOPIC, "kafka-topic-2")
          .setPushConfig(PushConfig.getDefaultInstance())
          .setAckDeadlineSeconds(45)
          .build();

  fakeSubscriptionManager1 =
      spy(
          new FakeSubscriptionManager(
              subscription1, mockSubscriptionManager1, mockKafkaClientFactory));
  fakeSubscriptionManager2 =
      spy(
          new FakeSubscriptionManager(
              subscription2, mockSubscriptionManager2, mockKafkaClientFactory));
  fakeSubscriptionManager3 =
      spy(
          new FakeSubscriptionManager(
              subscription3, mockSubscriptionManager3, mockKafkaClientFactory));

  when(mockSubscriptionManagerFactory.create(subscription1)).thenReturn(fakeSubscriptionManager1);
  when(mockSubscriptionManagerFactory.create(subscription2)).thenReturn(fakeSubscriptionManager2);
  when(mockSubscriptionManagerFactory.create(subscription3)).thenReturn(fakeSubscriptionManager3);

  return configurationManager;
}
 

void addSubscriberInformation(Subscription subscription) {
  subscriberInformationByTopic.putIfAbsent(subscription.getTopic(), new StatisticsInformation());
}