com.google.cloud.pubsub.v1.Publisher Java Examples

private void publish(String topic) throws IOException, InterruptedException {
  System.out.println("Publishing messages to " + topic + "...Press Ctrl-C to exit");
  Publisher publisher = Publisher.newBuilder(topic)
      .setCredentialsProvider(new NoCredentialsProvider())
      .setChannelProvider(getChannelProvider())
      .build();
  int messageNo = 1;
  while (true) {
    String message = "Message #" + messageNo++;
    ApiFuture<String> publishFuture = publisher.publish(PubsubMessage.newBuilder()
        .setData(ByteString.copyFromUtf8(message))
        .build());
    ApiFutures.addCallback(publishFuture, new ApiFutureCallback<String>() {
      @Override
      public void onFailure(Throwable throwable) {
        System.err.println("Error publishing " + message);
      }

      @Override
      public void onSuccess(String messageId) {
        System.out.println("Published " + message + " with message-id " + messageId);
      }
    });
    Thread.sleep(SLEEP_1S);
  }
}
 

private void publishLines(List<String> lines) throws Exception {
  Publisher publisher = Publisher.newBuilder(topicName).build();
  lines.forEach(line -> {
    try {
      org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage msg = Json.readPubsubMessage(line);
      Map<String, String> attributes = Optional.ofNullable(msg.getAttributeMap())
          .orElse(ImmutableMap.of());
      com.google.pubsub.v1.PubsubMessage outgoing = com.google.pubsub.v1.PubsubMessage
          .newBuilder().putAllAttributes(attributes)
          .setData(ByteString.copyFrom(msg.getPayload())).build();
      publisher.publish(outgoing);
    } catch (IOException e) {
      throw new UncheckedIOException(e);
    }
  });
  publisher.shutdown();
}
 

@Override
public void open(Configuration configuration) throws Exception {
	serializationSchema.open(() -> getRuntimeContext().getMetricGroup().addGroup("user"));

	Publisher.Builder builder = Publisher
		.newBuilder(ProjectTopicName.of(projectName, topicName))
		.setCredentialsProvider(FixedCredentialsProvider.create(credentials));

	if (hostAndPortForEmulator != null) {
		managedChannel = ManagedChannelBuilder
			.forTarget(hostAndPortForEmulator)
			.usePlaintext(true) // This is 'Ok' because this is ONLY used for testing.
			.build();
		channel = GrpcTransportChannel.newBuilder().setManagedChannel(managedChannel).build();
		builder.setChannelProvider(FixedTransportChannelProvider.create(channel))
				.setCredentialsProvider(NoCredentialsProvider.create());
	}

	publisher = builder.build();
	isRunning = true;
}
 

@Test
public void testPull() throws Exception {
	Publisher publisher = pubsubHelper.createPublisher(PROJECT_NAME, TOPIC_NAME);
	publisher
		.publish(PubsubMessage
			.newBuilder()
			.setData(ByteString.copyFromUtf8("Hello World PULL"))
			.build())
		.get();

	List<ReceivedMessage> receivedMessages = pubsubHelper.pullMessages(PROJECT_NAME, SUBSCRIPTION_NAME, 1);

	assertEquals(1, receivedMessages.size());
	assertEquals("Hello World PULL", receivedMessages.get(0).getMessage().getData().toStringUtf8());

	publisher.shutdown();
}
 

protected void publish(
    Publisher publisher,
    PubsubMessage message,
    java.util.function.Consumer<Throwable> onFailure,
    java.util.function.Consumer<String> onSuccess) {
  ApiFutures.addCallback(
      publisher.publish(message),
      new ApiFutureCallback<String>() {
        @Override
        public void onFailure(Throwable throwable) {
          onFailure.accept(throwable);
        }

        @Override
        public void onSuccess(String result) {
          onSuccess.accept(result);
        }
      });
}
 

@Test
public void servletPublishesPayloadMessage() throws Exception {
  assertNotNull(System.getenv("PUBSUB_TOPIC"));
  HttpServletRequest request = mock(HttpServletRequest.class);
  when(request.getParameter("payload")).thenReturn("test-message");

  HttpServletResponse response = mock(HttpServletResponse.class);
  Publisher publisher = mock(Publisher.class);
  PubsubMessage message = PubsubMessage.newBuilder()
      .setData(ByteString.copyFromUtf8("test-message")).build();
  when(publisher.publish(eq(message))).thenReturn(SettableApiFuture.create());
  PubSubPublish pubSubPublish = new PubSubPublish(publisher);
  // verify content of published test message
  pubSubPublish.doPost(request, response);
  verify(publisher, times(1)).publish(eq(message));
}
 

@Override
public void doPost(HttpServletRequest req, HttpServletResponse resp)
    throws IOException, ServletException {
  Publisher publisher = this.publisher;
  try {
    String topicId = System.getenv("PUBSUB_TOPIC");
    // create a publisher on the topic
    if (publisher == null) {
      publisher = Publisher.newBuilder(
          ProjectTopicName.of(ServiceOptions.getDefaultProjectId(), topicId))
          .build();
    }
    // construct a pubsub message from the payload
    final String payload = req.getParameter("payload");
    PubsubMessage pubsubMessage =
        PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(payload)).build();

    publisher.publish(pubsubMessage);
    // redirect to home page
    resp.sendRedirect("/");
  } catch (Exception e) {
    resp.sendError(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
  }
}
 

CPSPublisherTask(StartRequest request, MetricsHandler metricsHandler, int workerCount) {
  super(request, metricsHandler, workerCount);
  log.warn("constructing CPS publisher");
  this.payload = getPayload();
  try {
    this.publisher =
        Publisher.newBuilder(ProjectTopicName.of(request.getProject(), request.getTopic()))
            .setBatchingSettings(
                BatchingSettings.newBuilder()
                    .setElementCountThreshold((long) request.getPublisherOptions().getBatchSize())
                    .setRequestByteThreshold(9500000L)
                    .setDelayThreshold(
                        Duration.ofMillis(
                            Durations.toMillis(request.getPublisherOptions().getBatchDuration())))
                    .setIsEnabled(true)
                    .build())
            .build();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

@Test
public void testMessageBatchSentToPublisher() {
    final Publisher publisher = mockPublisher.orElseThrow(IllegalStateException::new);

    // Process a bunch of messages.
    final DivolteSchema schema = new DivolteSchema(MINIMAL_SCHEMA, Optional.empty());
    final GoogleCloudPubSubFlusher flusher = new GoogleCloudPubSubFlusher(publisher, schema);
    final Queue<Item<AvroRecordBuffer>> items =
        Stream.generate(this::generateMessage)
              .limit(10)
              .map(this::itemFromAvroRecordBuffer)
              .collect(Collectors.toCollection(() -> new ArrayBlockingQueue<>(10)));
    flusher.process(items);

    // Check the messages were all forwarded to the publisher.
    verify(publisher, times(10)).publish(any(PubsubMessage.class));
    verifyNoMoreInteractions(publisher);
}
 

private Publisher configurePubSub() {
  if (publishToPubSub) {
    ProjectTopicName topicName =
        ProjectTopicName.of(PROJECT_ID, tableName); // Topic name will always be the Table Name
    try {
      Publisher publisher = Publisher.newBuilder(topicName).build();
      return publisher;
    } catch (IOException e) {
      log.error("Was not able to create a publisher for topic: " + topicName, e);

      stop();
      System.exit(1);
    }
  }

  // If configured to publishToPubSub, this function will return a publisher or throw
  return null;
}
 

@Test
public void testMessagesAreRetriedOnRetriableFailure() throws IOException {
    // Simulate a failure on the first send that indicates a retry should succeed.
    final Publisher publisher = mockPublisher.orElseThrow(IllegalStateException::new);
    when(publisher.publish(any(PubsubMessage.class)))
        .thenReturn(failedFuture(new ApiException("simulated transient failure",
                                                  new IOException(),
                                                  GrpcStatusCode.of(Status.Code.INTERNAL),
                                                  true)))
        .thenAnswer(invocationOnMock -> completedFuture(String.valueOf(messageIdCounter++)));

    // Here we send the message.
    processSingleMessage();

    // Now we check the invocations…
    verify(publisher, times(2)).publish(any(PubsubMessage.class));
    verifyNoMoreInteractions(publisher);
}
 

@Test
public void testMessagesAreAbandonedOnNonRetriableFailure() throws IOException {
    // Simulate a failure on send that indicates a retry isn't allowed.
    final Publisher publisher = mockPublisher.orElseThrow(IllegalStateException::new);
    when(publisher.publish(any(PubsubMessage.class)))
        .thenReturn(failedFuture(new ApiException("simulated permanent failure",
                                                  new IOException(),
                                                  GrpcStatusCode.of(Status.Code.NOT_FOUND),
                                                  false)));
    // Here we send the message.
    processSingleMessage();

    // Now we check the invocations…
    verify(publisher).publish(any(PubsubMessage.class));
    verifyNoMoreInteractions(publisher);
}
 

@Test
public void testPull() throws Exception {
	Publisher publisher = pubsubHelper.createPublisher(PROJECT_NAME, TOPIC_NAME);
	publisher
		.publish(PubsubMessage
			.newBuilder()
			.setData(ByteString.copyFromUtf8("Hello World PULL"))
			.build())
		.get();

	List<ReceivedMessage> receivedMessages = pubsubHelper.pullMessages(PROJECT_NAME, SUBSCRIPTION_NAME, 10);
	assertEquals(1, receivedMessages.size());
	assertEquals("Hello World PULL", receivedMessages.get(0).getMessage().getData().toStringUtf8());

	publisher.shutdown();
}
 

@Override
public void open(Configuration configuration) throws Exception {
	Publisher.Builder builder = Publisher
		.newBuilder(ProjectTopicName.of(projectName, topicName))
		.setCredentialsProvider(FixedCredentialsProvider.create(credentials));

	if (hostAndPortForEmulator != null) {
		managedChannel = ManagedChannelBuilder
			.forTarget(hostAndPortForEmulator)
			.usePlaintext(true) // This is 'Ok' because this is ONLY used for testing.
			.build();
		channel = GrpcTransportChannel.newBuilder().setManagedChannel(managedChannel).build();
		builder.setChannelProvider(FixedTransportChannelProvider.create(channel))
				.setCredentialsProvider(NoCredentialsProvider.create());
	}

	publisher = builder.build();
	isRunning = true;
}
 

@Override
public void doPost(HttpServletRequest req, HttpServletResponse resp)
    throws ServletException, IOException {
  String text = req.getParameter("data");
  String sourceLang = req.getParameter("sourceLang");
  String targetLang = req.getParameter("targetLang");

  Enumeration<String> paramNames = req.getParameterNames();
  while (paramNames.hasMoreElements()) {
    String paramName = paramNames.nextElement();
    logger.warning("Param name: " + paramName + " = " + req.getParameter(paramName));
  }

  Publisher publisher = (Publisher) getServletContext().getAttribute("publisher");

  PubsubMessage pubsubMessage =
      PubsubMessage.newBuilder()
          .setData(ByteString.copyFromUtf8(text))
          .putAttributes("sourceLang", sourceLang)
          .putAttributes("targetLang", targetLang)
          .build();

  try {
    publisher.publish(pubsubMessage).get();
  } catch (InterruptedException | ExecutionException e) {
    throw new ServletException("Exception publishing message to topic.", e);
  }

  resp.sendRedirect("/");
}
 

@Before
public void setup() {
  publisher = mock(Publisher.class, RETURNS_DEEP_STUBS);
  task = new CloudPubSubSinkTask(publisher);
  props = new HashMap<>();
  props.put(ConnectorUtils.CPS_TOPIC_CONFIG, CPS_TOPIC);
  props.put(ConnectorUtils.CPS_PROJECT_CONFIG, CPS_PROJECT);
  props.put(CloudPubSubSinkConnector.MAX_BUFFER_SIZE_CONFIG, CPS_MIN_BATCH_SIZE2);
}
 

public static Publisher publisher(
    String projectId, String topicId, String endpoint
) throws IOException {
    ProjectTopicName topicName = ProjectTopicName.of(projectId, topicId);

    return Publisher.newBuilder(topicName)
        .setChannelProvider(channelProvider(endpoint))
        .setCredentialsProvider(credentialsProvider)
        .build();
}
 

@Override
public void doPost(HttpServletRequest req, HttpServletResponse resp)
    throws IOException, ServletException {
  Publisher publisher = this.publisher;
  try {
    String topicId = System.getenv("PUBSUB_TOPIC");
    // create a publisher on the topic
    if (publisher == null) {
      ProjectTopicName topicName =
          ProjectTopicName.newBuilder()
              .setProject(ServiceOptions.getDefaultProjectId())
              .setTopic(topicId)
              .build();
      publisher = Publisher.newBuilder(topicName).build();
    }
    // construct a pubsub message from the payload
    final String payload = req.getParameter("payload");
    PubsubMessage pubsubMessage =
        PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(payload)).build();

    publisher.publish(pubsubMessage);
    // redirect to home page
    resp.sendRedirect("/");
  } catch (Exception e) {
    resp.sendError(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage());
  }
}
 

@Override
public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
  Publisher publisher = this.publisher;
  // construct a pubsub message from the payload
  final String payload = req.getParameter("payload");
  Message message = new Message(null);
  message.setData(payload);
  PubsubMessage pubsubMessage =
      PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(payload))
          .putAttributes("sourceLang", req.getParameter("sourceLang"))
          .putAttributes("targetLang", req.getParameter("targetLang"))
          .build();
  String topicId = System.getenv("PUBSUB_TOPIC");
  // create a publisher on the topic
  if (publisher == null) {
    this.publisher = publisher = Publisher.newBuilder(
        ProjectTopicName.newBuilder()
            .setProject(ServiceOptions.getDefaultProjectId())
            .setTopic(topicId)
            .build())
        .build();
  }

  publisher.publish(pubsubMessage);
  // redirect to home page
  resp.sendRedirect("/");
}
 

@Override
public void contextInitialized(ServletContextEvent event) {
  String firestoreProjectId = System.getenv("FIRESTORE_CLOUD_PROJECT");
  Firestore firestore = (Firestore) event.getServletContext().getAttribute("firestore");
  if (firestore == null) {
    firestore =
        FirestoreOptions.getDefaultInstance().toBuilder()
            .setProjectId(firestoreProjectId)
            .build()
            .getService();
    event.getServletContext().setAttribute("firestore", firestore);
  }

  Translate translate = (Translate) event.getServletContext().getAttribute("translate");
  if (translate == null) {
    translate = TranslateOptions.getDefaultInstance().getService();
    event.getServletContext().setAttribute("translate", translate);
  }

  Publisher publisher = (Publisher) event.getServletContext().getAttribute("publisher");
  if (publisher == null) {
    try {
      String topicId = System.getenv("PUBSUB_TOPIC");
      publisher =
          Publisher.newBuilder(
                  ProjectTopicName.newBuilder()
                      .setProject(firestoreProjectId)
                      .setTopic(topicId)
                      .build())
              .build();
      event.getServletContext().setAttribute("publisher", publisher);
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
 

@Override
public void service(HttpRequest request, HttpResponse response) throws IOException {
  Optional<String> maybeTopicName = request.getFirstQueryParameter("topic");
  Optional<String> maybeMessage = request.getFirstQueryParameter("message");

  BufferedWriter responseWriter = response.getWriter();

  if (maybeTopicName.isEmpty() || maybeMessage.isEmpty()) {
    response.setStatusCode(HttpURLConnection.HTTP_BAD_REQUEST);

    responseWriter.write("Missing 'topic' and/or 'subscription' parameter(s).");
    return;
  }

  String topicName = maybeTopicName.get();
  logger.info("Publishing message to topic: " + topicName);

  // Create the PubsubMessage object
  // (This is different than the PubSubMessage POJO used in Pub/Sub-triggered functions)
  ByteString byteStr = ByteString.copyFrom(maybeMessage.get(), StandardCharsets.UTF_8);
  PubsubMessage pubsubApiMessage = PubsubMessage.newBuilder().setData(byteStr).build();

  Publisher publisher = Publisher.newBuilder(
      ProjectTopicName.of(PROJECT_ID, topicName)).build();

  // Attempt to publish the message
  String responseMessage;
  try {
    publisher.publish(pubsubApiMessage).get();
    responseMessage = "Message published.";
  } catch (InterruptedException | ExecutionException e) {
    logger.log(Level.SEVERE, "Error publishing Pub/Sub message: " + e.getMessage(), e);
    responseMessage = "Error publishing Pub/Sub message; see logs for more info.";
  }

  responseWriter.write(responseMessage);
}
 

public static GooglePubsubPublisher buildPublisher(
    GooglePubsubPublisherConfig config, ObjectMapper mapper) {
  GooglePubsubPublisher publisher = new GooglePubsubPublisher();
  publisher.setName(config.getName());
  ProjectTopicName fullName = ProjectTopicName.of(config.getProject(), config.getTopicName());
  publisher.setTopicName(config.getTopicName());
  publisher.setFullTopicName(fullName.toString());
  publisher.setContent(config.getContent());
  publisher.setMapper(mapper);

  BatchingSettings batchingSettings =
      BatchingSettings.newBuilder()
          .setElementCountThreshold(config.getBatchCountThreshold())
          .setDelayThreshold(Duration.ofMillis(config.getDelayMillisecondsThreshold()))
          .build();

  try {
    Publisher p =
        Publisher.newBuilder(fullName)
            .setCredentialsProvider(new GooglePubsubCredentialsProvider(config.getJsonPath()))
            .setBatchingSettings(batchingSettings)
            .build();
    publisher.setPublisher(p);
  } catch (IOException ioe) {
    log.error("Could not create Google Pubsub Publishers", ioe);
  }

  return publisher;
}
 

public Publisher createPublisher(String project, String topic) throws IOException {
	return Publisher
		.newBuilder(ProjectTopicName.of(project, topic))
		.setChannelProvider(channelProvider)
		.setCredentialsProvider(NoCredentialsProvider.create())
		.build();
}
 

@Test
public void testPub() throws Exception {
	List<PubsubMessage> receivedMessages = new ArrayList<>();
	Subscriber subscriber = pubsubHelper.
		subscribeToSubscription(
			PROJECT_NAME,
			SUBSCRIPTION_NAME,
			(message, consumer) -> {
				receivedMessages.add(message);
				consumer.ack();
			}
		);
	subscriber.awaitRunning(5, MINUTES);

	Publisher publisher = pubsubHelper.createPublisher(PROJECT_NAME, TOPIC_NAME);
	publisher
		.publish(PubsubMessage
			.newBuilder()
			.setData(ByteString.copyFromUtf8("Hello World"))
			.build())
		.get();

	LOG.info("Waiting a while to receive the message...");

	waitUntil(() -> receivedMessages.size() > 0);

	assertEquals(1, receivedMessages.size());
	assertEquals("Hello World", receivedMessages.get(0).getData().toStringUtf8());

	LOG.info("Received message. Shutting down ...");

	subscriber.stopAsync().awaitTerminated(5, MINUTES);
	publisher.shutdown();
}
 

public static void main(String... args) throws IOException {
  // [START pubsub_use_emulator]
  String hostport = System.getenv("PUBSUB_EMULATOR_HOST");
  ManagedChannel channel = ManagedChannelBuilder.forTarget(hostport).usePlaintext().build();
  try {
    TransportChannelProvider channelProvider =
        FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel));
    CredentialsProvider credentialsProvider = NoCredentialsProvider.create();

    // Set the channel and credentials provider when creating a `TopicAdminClient`.
    // Similarly for SubscriptionAdminClient
    TopicAdminClient topicClient =
        TopicAdminClient.create(
            TopicAdminSettings.newBuilder()
                .setTransportChannelProvider(channelProvider)
                .setCredentialsProvider(credentialsProvider)
                .build());

    TopicName topicName = TopicName.of("my-project-id", "my-topic-id");
    // Set the channel and credentials provider when creating a `Publisher`.
    // Similarly for Subscriber
    Publisher publisher =
        Publisher.newBuilder(topicName)
            .setChannelProvider(channelProvider)
            .setCredentialsProvider(credentialsProvider)
            .build();
  } finally {
    channel.shutdown();
  }
  // [END pubsub_use_emulator]
}
 

public GoogleCloudPubSubFlushingPool(final String name,
                                     final int numThreads,
                                     final int maxWriteQueue,
                                     final Publisher publisher,
                                     final Optional<ManagedChannel> channel,
                                     final DivolteSchema schema) {
    super(numThreads,
          maxWriteQueue,
          String.format("Google Cloud Pub/Sub Flusher [%s]", Objects.requireNonNull(name)),
          () -> new GoogleCloudPubSubFlusher(publisher, schema));
    this.publisher = Objects.requireNonNull(publisher);
    this.channel = Objects.requireNonNull(channel);
}
 

@Before
public void resetMessages() {
    final long now = System.currentTimeMillis();
    partyId = Optional.of(DivolteIdentifier.generate(now));
    sessionId = Optional.of(DivolteIdentifier.generate(now));

    generatedEventCounter = 0;
    messageIdCounter = 0;

    final Publisher mockPublisher = mock(Publisher.class);
    when(mockPublisher.publish(any(PubsubMessage.class)))
        .thenAnswer(invocationOnMock -> completedFuture(String.valueOf(messageIdCounter++)));
    this.mockPublisher = Optional.of(mockPublisher);
}
 

private void processSingleMessage(final Optional<Integer> confluentId) {
    final Publisher publisher = mockPublisher.orElseThrow(IllegalStateException::new);

    // Process a single message.
    final DivolteSchema schema = new DivolteSchema(MINIMAL_SCHEMA, confluentId);
    final GoogleCloudPubSubFlusher flusher = new GoogleCloudPubSubFlusher(publisher, schema);
    if (ItemProcessor.ProcessingDirective.PAUSE == flusher.process(itemFromAvroRecordBuffer(generateMessage()))) {
        flusher.heartbeat();
    }
}
 

@Test
public void testSingleMessageSentToPublisher() {
    // Process a single message.
    processSingleMessage();

    // Check it was forwarded to the publisher.
    final Publisher publisher = mockPublisher.orElseThrow(IllegalStateException::new);
    verify(publisher).publish(any(PubsubMessage.class));
    verifyNoMoreInteractions(publisher);
}
 

private Publisher.Builder getPublisherBuilder(ProcessContext context) {
    final Long batchSize = context.getProperty(BATCH_SIZE).asLong();

    return Publisher.newBuilder(getTopicName(context))
            .setCredentialsProvider(FixedCredentialsProvider.create(getGoogleCredentials(context)))
            .setBatchingSettings(BatchingSettings.newBuilder()
            .setElementCountThreshold(batchSize)
            .setIsEnabled(true)
            .build());
}