org.apache.samza.serializers.Serde Java Examples

/**
 * Internal helper for {@link MessageStreamImpl} to add an intermediate {@link MessageStream} to the graph.
 * An intermediate {@link MessageStream} is both an output and an input stream.
 *
 * @param streamId the id of the stream to be created.
 * @param serde the {@link Serde} to use for the message in the intermediate stream. If null, the default serde
 *              is used.
 * @param isBroadcast whether the stream is a broadcast stream.
 * @param <M> the type of messages in the intermediate {@link MessageStream}
 * @return  the intermediate {@link MessageStreamImpl}
 */
@VisibleForTesting
public <M> IntermediateMessageStreamImpl<M> getIntermediateStream(String streamId, Serde<M> serde, boolean isBroadcast) {
  Preconditions.checkNotNull(serde, "serde must not be null for intermediate stream: " + streamId);
  Preconditions.checkState(!inputOperators.containsKey(streamId) && !outputStreams.containsKey(streamId),
      "getIntermediateStream must not be called multiple times with the same streamId: " + streamId);

  if (isBroadcast) {
    intermediateBroadcastStreamIds.add(streamId);
  }

  boolean isKeyed = serde instanceof KVSerde;
  KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);

  InputTransformer transformer = (InputTransformer) getDefaultSystemDescriptor()
      .flatMap(SystemDescriptor::getTransformer).orElse(null);

  InputOperatorSpec inputOperatorSpec =
      OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(),
          transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
  inputOperators.put(streamId, inputOperatorSpec);
  outputStreams.put(streamId, new OutputStreamImpl(streamId, kvSerdes.getKey(), kvSerdes.getValue(), isKeyed));
  return new IntermediateMessageStreamImpl<>(this, inputOperators.get(streamId), outputStreams.get(streamId));
}
 

@Test
public void testSmallMessagePutWithSerdeAndDropLargeMessageDisabled() {
  LargeMessageSafeStore largeMessageSafeKeyValueStore = new LargeMessageSafeStore(store, storeName, false, maxMessageSize);

  Serde<Long> longSerde = new LongSerde();
  long longObj = 1000L;
  byte[] key = longSerde.toBytes(longObj);

  JsonSerdeV2<Map<String, Object>> jsonSerde = new JsonSerdeV2<>();
  Map<String, Object> obj = new HashMap<>();
  obj.put("jack", "jill");
  obj.put("john", 2);
  byte[] smallMessage = jsonSerde.toBytes(obj);

  List<Entry<byte[], byte[]>> entries = new ArrayList<>();
  entries.add(new Entry<>(key, smallMessage));

  largeMessageSafeKeyValueStore.putAll(entries);
  Mockito.verify(store).putAll(Matchers.eq(entries));
  Mockito.verify(store, Mockito.never()).put(Matchers.any(byte[].class), Matchers.any(byte[].class));
}
 

@Test
public void testWindowWithRelaxedTypes() throws Exception {
  StreamApplicationDescriptorImpl mockGraph = mock(StreamApplicationDescriptorImpl.class);
  OperatorSpec mockOpSpec = mock(OperatorSpec.class);
  MessageStream<TestInputMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);

  MapFunction<TestMessageEnvelope, String> keyExtractor = m -> m.getKey();
  FoldLeftFunction<TestMessageEnvelope, Integer> aggregator = (m, c) -> c + 1;
  SupplierFunction<Integer> initialValue = () -> 0;

  // should compile since TestMessageEnvelope (input for functions) is base class of TestInputMessageEnvelope (M)
  Window<TestInputMessageEnvelope, String, Integer> window = Windows
      .keyedTumblingWindow(keyExtractor, Duration.ofHours(1), initialValue, aggregator, null, mock(Serde.class));
  MessageStream<WindowPane<String, Integer>> windowedStream = inputStream.window(window, "w1");

  ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
  verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
  OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();

  assertTrue(registeredOpSpec instanceof WindowOperatorSpec);
  assertEquals(OpCode.WINDOW, registeredOpSpec.getOpCode());
  assertEquals(window, ((WindowOperatorSpec) registeredOpSpec).getWindow());
}
 

@Override
public <K, OM, JM> MessageStream<JM> join(MessageStream<OM> otherStream,
    JoinFunction<? extends K, ? super M, ? super OM, ? extends JM> joinFn,
    Serde<K> keySerde, Serde<M> messageSerde, Serde<OM> otherMessageSerde,
    Duration ttl, String userDefinedId) {
  if (otherStream.equals(this)) throw new SamzaException("Cannot join a MessageStream with itself.");
  String opId = this.streamAppDesc.getNextOpId(OpCode.JOIN, userDefinedId);
  OperatorSpec<?, OM> otherOpSpec = ((MessageStreamImpl<OM>) otherStream).getOperatorSpec();
  JoinOperatorSpec<K, M, OM, JM> op =
      OperatorSpecs.createJoinOperatorSpec(this.operatorSpec, otherOpSpec, (JoinFunction<K, M, OM, JM>) joinFn, keySerde,
          messageSerde, otherMessageSerde, ttl.toMillis(), opId);
  this.operatorSpec.registerNextOperatorSpec(op);
  otherOpSpec.registerNextOperatorSpec((OperatorSpec<OM, ?>) op);

  return new MessageStreamImpl<>(this.streamAppDesc, op);
}
 

@Test
public void testConfigureSerdesForRepartitionWithNoDefaultSystem() {
  // set the application to RepartitionOnlyStreamApplication
  mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionOnlyStreamApplication(), mockConfig);
  configureJobNode(mockStreamAppDesc);

  // create the JobGraphConfigureGenerator and generate the jobConfig for the jobNode
  JobNodeConfigurationGenerator configureGenerator = new JobNodeConfigurationGenerator();
  JobConfig jobConfig = configureGenerator.generateJobConfig(mockJobNode, "testJobGraphJson");

  // Verify the results
  Config expectedJobConfig = getExpectedJobConfig(mockConfig, mockJobNode.getInEdges());
  validateJobConfig(expectedJobConfig, jobConfig);

  Map<String, Serde> deserializedSerdes = validateAndGetDeserializedSerdes(jobConfig, 2);
  validateStreamConfigures(jobConfig, null);

  String partitionByKeySerde = jobConfig.get("streams.jobName-jobId-partition_by-p1.samza.key.serde");
  String partitionByMsgSerde = jobConfig.get("streams.jobName-jobId-partition_by-p1.samza.msg.serde");
  assertTrue("Serialized serdes should not contain intermediate stream key serde",
      !deserializedSerdes.containsKey(partitionByKeySerde));
  assertTrue("Serialized serdes should not contain intermediate stream msg serde",
      !deserializedSerdes.containsKey(partitionByMsgSerde));
}
 

private void readMessagesFromCoordinatorStream() {
  synchronized (bootstrapLock) {
    while (iterator.hasNext()) {
      IncomingMessageEnvelope envelope = iterator.next();
      byte[] keyAsBytes = (byte[]) envelope.getKey();
      Serde<List<?>> serde = new JsonSerde<>();
      Object[] keyArray = serde.fromBytes(keyAsBytes).toArray();
      CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>());
      String namespacedKey = serializeCoordinatorMessageKeyToJson(coordinatorStreamMessage.getType(), coordinatorStreamMessage.getKey());
      if (envelope.getMessage() != null) {
        messagesReadFromCoordinatorStream.put(namespacedKey, (byte[]) envelope.getMessage());
      } else {
        messagesReadFromCoordinatorStream.remove(namespacedKey);
      }
    }
  }
}
 

@Test
public void testGetInputStreamWithValueSerde() {

  String streamId = "test-stream-1";
  Serde mockValueSerde = mock(Serde.class);
  GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
  GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde);
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
    appDesc.getInputStream(isd);
  }, getConfig());

  InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId);
  assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
  assertEquals(streamId, inputOpSpec.getStreamId());
  assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId));
  assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, inputOpSpec.getValueSerde());
}
 

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamStreamJoin() {
  /**
   * Creates the following stream-stream join which is invalid due to partition count disagreement
   * between the 2 input streams.
   *
   *   input1 (64) --
   *                 |
   *                join -> output1 (8)
   *                 |
   *   input3 (32) --
   */
  return new StreamApplicationDescriptorImpl(appDesc -> {
    MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
    MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
    OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);

    messageStream1
        .join(messageStream3,
            mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(2), "j1")
        .sendTo(output1);
  }, config);
}
 

@Test
public void testGetIntermediateStreamWithValueSerde() {
  String streamId = "stream-1";
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig());

  Serde mockValueSerde = mock(Serde.class);
  IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl =
      streamAppDesc.getIntermediateStream(streamId, mockValueSerde, false);

  assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec());
  assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream());
  assertEquals(streamId, intermediateStreamImpl.getStreamId());
  assertTrue(intermediateStreamImpl.getOutputStream().getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde());
  assertTrue(((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde());
}
 

@Override
public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) {
  SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor();
  Optional<StreamExpander> expander = systemDescriptor.getExpander();
  if (expander.isPresent()) {
    return expander.get().apply(this, inputDescriptor);
  }

  // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream
  addInputDescriptor(inputDescriptor);

  String streamId = inputDescriptor.getStreamId();
  Serde serde = inputDescriptor.getSerde();
  KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);
  boolean isKeyed = serde instanceof KVSerde;
  InputTransformer transformer = inputDescriptor.getTransformer().orElse(null);
  InputOperatorSpec inputOperatorSpec =
      OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(),
          transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
  inputOperators.put(streamId, inputOperatorSpec);
  return new MessageStreamImpl(this, inputOperators.get(streamId));
}
 

@Test
public void testByteArray() {
  ByteArray key1 = ByteArray.of("hello world".getBytes(StandardCharsets.UTF_8));
  Serde<ByteArray> serde = new ByteArraySerdeFactory().getSerde("", null);
  byte[] keyBytes = serde.toBytes(key1);
  ByteArray key2 = serde.fromBytes(keyBytes);
  assertEquals(key1, key2);

  Map<ByteArray, String> map = new HashMap<>();
  map.put(key1, "found it");
  assertEquals("found it", map.get(key2));

  map.remove(key1);
  assertTrue(!map.containsKey(key2));
  assertTrue(map.isEmpty());
}
 

StoreDescriptor(String storeName, String storeFactory, Serde keySerde, Serde msgSerde,
    String changelogStream, Map<String, String> otherProperties) {
  this.storeName = storeName;
  this.storeFactory = storeFactory;
  this.keySerde = keySerde;
  this.msgSerde = msgSerde;
  this.changelogStream = changelogStream;
  this.otherProperties = otherProperties;
}
 

public InputOperatorSpec(String streamId, Serde keySerde, Serde valueSerde,
    InputTransformer transformer, boolean isKeyed, String opId) {
  super(OpCode.INPUT, opId);
  this.streamId = streamId;
  this.isKeyed = isKeyed;
  this.transformer = transformer;
  this.keySerde = keySerde;
  this.valueSerde = valueSerde;
}
 

@Override
public Collection<StoreDescriptor> getStoreDescriptors() {
  String storeName = getOpId();
  String storeFactory = "org.apache.samza.storage.kv.RocksDbKeyValueStorageEngineFactory";

  Serde storeKeySerde = new TimeSeriesKeySerde<>(window.getKeySerde());
  Serde storeValSerde = window.getFoldLeftFunction() == null ? window.getMsgSerde() : window.getWindowValSerde();

  StoreDescriptor descriptor = new StoreDescriptor(storeName, storeFactory, storeKeySerde, storeValSerde, storeName,
      Collections.emptyMap());
  return Collections.singletonList(descriptor);
}
 

@Override
public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) {
  String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId);
  IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true);
  BroadcastOperatorSpec<M> broadcastOperatorSpec =
      OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId);
  this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec);
  return intermediateStream;
}
 

private void validateStreamSerdeConfigure(String streamId, Config config, Map<String, Serde> deserializedSerdes) {
  Config streamConfig = config.subset(String.format("streams.%s.samza.", streamId));
  String keySerdeName = streamConfig.get("key.serde");
  String valueSerdeName = streamConfig.get("msg.serde");
  assertTrue(String.format("Serialized serdes should contain %s key serde", streamId), deserializedSerdes.containsKey(keySerdeName));
  assertTrue(String.format("Serialized %s key serde should be a StringSerde", streamId), keySerdeName.startsWith(StringSerde.class.getSimpleName()));
  assertTrue(String.format("Serialized serdes should contain %s msg serde", streamId), deserializedSerdes.containsKey(valueSerdeName));
  assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", streamId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName()));
}
 

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoin() {
  /**
   * Example stream-table join app. Expected partition counts of intermediate streams introduced
   * by partitionBy operations are enclosed in quotes.
   *
   *    input2 (16) -> partitionBy ("32") -> send-to-table t
   *
   *                                      join-table t —————
   *                                       |                |
   *    input1 (64) -> partitionBy ("32") _|                |
   *                                                       join -> output1 (8)
   *                                                        |
   *                                      input3 (32) ——————
   *
   */
  return new StreamApplicationDescriptorImpl(appDesc -> {
    MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
    MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
    MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
    OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);

    TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
        "table-id", new KVSerde(new StringSerde(), new StringSerde()));
    Table table = appDesc.getTable(tableDescriptor);

    messageStream2
        .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
        .sendTo(table);

    messageStream1
        .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p2")
        .join(table, mock(StreamTableJoinFunction.class))
        .join(messageStream3,
              mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2")
        .sendTo(output1);
  }, config);
}
 

private WindowOperatorSpec getWindowOperatorSpec(String opId) {
  WindowInternal<Object, Object, Collection> window = new WindowInternal<Object, Object, Collection>(
      defaultTrigger, supplierFunction, foldFn, keyFn, timeFn, WindowType.SESSION, null,
      mock(Serde.class), mock(Serde.class));
  if (earlyTrigger != null) {
    window.setEarlyTrigger(earlyTrigger);
  }
  if (lateTrigger != null) {
    window.setLateTrigger(lateTrigger);
  }
  return new WindowOperatorSpec<>(window, opId);
}
 

/**
 * Constructs an {@link InputDescriptor} instance.
 *
 * @param streamId id of the stream
 * @param serde serde for messages in the stream
 * @param systemDescriptor system descriptor this stream descriptor was obtained from
 * @param transformer stream level input stream transform function if available, else null
 */
public InputDescriptor(String streamId, Serde serde, SystemDescriptor systemDescriptor, InputTransformer transformer) {
  super(streamId, serde, systemDescriptor);

  // stream level transformer takes precedence over system level transformer
  if (transformer != null) {
    this.transformerOptional = Optional.of(transformer);
  } else {
    this.transformerOptional = systemDescriptor.getTransformer();
  }
}
 

@Test
public void testJoinOperatorSpec() {

  InputOperatorSpec leftOpSpec = new InputOperatorSpec(
      "test-input-1", new NoOpSerde<>(), new NoOpSerde<>(), null, false, "op0");
  InputOperatorSpec rightOpSpec = new InputOperatorSpec(
      "test-input-2", new NoOpSerde<>(), new NoOpSerde<>(), null, false, "op1");

  Serde<Object> objSerde = new Serde<Object>() {

    @Override
    public Object fromBytes(byte[] bytes) {
      return null;
    }

    @Override
    public byte[] toBytes(Object object) {
      return new byte[0];
    }
  };

  JoinFunction<String, Object, Object, TestOutputMessageEnvelope> joinFn = new TestJoinFunction();
  JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope> joinOperatorSpec =
      new JoinOperatorSpec<>(leftOpSpec, rightOpSpec, joinFn, new StringSerde("UTF-8"), objSerde, objSerde, 50000, "op2");
  JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope> joinOpCopy =
      (JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(joinOperatorSpec);
  assertNotEquals("Expected deserialized copy of operator spec should not be the same as the original operator spec", joinOperatorSpec, joinOpCopy);
  assertTrue(joinOperatorSpec.isClone(joinOpCopy));
  assertTrue(joinOpCopy.getLeftInputOpSpec().isClone(leftOpSpec));
  assertTrue(joinOpCopy.getRightInputOpSpec().isClone(rightOpSpec));
}
 

@Test(expected = IllegalStateException.class)
public void testGetSameIntermediateStreamTwice() {
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig());
  streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false);
  // should throw exception
  streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false);
}
 

@Override
public StorageEngine getStorageEngine(String storeName,
    File storeDir,
    Serde<Object> keySerde,
    Serde<Object> msgSerde,
    MessageCollector collector,
    MetricsRegistry registry,
    SystemStreamPartition changeLogSystemStreamPartition,
    JobContext jobContext,
    ContainerContext containerContext, StoreMode storeMode) {
  StoreProperties storeProperties = new StoreProperties.StorePropertiesBuilder().setLoggedStore(true).build();
  return new MockStorageEngine(storeName, storeDir, changeLogSystemStreamPartition, storeProperties);
}
 

@Test(expected = IllegalStateException.class)
public void testSetDefaultSystemDescriptorAfterGettingInputStream() {
  String streamId = "test-stream-1";
  GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
  GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class));

  new StreamApplicationDescriptorImpl(appDesc -> {
    appDesc.getInputStream(isd);
    appDesc.withDefaultSystem(sd); // should throw exception
  }, getConfig());
}
 

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  Serde<Profile> profileSerde = new JsonSerdeV2<>(Profile.class);
  Serde<PageView> pageViewSerde = new JsonSerdeV2<>(PageView.class);
  Serde<EnrichedPageView> joinResultSerde = new JsonSerdeV2<>(EnrichedPageView.class);

  KafkaSystemDescriptor kafkaSystemDescriptor = new KafkaSystemDescriptor(KAFKA_SYSTEM_NAME)
      .withConsumerZkConnect(KAFKA_CONSUMER_ZK_CONNECT)
      .withProducerBootstrapServers(KAFKA_PRODUCER_BOOTSTRAP_SERVERS)
      .withDefaultStreamConfigs(KAFKA_DEFAULT_STREAM_CONFIGS);

  KafkaInputDescriptor<Profile> profileInputDescriptor =
      kafkaSystemDescriptor.getInputDescriptor(PROFILE_STREAM_ID, profileSerde);
  KafkaInputDescriptor<PageView> pageViewInputDescriptor =
      kafkaSystemDescriptor.getInputDescriptor(PAGEVIEW_STREAM_ID, pageViewSerde);
  KafkaOutputDescriptor<EnrichedPageView> joinResultOutputDescriptor =
      kafkaSystemDescriptor.getOutputDescriptor(OUTPUT_TOPIC, joinResultSerde);

  RocksDbTableDescriptor<String, Profile> profileTableDescriptor =
      new RocksDbTableDescriptor<String, Profile>("profile-table", KVSerde.of(new StringSerde(), profileSerde));

  appDescriptor.withDefaultSystem(kafkaSystemDescriptor);

  MessageStream<Profile> profileStream = appDescriptor.getInputStream(profileInputDescriptor);
  MessageStream<PageView> pageViewStream = appDescriptor.getInputStream(pageViewInputDescriptor);
  OutputStream<EnrichedPageView> joinResultStream = appDescriptor.getOutputStream(joinResultOutputDescriptor);
  Table<KV<String, Profile>> profileTable = appDescriptor.getTable(profileTableDescriptor);

  profileStream
      .map(profile -> KV.of(profile.userId, profile))
      .sendTo(profileTable);

  pageViewStream
      .partitionBy(pv -> pv.userId, pv -> pv, KVSerde.of(new StringSerde(), pageViewSerde), "join")
      .join(profileTable, new JoinFn())
      .sendTo(joinResultStream);
}
 

private void validateTableConfigure(JobConfig jobConfig, Map<String, Serde> deserializedSerdes,
    TableDescriptor tableDescriptor) {
  Config tableConfig = jobConfig.subset(String.format("tables.%s.", tableDescriptor.getTableId()));
  assertEquals(MockTableProviderFactory.class.getName(), tableConfig.get("provider.factory"));
  assertEquals("mock.config.value", jobConfig.get("mock.table.provider.config"));
  validateTableSerdeConfigure(tableDescriptor.getTableId(), jobConfig, deserializedSerdes);
}
 

@Override
public StorageEngine getStorageEngine(String storeName,
    File storeDir,
    Serde<Object> keySerde,
    Serde<Object> msgSerde,
    MessageCollector collector,
    MetricsRegistry registry,
    SystemStreamPartition changeLogSystemStreamPartition,
    JobContext jobContext,
    ContainerContext containerContext, StoreMode storeMode) {
  StoreProperties storeProperties = new StoreProperties.StorePropertiesBuilder().setLoggedStore(false).build();
  return new MockStorageEngine(storeName, storeDir, changeLogSystemStreamPartition, storeProperties);
}
 

private <V> CouchbaseTableReadFunction<V> createAndInit(Class<V> valueClass, Serde<V> serde, Bucket bucket,
    AsyncBucket asyncBucket) {
  when(bucket.async()).thenReturn(asyncBucket);
  PowerMockito.stub(PowerMockito.method(CouchbaseBucketRegistry.class, "getBucket", String.class, List.class,
      CouchbaseEnvironmentConfigs.class)).toReturn(bucket);
  CouchbaseTableReadFunction<V> readFunction =
      new CouchbaseTableReadFunction<>(DEFAULT_BUCKET_NAME, valueClass, DEFAULT_CLUSTER_NODE).withSerde(serde);
  readFunction.init(mock(Context.class), mock(AsyncReadWriteTable.class));
  return readFunction;
}
 

private void validateTableSerdeConfigure(String tableId, Config config, Map<String, Serde> deserializedSerdes) {
  Config streamConfig = config.subset(String.format("stores.%s.", tableId));
  String keySerdeName = streamConfig.get("key.serde");
  String valueSerdeName = streamConfig.get("msg.serde");
  assertTrue(String.format("Serialized serdes should contain %s key serde", tableId), deserializedSerdes.containsKey(keySerdeName));
  assertTrue(String.format("Serialized %s key serde should be a StringSerde", tableId), keySerdeName.startsWith(StringSerde.class.getSimpleName()));
  assertTrue(String.format("Serialized serdes should contain %s value serde", tableId), deserializedSerdes.containsKey(valueSerdeName));
  assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", tableId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName()));
}
 

/**
 * Constructs an {@link OutputDescriptor} instance.
 *
 * @param streamId id of the stream
 * @param namespace namespace for the Event Hubs entity to produce to, not null
 * @param entityPath entity path for the Event Hubs entity to produce to, not null
 * @param valueSerde serde the values in the messages in the stream
 * @param systemDescriptor system descriptor this stream descriptor was obtained from
 */
EventHubsOutputDescriptor(String streamId, String namespace, String entityPath, Serde valueSerde,
    SystemDescriptor systemDescriptor) {
  super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor);
  this.namespace = StringUtils.stripToNull(namespace);
  this.entityPath = StringUtils.stripToNull(entityPath);
  if (this.namespace == null || this.entityPath == null) {
    throw new ConfigException(String.format("Missing namespace and entity path Event Hubs output descriptor in "
        + "system: {%s}, stream: {%s}", getSystemName(), streamId));
  }
}
 

@Test
public void testBroadcastChain() {
  String inputStreamId = "input";
  String inputSystem = "input-system";
  String inputPhysicalName = "input-stream";
  HashMap<String, String> configMap = new HashMap<>();
  configMap.put(JobConfig.JOB_NAME, "test-job");
  configMap.put(JobConfig.JOB_ID, "1");
  StreamTestUtils.addStreamConfigs(configMap, inputStreamId, inputSystem, inputPhysicalName);
  Config config = new MapConfig(configMap);
  when(this.context.getJobContext().getConfig()).thenReturn(config);
  StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> {
    GenericSystemDescriptor sd = new GenericSystemDescriptor(inputSystem, "mockFactoryClass");
    GenericInputDescriptor inputDescriptor = sd.getInputDescriptor(inputStreamId, mock(Serde.class));
    MessageStream<Object> inputStream = appDesc.getInputStream(inputDescriptor);
    inputStream.filter(mock(FilterFunction.class));
    inputStream.map(mock(MapFunction.class));
  }, config);

  OperatorImplGraph opImplGraph =
      new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), this.context, mock(Clock.class));

  InputOperatorImpl inputOpImpl = opImplGraph.getInputOperator(new SystemStream(inputSystem, inputPhysicalName));
  assertEquals(2, inputOpImpl.registeredOperators.size());
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.FILTER));
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.MAP));
}