Java Code Examples for org.apache.samza.operators.KV#getValue()

private PartialJoinOperatorImpl getOrCreatePartialJoinOpImpls(JoinOperatorSpec joinOpSpec, boolean isLeft,
    Clock clock) {
  // get the per task pair of PartialJoinOperatorImpl for the corresponding {@code joinOpSpec}
  KV<PartialJoinOperatorImpl, PartialJoinOperatorImpl> partialJoinOpImpls = joinOpImpls.computeIfAbsent(joinOpSpec.getOpId(),
    joinOpId -> {
      PartialJoinFunction leftJoinFn = createLeftJoinFn(joinOpSpec);
      PartialJoinFunction rightJoinFn = createRightJoinFn(joinOpSpec);
      return new KV(new PartialJoinOperatorImpl(joinOpSpec, true, leftJoinFn, rightJoinFn, clock),
          new PartialJoinOperatorImpl(joinOpSpec, false, rightJoinFn, leftJoinFn, clock));
    });

  if (isLeft) { // we got here from the left side of the join
    return partialJoinOpImpls.getKey();
  } else { // we got here from the right side of the join
    return partialJoinOpImpls.getValue();
  }
}
 

/**
 * Converts the nested avro object in SamzaMessage to relational message corresponding to
 * the tableName with relational schema.
 */
@Override
public SamzaSqlRelMessage convertToRelMessage(KV<Object, Object> samzaMessage) {
  List<String> payloadFieldNames = new ArrayList<>();
  List<Object> payloadFieldValues = new ArrayList<>();
  Object value = samzaMessage.getValue();
  if (value instanceof IndexedRecord) {
    fetchFieldNamesAndValuesFromIndexedRecord((IndexedRecord) value, payloadFieldNames, payloadFieldValues,
        payloadSchema);
  } else if (value == null) {
    // If the payload is null, set each record value as null
    payloadFieldNames.addAll(payloadSchema.getFields().stream().map(Schema.Field::name).collect(Collectors.toList()));
    IntStream.range(0, payloadFieldNames.size()).forEach(x -> payloadFieldValues.add(null));
  } else {
    String msg = "Avro message converter doesn't support messages of type " + value.getClass();
    LOG.error(msg);
    throw new SamzaException(msg);
  }

  return new SamzaSqlRelMessage(samzaMessage.getKey(), payloadFieldNames, payloadFieldValues,
      new SamzaSqlRelMsgMetadata(0L, 0L));
}
 

@Test
public void testNestedRecordConversion() {
  Map<String, String> props = new HashMap<>();
  SystemStream ss1 = new SystemStream("test", "nestedRecord");
  props.put(
      String.format(ConfigBasedAvroRelSchemaProviderFactory.CFG_SOURCE_SCHEMA, ss1.getSystem(), ss1.getStream()),
      Profile.SCHEMA$.toString());
  ConfigBasedAvroRelSchemaProviderFactory factory = new ConfigBasedAvroRelSchemaProviderFactory();
  AvroRelSchemaProvider nestedRecordSchemaProvider = (AvroRelSchemaProvider) factory.create(ss1, new MapConfig(props));
  AvroRelConverter nestedRecordAvroRelConverter = new AvroRelConverter(ss1, nestedRecordSchemaProvider, new MapConfig());

  Pair<SamzaSqlRelMessage, GenericData.Record> messageRecordPair =
      createNestedSamzaSqlRelMessage(nestedRecordAvroRelConverter);
  SamzaSqlRelMessageSerde serde =
      (SamzaSqlRelMessageSerde) new SamzaSqlRelMessageSerdeFactory().getSerde(null, null);
  SamzaSqlRelMessage resultMsg = serde.fromBytes(serde.toBytes(messageRecordPair.getKey()));
  KV<Object, Object> samzaMessage = nestedRecordAvroRelConverter.convertToSamzaMessage(resultMsg);
  GenericRecord recordPostConversion = (GenericRecord) samzaMessage.getValue();

  for (Schema.Field field : Profile.SCHEMA$.getFields()) {
    // equals() on GenericRecord does the nested record equality check as well.
    Assert.assertEquals(messageRecordPair.getValue().get(field.name()), recordPostConversion.get(field.name()));
  }
}
 

@Test
public void testSyncTaskWithMultiplePartitionMultithreadedWithCustomIME() throws Exception {
  Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
  Map<Integer, List<KV>> inputPartitionIME = new HashMap<>();
  Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
  genData(inputPartitionData, expectedOutputPartitionData);

  for (Map.Entry<Integer, List<KV>> entry: inputPartitionData.entrySet()) {
    Integer partitionId = entry.getKey();
    List<KV> messages = entry.getValue();
    SystemStreamPartition ssp = new SystemStreamPartition("test", "input", new Partition(partitionId));
    inputPartitionIME.put(partitionId, new ArrayList<>());
    int offset = 0;
    for (KV message: messages) {
      IncomingMessageEnvelope ime = new IncomingMessageEnvelope(ssp, String.valueOf(offset++), message.key, message.getValue());
      inputPartitionIME.get(partitionId).add(KV.of(message.key, ime));
    }
  }
  syncTaskWithMultiplePartitionMultithreadedHelper(inputPartitionIME, expectedOutputPartitionData);
}
 

private void addSerdes(KV<Serde, Serde> serdes, String streamId, Map<String, Serde> keySerdeMap,
    Map<String, Serde> msgSerdeMap) {
  if (serdes != null) {
    if (serdes.getKey() != null && !(serdes.getKey() instanceof NoOpSerde)) {
      keySerdeMap.put(streamId, serdes.getKey());
    }
    if (serdes.getValue() != null && !(serdes.getValue() instanceof NoOpSerde)) {
      msgSerdeMap.put(streamId, serdes.getValue());
    }
  }
}
 

@Ignore
@Test
public void testRecordConversionWithNullPayload() throws IOException {
  GenericData.Record record = null;
  SamzaSqlRelMessage relMessage = nestedRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record));

  LOG.info(relMessage.toString());

  KV<Object, Object> samzaMessage = nestedRecordAvroRelConverter.convertToSamzaMessage(relMessage);
  GenericRecord recordPostConversion = (GenericRecord) samzaMessage.getValue();

  Assert.assertTrue(recordPostConversion == null);
}
 

@Test
public void testNestedRecordConversionWithSubRecordsBeingNull() throws IOException {
  GenericData.Record record = new GenericData.Record(Profile.SCHEMA$);
  record.put("id", 1);
  record.put("name", "name1");
  record.put("companyId", 0);
  GenericData.Record addressRecord = null;
  record.put("address", addressRecord);
  record.put("selfEmployed", "True");

  List<GenericData.Record> phoneNumbers = null;
  record.put("phoneNumbers", phoneNumbers);

  HashMap<String, IndexedRecord> mapValues = null;
  record.put("mapValues", mapValues);

  SamzaSqlRelMessage relMessage = nestedRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record));

  LOG.info(relMessage.toString());

  KV<Object, Object> samzaMessage = nestedRecordAvroRelConverter.convertToSamzaMessage(relMessage);
  GenericRecord recordPostConversion = (GenericRecord) samzaMessage.getValue();

  for (Schema.Field field : Profile.SCHEMA$.getFields()) {
    // equals() on GenericRecord does the nested record equality check as well.
    Assert.assertEquals(record.get(field.name()), recordPostConversion.get(field.name()));
  }
}
 

@Test
public void testNestedRecordConversion() throws IOException {
  GenericData.Record record = new GenericData.Record(Profile.SCHEMA$);
  record.put("id", 1);
  record.put("name", "name1");
  record.put("companyId", 0);
  GenericData.Record addressRecord = new GenericData.Record(AddressRecord.SCHEMA$);
  addressRecord.put("zip", 90000);
  GenericData.Record streetNumRecord = new GenericData.Record(StreetNumRecord.SCHEMA$);
  streetNumRecord.put("number", 1200);
  addressRecord.put("streetnum", streetNumRecord);
  record.put("address", addressRecord);
  record.put("selfEmployed", "True");

  GenericData.Record phoneNumberRecordH = new GenericData.Record(PhoneNumber.SCHEMA$);
  phoneNumberRecordH.put("kind", Kind.Home);
  phoneNumberRecordH.put("number", "111-111-1111");
  GenericData.Record phoneNumberRecordC = new GenericData.Record(PhoneNumber.SCHEMA$);
  phoneNumberRecordC.put("kind", Kind.Cell);
  phoneNumberRecordC.put("number", "111-111-1112");
  List<GenericData.Record> phoneNumbers = new ArrayList<>();
  phoneNumbers.add(phoneNumberRecordH);
  phoneNumbers.add(phoneNumberRecordC);
  record.put("phoneNumbers", phoneNumbers);

  GenericData.Record simpleRecord1 = new GenericData.Record(SimpleRecord.SCHEMA$);
  simpleRecord1.put("id", 1);
  simpleRecord1.put("name", "name1");
  GenericData.Record simpleRecord2 = new GenericData.Record(SimpleRecord.SCHEMA$);
  simpleRecord2.put("id", 2);
  simpleRecord2.put("name", "name2");
  HashMap<String, IndexedRecord> mapValues = new HashMap<>();
  mapValues.put("key1", simpleRecord1);
  mapValues.put("key2", simpleRecord2);
  record.put("mapValues", mapValues);

  SamzaSqlRelMessage relMessage = nestedRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record));

  LOG.info(relMessage.toString());

  KV<Object, Object> samzaMessage = nestedRecordAvroRelConverter.convertToSamzaMessage(relMessage);
  GenericRecord recordPostConversion = (GenericRecord) samzaMessage.getValue();

  for (Schema.Field field : Profile.SCHEMA$.getFields()) {
    // equals() on GenericRecord does the nested record equality check as well.
    Assert.assertEquals(record.get(field.name()), recordPostConversion.get(field.name()));
  }
}
 

private void validateAvroSerializedData(byte[] serializedData, Object unionValue) throws IOException {
  GenericRecord complexRecordValue = genericRecordFromBytes(serializedData, ComplexRecord.SCHEMA$);

  SamzaSqlRelMessage message = complexRecordAvroRelConverter.convertToRelMessage(new KV<>("key", complexRecordValue));
  Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(),
      ComplexRecord.SCHEMA$.getFields().size() + 1);

  Assert.assertEquals(message.getSamzaSqlRelRecord().getField("id").get(), id);
  Assert.assertEquals(message.getSamzaSqlRelRecord().getField("bool_value").get(), boolValue);
  Assert.assertEquals(message.getSamzaSqlRelRecord().getField("double_value").get(), doubleValue);
  Assert.assertEquals(message.getSamzaSqlRelRecord().getField("string_value").get(), new Utf8(testStrValue));
  Assert.assertEquals(message.getSamzaSqlRelRecord().getField("float_value0").get(), floatValue);
  Assert.assertEquals(message.getSamzaSqlRelRecord().getField("long_value").get(), longValue);
  if (unionValue instanceof String) {
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("union_value").get(), new Utf8((String) unionValue));
  } else {
    Assert.assertEquals(message.getSamzaSqlRelRecord().getField("union_value").get(), unionValue);
  }
  Assert.assertTrue(arrayValue.stream()
      .map(Utf8::new)
      .collect(Collectors.toList())
      .equals(message.getSamzaSqlRelRecord().getField("array_values").get()));
  Assert.assertTrue(mapValue.entrySet()
      .stream()
      .collect(Collectors.toMap(x -> new Utf8(x.getKey()), y -> new Utf8(y.getValue())))
      .equals(message.getSamzaSqlRelRecord().getField("map_values").get()));

  Assert.assertTrue(
      Arrays.equals(((ByteString) message.getSamzaSqlRelRecord().getField("bytes_value").get()).getBytes(),
          testBytes.array()));
  Assert.assertTrue(
      Arrays.equals(((ByteString) message.getSamzaSqlRelRecord().getField("fixed_value").get()).getBytes(),
          DEFAULT_TRACKING_ID_BYTES));

  LOG.info(message.toString());

  KV<Object, Object> samzaMessage = complexRecordAvroRelConverter.convertToSamzaMessage(message);
  GenericRecord record = (GenericRecord) samzaMessage.getValue();

  for (Schema.Field field : ComplexRecord.SCHEMA$.getFields()) {
    if (field.name().equals("array_values")) {
      Assert.assertTrue(record.get(field.name()).equals(complexRecordValue.get(field.name())));
    } else {
      Object expected = complexRecordValue.get(field.name());
      Assert.assertEquals(expected, record.get(field.name()));
    }
  }
}
 

@Override
public StockPrice apply(KV<String, Void> message, KV<String, Double> record) {
  return record == null ? null : new StockPrice(message.getKey(), record.getValue());
}
 

@Override
public EnrichedPageView apply(KV<String, PageView> message, KV<String, Profile> record) {
  return record == null ? null :
      new EnrichedPageView(message.getKey(), record.getValue().company, message.getValue().pageId);
}