org.apache.beam.sdk.transforms.DoFnTester Java Examples

@Test
public void testHierarchical_TFD2119_B2_AllSubRecordsHaveValueGt10() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1{.value <= 10}", //
                    ConditionsRowConstant.Function.COUNT, //
                    ConditionsRowConstant.Operator.EQUAL, //
                    "0") //
            ));

    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        for (IndexedRecord subrecord : getSubrecords(main)) {
            assertThat(main.toString(), (double) subrecord.get(2), greaterThan(10d));
        }
    }
    assertThat(output, hasSize(58));
}
 

/** Tests that {@link ReadFn} retries after an error. */
@Test
public void testReadFnRetriesErrors() throws Exception {
  // An empty query to read entities.
  Query query = Query.newBuilder().setLimit(Int32Value.newBuilder().setValue(1)).build();

  // Use mockResponseForQuery to generate results.
  when(mockDatastore.runQuery(any(RunQueryRequest.class)))
      .thenThrow(new DatastoreException("RunQuery", Code.DEADLINE_EXCEEDED, "", null))
      .thenAnswer(
          invocationOnMock -> {
            Query q = ((RunQueryRequest) invocationOnMock.getArguments()[0]).getQuery();
            return mockResponseForQuery(q);
          });

  ReadFn readFn = new ReadFn(V_1_OPTIONS, mockDatastoreFactory);
  DoFnTester<Query, Entity> doFnTester = DoFnTester.of(readFn);
  doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
  doFnTester.processBundle(query);
}
 

/**
 * Retrieve the third element inside a loop. Since the first record have only two elements on its loops, it's going
 * to retrieve the last element.
 */
@Test
public void selectHierarchicalValuesGetOneElementFromLoops() throws Exception {
    FieldSelectorProperties properties = addSelector(null, "subElementFromList", ".b1[2].id");
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FieldSelectorDoFn().withProperties(properties));

    List<IndexedRecord> output =
            fnTester.processBundle(inputNullableHierarchicalWithLoops1, inputNullableHierarchicalWithLoops2);

    IndexedRecord output1 = output.get(0);
    List<Field> fields = output1.getSchema().getFields();
    assertEquals(1, fields.size());
    assertEquals("subElementFromList", fields.get(0).name());
    assertThat((Integer) output1.get(0), is(8));

    IndexedRecord output2 = output.get(1);
    fields = output2.getSchema().getFields();
    assertEquals(1, fields.size());
    assertEquals("subElementFromList", fields.get(0).name());
    assertThat((Integer) output2.get(0), is(2));
}
 

@Test
public void test_FlatMap_DuplicateInput() throws Exception {

    PythonRowProperties properties = new PythonRowProperties("test");
    properties.init();
    properties.mapType.setValue(MapType.FLATMAP);
    StringBuilder sb = new StringBuilder();
    sb.append("import copy\n");
    sb.append("outputList.append(input)\n");
    sb.append("outputList.append(input)\n");
    sb.append("outputList.append(copy.deepcopy(input))\n");
    properties.pythonCode.setValue(sb.toString());
    PythonRowDoFn function = new PythonRowDoFn();
    assertEquals(ValidationResult.OK, function.initialize(null, properties));
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputIndexedRecord);
    assertEquals(3, outputs.size());

    for (int i = 0; i < 3; i++) {
        GenericRecord outputRecord = (GenericRecord) outputs.get(i);
        compareRecords(inputIndexedRecord, outputRecord);
    }
}
 

@Test
public void test_FlatMap_doNothing() throws Exception {

    PythonRowProperties properties = new PythonRowProperties("test");
    properties.init();
    properties.mapType.setValue(MapType.FLATMAP);
    properties.pythonCode.setValue("outputList.append(input)");
    PythonRowDoFn function = new PythonRowDoFn();
    assertEquals(ValidationResult.OK, function.initialize(null, properties));
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputIndexedRecord);
    assertEquals(1, outputs.size());

    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    compareRecords(inputIndexedRecord, outputRecord);
}
 

private void datastoreWriterFnTest(int numMutations) throws Exception {
  // Create the requested number of mutations.
  List<Mutation> mutations = new ArrayList<>(numMutations);
  for (int i = 0; i < numMutations; ++i) {
    mutations.add(
        makeUpsert(Entity.newBuilder().setKey(makeKey("key" + i, i + 1)).build()).build());
  }

  DatastoreWriterFn datastoreWriter =
      new DatastoreWriterFn(
          StaticValueProvider.of(PROJECT_ID), null, mockDatastoreFactory, new FakeWriteBatcher());
  DoFnTester<Mutation, Void> doFnTester = DoFnTester.of(datastoreWriter);
  doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
  doFnTester.processBundle(mutations);

  int start = 0;
  while (start < numMutations) {
    int end = Math.min(numMutations, start + DatastoreV1.DATASTORE_BATCH_UPDATE_ENTITIES_START);
    CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
    commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
    commitRequest.addAllMutations(mutations.subList(start, end));
    // Verify all the batch requests were made with the expected mutations.
    verify(mockDatastore, times(1)).commit(commitRequest.build());
    start = end;
  }
}
 

@Test
public void test_FlatMap_MultipleInputs() throws Exception {

    PythonRowProperties properties = new PythonRowProperties("test");
    properties.init();
    properties.mapType.setValue(MapType.FLATMAP);
    StringBuilder sb = new StringBuilder();
    sb.append("import copy\n");
    sb.append("outputList.append(input)\n");
    sb.append("outputList.append(input)\n");
    sb.append("outputList.append(copy.deepcopy(input))\n");
    properties.pythonCode.setValue(sb.toString());
    PythonRowDoFn function = new PythonRowDoFn();
    assertEquals(ValidationResult.OK, function.initialize(null, properties));
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputIndexedRecord, inputIndexedRecord, inputIndexedRecord);
    assertEquals(9, outputs.size());

    for (int i = 0; i < 9; i++) {
        GenericRecord outputRecord = (GenericRecord) outputs.get(i);
        compareRecords(inputIndexedRecord, outputRecord);
    }
}
 

@Test
public void test_Map_doNothing() throws Exception {

    PythonRowProperties properties = new PythonRowProperties("test");
    properties.init();
    properties.mapType.setValue(MapType.MAP);
    properties.pythonCode.setValue("output = input");
    PythonRowDoFn function = new PythonRowDoFn();
    assertEquals(ValidationResult.OK, function.initialize(null, properties));
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputIndexedRecord);
    assertEquals(1, outputs.size());

    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    compareRecords(inputIndexedRecord, outputRecord);
}
 

@Test
public void testBatchViewAsSingletonToIsmRecord() throws Exception {
  DoFnTester<
          KV<Integer, Iterable<KV<GlobalWindow, WindowedValue<String>>>>,
          IsmRecord<WindowedValue<String>>>
      doFnTester =
          DoFnTester.of(
              new BatchViewOverrides.BatchViewAsSingleton.IsmRecordForSingularValuePerWindowDoFn<
                  String, GlobalWindow>(GlobalWindow.Coder.INSTANCE));

  assertThat(
      doFnTester.processBundle(
          ImmutableList.of(
              KV.of(
                  0, ImmutableList.of(KV.of(GlobalWindow.INSTANCE, valueInGlobalWindow("a")))))),
      contains(IsmRecord.of(ImmutableList.of(GlobalWindow.INSTANCE), valueInGlobalWindow("a"))));
}
 

@Test
public void test_Map_ApplyATransformation() throws Exception {

    PythonRowProperties properties = new PythonRowProperties("test");
    properties.init();
    properties.mapType.setValue(MapType.MAP);

    StringBuilder sb = new StringBuilder();
    sb.append("output = input\n");
    sb.append("output['a1'] = \"rootdata2\"\n");
    sb.append("output['B']['b1'] = \"subdatabefore\"\n");
    sb.append("output['B']['C']['c1'] = \"subsubdatabefore\"\n");
    sb.append("output['B']['C']['c2'] = 33\n");
    sb.append("output['B']['C']['c3'] = 55l\n");
    sb.append("output['B']['b2'] = \"subdataend\"\n");
    properties.pythonCode.setValue(sb.toString());
    PythonRowDoFn function = new PythonRowDoFn();
    assertEquals(ValidationResult.OK, function.initialize(null, properties));
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputIndexedRecord);
    assertEquals(1, outputs.size());

    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    compareRecords(outputIndexedRecord, outputRecord);
}
 

@Test
public void testBatchViewAsSingletonToIsmRecordWithMultipleValuesThrowsException()
    throws Exception {
  DoFnTester<
          KV<Integer, Iterable<KV<GlobalWindow, WindowedValue<String>>>>,
          IsmRecord<WindowedValue<String>>>
      doFnTester =
          DoFnTester.of(
              new BatchViewOverrides.BatchViewAsSingleton.IsmRecordForSingularValuePerWindowDoFn<
                  String, GlobalWindow>(GlobalWindow.Coder.INSTANCE));

  thrown.expect(IllegalStateException.class);
  thrown.expectMessage("found for singleton within window");
  doFnTester.processBundle(
      ImmutableList.of(
          KV.of(
              0,
              ImmutableList.of(
                  KV.of(GlobalWindow.INSTANCE, valueInGlobalWindow("a")),
                  KV.of(GlobalWindow.INSTANCE, valueInGlobalWindow("b"))))));
}
 

@Test
public void testHierarchicalSubRecordHasValueGt10() throws Exception {
    FieldSelectorProperties properties = addSelector(null, "value", ".b1{.value > 10}");
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FieldSelectorDoFn().withProperties(properties));

    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        List<Field> fields = main.getSchema().getFields();
        assertEquals(1, fields.size());
        assertEquals("value", fields.get(0).name());
        for (IndexedRecord element : (List<IndexedRecord>) main.get(0)) {
            List<Field> subFields = element.getSchema().getFields();
            assertEquals("id", subFields.get(0).name());
            assertEquals("name", subFields.get(1).name());
            assertEquals("value", subFields.get(2).name());
            assertEquals("b2", subFields.get(3).name());
            assertThat((Double) element.get(2), greaterThan(10d));
        }
    }
}
 

@Test
public void testHierarchical_TFD2119_B5_AtLeast1SubRecordsWithId1Or2HasValueGt10_Alternative() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1{.id == 1 && .value > 10 || .id == 2 && .value > 10}", //
                    ConditionsRowConstant.Function.COUNT, //
                    ConditionsRowConstant.Operator.GREATER, //
                    "0") //
            ));

    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        boolean atLeastOne = false;
        for (IndexedRecord subrecord : getSubrecords(main)) {
            int id = (int) subrecord.get(0);
            if ((double) subrecord.get(2) > 10 && (id == 1 || id == 2))
                atLeastOne = true;
        }
        assertThat(main.toString(), atLeastOne, is(true));
    }
    assertThat(output, hasSize(57));
}
 

@Ignore("Parenthesis are not correctly implemented in avpath.")
@Test
public void testHierarchical_TFD2119_B5_AtLeast1SubRecordsWithId1Or2HasValueGt10() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1{(.id == 1) || (.id == 2)) && .value > 10}", //
                    ConditionsRowConstant.Function.COUNT, //
                    ConditionsRowConstant.Operator.GREATER, //
                    "0") //
            ));

    // TODO: for the moment, this just throws an exception.
    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        boolean atLeastOne = false;
        for (IndexedRecord subrecord : getSubrecords(main)) {
            int id = (int) subrecord.get(0);
            if ((double) subrecord.get(2) > 10 && (id == 1 || id == 2))
                atLeastOne = true;
        }
        assertThat(main.toString(), atLeastOne, is(true));
    }
    assertThat(output, hasSize(57));
}
 

@Test
public void testHierarchical_TFD2119_B3_FirstRecordValueGt10() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1[0].value", //
                    null, //
                    ConditionsRowConstant.Operator.GREATER, //
                    "10") //
            ));

    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        assertThat(main.toString(), (double) getSubrecords(main).get(0).get(2), greaterThan(10d));
    }
    assertThat(output, hasSize(155));
}
 

@Test
public void testHierarchical_TFD2119_B7_HasAtLeastOneSubrecordWithSubSubRecordValueGt10() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1{.b2.value > 10}", //
                    ConditionsRowConstant.Function.COUNT, //
                    ConditionsRowConstant.Operator.GREATER, //
                    "0") //
            ));

    List<IndexedRecord> output = fnTester.processBundle(inputB);

    for (IndexedRecord main : output) {
        boolean atLeastOne = false;
        for (IndexedRecord subrecord : getSubrecords(main)) {
            for (IndexedRecord subsubrecord : getSubrecords(subrecord)) {
                if ((double) subsubrecord.get(2) > 10)
                    atLeastOne = true;
            }
        }
        assertThat(main.toString(), atLeastOne, is(true));
    }
    assertThat(output, hasSize(311));
}
 

@Test
public void testHierarchical_TFD2119_B1_AtLeastOneSubRecordHasValueGt10() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1{.value > 10}", //
                    ConditionsRowConstant.Function.COUNT, //
                    ConditionsRowConstant.Operator.GREATER, //
                    "0") //
            ));

    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        boolean atLeastOne = false;
        for (IndexedRecord subrecord : getSubrecords(main)) {
            if ((double) subrecord.get(2) > 10)
                atLeastOne = true;
        }
        assertThat(main.toString(), atLeastOne, is(true));
    }
    assertThat(output, hasSize(274));
}
 

@Test
public void testSplitReads_singleMatchedBase() throws Exception {
  DoFnTester<Read, KV<Position, ReadBaseQuality>> splitReads = DoFnTester.of(new SplitReads());

  // single matched base -> one SingleReadQuality proto
  Read r = Read.newBuilder()
      .setProperPlacement(true)
      .setAlignment(LinearAlignment.newBuilder()
          .setPosition(com.google.genomics.v1.Position.newBuilder()
              .setReferenceName("1")
              .setPosition(123))
          .addCigar(CigarUnit.newBuilder()
              .setOperation(Operation.ALIGNMENT_MATCH)
              .setOperationLength(1)))
      .setAlignedSequence("A")
      .addAlignedQuality(3)
      .build();
  Assert.assertThat(splitReads.processBundle(r), CoreMatchers.hasItems(KV.of(Position.newBuilder()
          .setReferenceName("1")
          .setPosition(123L)
          .build(),
          new ReadBaseQuality("A", 3))));
}
 

/**
 * Invalid path to normalize => throw error
 */
@Test
public void testNormalize_nothing() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();
    properties.isList.setValue(false);
    properties.trim.setValue(true);
    properties.discardTrailingEmptyStr.setValue(true);

    // Normalize `a` simple field
    properties.columnToNormalize.setValue(null);

    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(0, outputs.size());
}
 

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize simple field: `a`
 *
 * Expected normalized results of the field `a`:
 *
 * {"a": "aaa", "b": {"x": "x1;x2", "y": {"d": {"j": [{"l": "l1"}, {"l": "l2"}], "k": "k1;k2"}, "e": "e"}}, "c":
 * {"f": "f", "g": [{"h": "h1", "i": "i2"}, {"h": "h2", "i": "i1"}]}, "m": ["m1", "m2", "m3"]}
 *
 * @throws Exception
 */
@Test
public void testNormalizeSimpleFields_a() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();
    properties.isList.setValue(false);
    properties.trim.setValue(true);
    properties.discardTrailingEmptyStr.setValue(true);

    // Normalize `a` simple field
    properties.columnToNormalize.setValue("a");

    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(1, outputs.size());
    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    Assert.assertEquals(inputParentRecord.toString(), outputRecord.toString());
    Assert.assertEquals(inputParentRecord.getSchema().toString(), outputRecord.getSchema().toString());
}
 

@Test
public void selectSimpleElements() throws Exception {
    FieldSelectorProperties properties = addSelector(null, "aOutput", "a");
    properties = addSelector(properties, "cOutput", "c");
    properties = addSelector(properties, "aSecondOutput", "a");
    properties = addSelector(properties, "bOutput", "b");

    FieldSelectorDoFn function = new FieldSelectorDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputSimpleRecord);

    assertEquals(1, outputs.size());
    List<Field> fields = outputs.get(0).getSchema().getFields();
    assertEquals(4, fields.size());
    assertEquals("aOutput", fields.get(0).name());
    assertEquals("aaa", outputs.get(0).get(0));
    assertEquals("cOutput", fields.get(1).name());
    assertEquals("Ccc", outputs.get(0).get(1));
    assertEquals("aSecondOutput", fields.get(2).name());
    assertEquals("aaa", outputs.get(0).get(2));
    assertEquals("bOutput", fields.get(3).name());
    assertEquals("BBB", outputs.get(0).get(3));
}
 

@Test
public void testHierarchical_TFD2119_A2_Subrecord() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".a1.id", //
                    null, //
                    ConditionsRowConstant.Operator.EQUAL, //
                    "1") //
            ));

    List<IndexedRecord> output = fnTester.processBundle(inputA);
    for (IndexedRecord main : output) {
        List<IndexedRecord> subrecords = getSubrecords(main);
        assertThat(main.toString(), subrecords.get(0).get(0), is((Object) 1));
    }
    assertThat(output, hasSize(98));
}
 

@Test
public void testHierarchical_TFD2119_A1_TopLevel() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".id", //
                    null, //
                    ConditionsRowConstant.Operator.EQUAL, //
                    "1") //
            ));

    List<IndexedRecord> output = fnTester.processBundle(inputA);
    for (IndexedRecord main : output) {
        assertThat(main.toString(), main.get(0), is((Object) 1));
    }
    assertThat(output, hasSize(103));
}
 

@Test
public void testHierarchical_TFD2119_ERR4_NullArray() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1[0].id", //
                    null, //
                    ConditionsRowConstant.Operator.EQUAL, //
                    "1") //
            ));

    // Looks like this is not an exception -- it just doesn't match.
    IndexedRecord[] input = copyAndReplaceSubrecordArray(inputB, 10, true);
    List<IndexedRecord> output = fnTester.processBundle(input);
    for (IndexedRecord main : output) {
        List<IndexedRecord> subrecords = getSubrecords(main);
        assertThat(main.toString(), subrecords.get(0).get(0), is((Object) 1));
    }
    assertThat(output, hasSize(102));
}
 

@Test
public void testHierarchical_TFD2119_ERR3_ArrayOutOfBounds() throws Exception {
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of( //
            new FilterRowDoFn(addCriteria(null, //
                    ".b1[99].id", //
                    null, //
                    ConditionsRowConstant.Operator.EQUAL, //
                    "1") //
            ));

    // Looks like this is not an exception -- it considers .b1[99] to be the last record.
    List<IndexedRecord> output = fnTester.processBundle(inputB);
    for (IndexedRecord main : output) {
        List<IndexedRecord> subrecords = getSubrecords(main);
        assertThat(main.toString(), subrecords.get(subrecords.size() - 1).get(0), is((Object) 1));
    }
    assertThat(output, hasSize(114));
}
 

/**
 * When there are no user input, the component not return any data
 */
@Test
public void noSelectorForPath() throws Exception {
    // Create a filter row with exactly one criteria that hasn't been filled by the user.
    FieldSelectorProperties properties = addSelector(null, null, ".test");
    assertThat(properties.selectors.getPropertiesList(), hasSize(1));

    SelectorProperties selector = properties.selectors.getPropertiesList().iterator().next();
    assertThat(selector.field.getStringValue(), is(""));
    assertThat(selector.path.getStringValue(), is(".test"));

    FieldSelectorDoFn function = new FieldSelectorDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);

    //
    List<IndexedRecord> outputs = fnTester.processBundle(inputSimpleRecord);
    assertEquals(0, outputs.size());
}
 

@Test
public void selectSimpleElementsWithEmptyValues() throws Exception {
    FieldSelectorProperties properties = addSelector(null, "aOutput", "a");
    properties = addSelector(properties, "cOutput", "c");
    properties = addSelector(properties, "aSecondOutput", "a");
    properties = addSelector(properties, "bOutput", "b");

    FieldSelectorDoFn function = new FieldSelectorDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputRecordWithEmptyValue);

    assertEquals(1, outputs.size());
    List<Field> fields = outputs.get(0).getSchema().getFields();
    assertEquals(4, fields.size());
    assertEquals("aOutput", fields.get(0).name());
    assertEquals("aaa", outputs.get(0).get(0));
    assertEquals("cOutput", fields.get(1).name());
    assertNull(outputs.get(0).get(1));
    assertEquals("aSecondOutput", fields.get(2).name());
    assertEquals("aaa", outputs.get(0).get(2));
    assertEquals("bOutput", fields.get(3).name());
    assertEquals("BBB", outputs.get(0).get(3));
}
 

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize complex field: `b`
 *
 * Expected: no change
 *
 * @throws Exception
 */
@Test
public void testNormalizeComplexFields_b() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();

    // Normalize `b` complex field
    properties.columnToNormalize.setValue("b");

    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(1, outputs.size());
    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    Assert.assertEquals(inputParentRecord.toString(), outputRecord.toString());
    Assert.assertEquals(inputParentRecord.getSchema().toString(), outputRecord.getSchema().toString());
}
 

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize complex field: `b.y`
 *
 * Expected: no change
 *
 * @throws Exception
 */
@Test
public void testNormalizeComplexFields_by() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();

    // Normalize `b.y` complex field
    properties.columnToNormalize.setValue("b.y");

    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(1, outputs.size());
    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    Assert.assertEquals(inputParentRecord.toString(), outputRecord.toString());
    Assert.assertEquals(inputParentRecord.getSchema().toString(), outputRecord.getSchema().toString());

}
 

/**
 * Input parent record: {@link NormalizeDoFnTest#inputParentRecord}
 *
 * Normalize complex field: `b.y.d`
 *
 * Expected: no change
 *
 * @throws Exception
 */
@Test
public void testNormalizeComplexFields_byd() throws Exception {
    NormalizeProperties properties = new NormalizeProperties("test");
    properties.init();
    properties.schemaListener.afterSchema();

    // Normalize `b.y.d` complex field
    properties.columnToNormalize.setValue("b.y.d");

    NormalizeDoFn function = new NormalizeDoFn().withProperties(properties);
    DoFnTester<IndexedRecord, IndexedRecord> fnTester = DoFnTester.of(function);
    List<IndexedRecord> outputs = fnTester.processBundle(inputParentRecord);
    Assert.assertEquals(1, outputs.size());
    GenericRecord outputRecord = (GenericRecord) outputs.get(0);
    Assert.assertEquals(inputParentRecord.toString(), outputRecord.toString());
    Assert.assertEquals(inputParentRecord.getSchema().toString(), outputRecord.getSchema().toString());
}