Java Code Examples for com.streamsets.pipeline.api.Record#set()

@Test
public void testCreateLongField() throws Exception {
  String schema = "{\"name\": \"name\", \"type\": \"long\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, 3458236L, false);
  Assert.assertEquals(Field.Type.LONG, field.getType());
  Assert.assertEquals(3458236L, field.getValueAsLong());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof Long);
  Assert.assertTrue(3458236L == (Long) avroObject);

  //Check invalid type - String to Long
  makeBadType(Field.create("notLong"), record, avroSchema);
}
 

@Test
public void testSingleRecordAttrsNS() throws Exception {
  String xml = getXML("");
  Record expected = RecordCreator.create();
  expected.set(Field.create(createExpectedRecord("", "", "", "_", ".", "", true, true)));
  doTest(xml, "contact", "_", ".", "", ImmutableList.of(expected), Collections.EMPTY_LIST, OnRecordError.DISCARD,
      false, false, false, false);
}
 

private void setFieldsInRecord(Record record, Map<String, Field> fieldMap) {
  for (Map.Entry<String, Field> entry : fieldMap.entrySet()) {
    String columnName = entry.getKey();
    String fieldPath = columnsToFields.get(columnName.toLowerCase());
    Field field = entry.getValue();
    if (fieldPath == null) {
      Field root = record.get();
      // No mapping
      switch (root.getType()) {
        case LIST:
          // Add new field to the end of the list
          fieldPath = "[" + root.getValueAsList().size() + "]";
          field = Field.create(ImmutableMap.of(
              "header", Field.create(columnName),
              "value", field));
          break;
        case LIST_MAP:
        case MAP:
          // Just use the column name
          fieldPath = "/" + columnName;
          break;
        default:
          break;
      }
    }
    record.set(fieldPath, field);
  }
}
 

@Test
public void testConvertIntToStringInUnion() throws Exception {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  DataGenerator gen = new AvroDataOutputStreamGenerator(
    true,
    baos,
    COMPRESSION_CODEC_DEFAULT,
    null,
    null,
    null,
    null,
    0
  );

  Map<String, Field> rootField = new HashMap<>();
  rootField.put("string", Field.create(Field.Type.INTEGER, 10));

  Record r = RecordCreator.create();
  r.getHeader().setAttribute(BaseAvroDataGenerator.AVRO_SCHEMA_HEADER, STRING_UNION_SCHEMA);
  r.set(Field.create(rootField));
  gen.write(r);
  gen.close();

  GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(null);
  DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(
    new SeekableByteArrayInput(baos.toByteArray()), reader);
  Assert.assertTrue(dataFileReader.hasNext());
  GenericRecord readRecord = dataFileReader.next();

  Assert.assertEquals(new Utf8("10"), readRecord.get("string"));
  Assert.assertFalse(dataFileReader.hasNext());
}
 

@Test
public void testMapMergeOverwriteError() throws StageException {
  // Case where overwrite is required, but config is set to disallow overwriting
  FieldMergerConfig mergeConfig = new FieldMergerConfig();
  mergeConfig.fromField = "/source";
  mergeConfig.toField = "/target";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldMergerDProcessor.class)
      .setOnRecordError(OnRecordError.TO_ERROR)
      .addConfiguration("mergeMapping", ImmutableList.of(mergeConfig))
      .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
      .addConfiguration("overwriteExisting", false)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    String overwrittenKey = "c";
    String overwrittenValue = "cval_overwritten";
    Map<String, Field> sourceMap = ImmutableMap.of(
        "a", Field.create("aval"),
        overwrittenKey, Field.create(overwrittenValue));
    Map<String, Field> targetMap = ImmutableMap.of(
        overwrittenKey, Field.create("cval"),
        "d", Field.create("dval"));

    Map<String, Field> map = new LinkedHashMap<>();
    map.put("source", Field.create(Field.Type.MAP, sourceMap));
    map.put("target", Field.create(Field.Type.MAP, targetMap));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Assert.assertEquals(0, output.getRecords().get("a").size());
    Assert.assertEquals(1, runner.getErrorRecords().size());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testReplaceNullStringFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/stringField");
  stringFieldReplacement.newValue = "StreamSets";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("stringField", Field.create(Field.Type.STRING, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("stringField"));
    Assert.assertEquals("StreamSets", result.get("stringField").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testBooleanToInt() throws StageException {
  FieldTypeConverterConfig fieldTypeConverterConfig = new FieldTypeConverterConfig();
  fieldTypeConverterConfig.fields = ImmutableList.of("/t", "/f");
  fieldTypeConverterConfig.targetType = Field.Type.INTEGER;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
      .addConfiguration("convertBy", ConvertBy.BY_FIELD)
      .addConfiguration("fieldTypeConverterConfigs", ImmutableList.of(fieldTypeConverterConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("t", Field.create(true));
    map.put("f", Field.create(false));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.containsKey("f"));
    Assert.assertEquals(0, result.get("f").getValue());
    Assert.assertTrue(result.containsKey("t"));
    Assert.assertEquals(1, result.get("t").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test(expected = IOException.class)
public void testWriteAfterClose() throws Exception {
  ByteArrayOutputStream writer = new ByteArrayOutputStream();
  DataGenerator gen = new SdcRecordDataGenerator(
      getContextExtensions().createRecordWriter(writer),
      getContextExtensions()
  );
  Record record = RecordCreator.create();
  record.set(Field.create("Hello"));
  gen.close();
  gen.write(record);
}
 

private Record createRecordWithValue(String value) {
  Record record = RecordCreator.create();
  Map<String, Field> map = new HashMap<>();
  map.put("value", Field.create(value));
  record.set(Field.create(map));
  return record;
}
 

@Test
public void testMultipleRecordsNoDelimiter() throws Exception {
  String xml = "<contacts>" + getXML("0") + getXML("1") + "</contacts>";
  Record expected = RecordCreator.create();
  Map<String, Field> allFields = createExpectedRecord("contacts.", "0", "(0)", ".", "#", "", false, false);
  allFields.putAll(createExpectedRecord("contacts.", "1", "(1)", ".", "#", "", false, false));
  expected.set(Field.create(allFields));
  doTest(xml, "", ImmutableList.of(expected), Collections.EMPTY_LIST, OnRecordError.DISCARD, true, true, false, false);
}
 

@Test
public void testGetColumnsToParametersDeleteRecord() {
  Record record = RecordCreator.create();
  Map<String, Field> fields = new HashMap<>();
  Map<String, Field> o = new HashMap<>();
  o.put("_id", Field.create("5875d72a16cfd5cfcd99d0cb"));
  fields.put("o", Field.create(o));
  record.set(Field.create(fields));
  // column name to parameter mapping
  Map<String, String> parameter = ImmutableMap.of(
      "id", "?",
      "name", "?",
      "city", "?"
  );
  // column name to field name mapping
  Map<String, String> columnsToFields = ImmutableMap.of(
      "id", "/o/_id",
      "name", "/o/name",
      "city", "/o/city"
  );
  // expected result
  Map<String, String> expected = ImmutableMap.of(
    "id", "?"
  );

  JdbcRecordReader reader = new JdbcMongoDBOplogRecordReader();
  // record has all fields defined in column-to-field mapping
  Assert.assertEquals(
      expected,
      reader.getColumnsToParameters(record,
          OperationType.DELETE_CODE, parameter, columnsToFields));

}
 

@Test
public void testSimpleFlattenToRoot() throws Exception {
  FieldFlattenerConfig config = new FieldFlattenerConfig();
  config.nameSeparator = ".";
  config.flattenType = FlattenType.SPECIFIC_FIELDS;
  config.flattenInPlace = false;
  config.flattenTargetField = "/";
  config.fields = Arrays.asList("/map");

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFlattenerDProcessor.class, new FieldFlattenerProcessor(config))
          .addOutputLane("a").build();

  Record record = RecordCreator.create();
  final MapFieldBuilder builder = MapFieldBuilder.builder();
  builder.add("top-level", "top-level-value")
    .startMap("map")
      .startMap("map1").add("map1-1", "a").add("map1-2", "b").end()
    .end();

  record.set(builder.build());

  final Record output = runPipeline(runner, record);

  Field field = output.get();
  assertEquals(Field.Type.MAP, field.getType());

  Map<String, Field> result = field.getValueAsMap();
  assertThat(result.size(), equalTo(3));
  assertThat(result, hasKey("top-level"));
  assertThat(result, hasKey("map1.map1-1"));
  assertThat(result, hasKey("map1.map1-2"));
}
 

@Test
public void testInvalidType() throws Exception {
  FieldTypeConverterConfig config1 =
      new FieldTypeConverterConfig();
  config1.fields = ImmutableList.of("/zdt-1");
  config1.targetType = Field.Type.LONG;
  config1.dataLocale = "en";
  config1.zonedDateTimeFormat = ZonedDateTimeFormat.ISO_ZONED_DATE_TIME;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
      .addConfiguration("convertBy", ConvertBy.BY_FIELD)
      .addConfiguration("fieldTypeConverterConfigs",
          ImmutableList.of(config1))
      .addOutputLane("a").build();
  runner.runInit();

  Map<String, Field> map = new LinkedHashMap<>();
  ZonedDateTime current = ZonedDateTime.now();
  map.put("zdt-1", Field.createZonedDateTime(current));
  Record record = RecordCreator.create("s", "s:1");
  record.set(Field.create(map));

  try {
    runner.runProcess(ImmutableList.of(record));
    Assert.fail();
  } catch (StageException ex) {
    Assert.assertEquals(CONVERTER_04.getCode(), ex.getErrorCode().getCode());
  } finally {
    runner.runDestroy();
  }
}
 

/********************************************************/

  @Test
  public void testRemoveSimple() throws StageException {
    ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of("/name", "/age"))
      .addConfiguration("filterOperation", FilterOperation.REMOVE)
      .addOutputLane("a").build();
    runner.runInit();

    try {
      Map<String, Field> map = new LinkedHashMap<>();
      map.put("name", Field.create("a"));
      map.put("age", Field.create("b"));
      map.put("streetAddress", Field.create("c"));
      map.put("city", Field.create("d"));
      map.put("state", Field.create("e"));
      map.put("zip", Field.create(94040));
      Record record = RecordCreator.create("s", "s:1");
      record.set(Field.create(map));

      StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
      Assert.assertEquals(1, output.getRecords().get("a").size());
      Field field = output.getRecords().get("a").get(0).get();
      Assert.assertTrue(field.getValue() instanceof Map);
      Map<String, Field> result = field.getValueAsMap();
      Assert.assertTrue(result.size() == 4);
      Assert.assertTrue(result.containsKey("streetAddress"));
      Assert.assertTrue(result.containsKey("city"));
      Assert.assertTrue(result.containsKey("state"));
      Assert.assertTrue(result.containsKey("zip"));
    } finally {
      runner.runDestroy();
    }
  }
 

@Test
public void testSimpleJobExecution() throws Exception{
  File validationFile = new File(getInputDir(), "created");
  MapReduceExecutor executor = generateExecutor(ImmutableMap.<String, String>builder()
    .put("mapreduce.job.inputformat.class", SimpleTestInputFormat.class.getCanonicalName())
    .put("mapreduce.output.fileoutputformat.outputdir", getOutputDir())
    .put(SimpleTestInputFormat.FILE_LOCATION, validationFile.getAbsolutePath())
    .put(SimpleTestInputFormat.FILE_VALUE, "secret")
  .build());

  ExecutorRunner runner = new ExecutorRunner.Builder(MapReduceDExecutor.class, executor)
    .setOnRecordError(OnRecordError.TO_ERROR)
    .build();
  runner.runInit();

  Record record = RecordCreator.create();
  record.set(Field.create(ImmutableMap.of(
    "key", Field.create("value")
  )));

  runner.runWrite(ImmutableList.of(record));
  Assert.assertEquals(0, runner.getErrorRecords().size());

  // Event should be properly generated
  Assert.assertEquals(1, runner.getEventRecords().size());
  Record event = runner.getEventRecords().get(0);
  Assert.assertNotNull(event.get("/job-id").getValueAsString());

  // And we should create a special mark file while starting the mr job
  Assert.assertTrue(validationFile.exists());
  Assert.assertEquals("secret", FileUtils.readFileToString(validationFile));

  runner.runDestroy();
}
 

private Record createRecord() throws IOException {
  Map<String, Object> obj = new HashMap<>();
  obj.put("name", "hari");
  obj.put("age", 3100);
  obj.put("emails", ImmutableList.of("[email protected]", "[email protected]", "[email protected]"));

  Field field = JsonUtil.jsonToField(obj);
  Record r = RecordCreator.create();
  r.set(field);
  return r;
}
 

protected Record createRecord(long offset, Object json) throws DataParserException {
  Record record = context.createRecord(readerId + "::" + offset);
  record.set(jsonToField(json,  offset));
  return record;
}
 

@Test
public void testStringToIntegerGermanLocale() throws StageException {
  FieldTypeConverterConfig fieldTypeConverterConfig =
      new FieldTypeConverterConfig();
  fieldTypeConverterConfig.fields = ImmutableList.of("/beginner", "/intermediate", "/skilled", "/advanced",
      "/expert", "/null");
  fieldTypeConverterConfig.targetType = Field.Type.INTEGER;
  fieldTypeConverterConfig.dataLocale = "de";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
      .addConfiguration("convertBy", ConvertBy.BY_FIELD)
      .addConfiguration("fieldTypeConverterConfigs", ImmutableList.of(fieldTypeConverterConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("beginner", Field.create("-12234"));
    map.put("intermediate", Field.create("-1,234"));
    map.put("advanced", Field.create("12,567"));
    map.put("expert", Field.create("1.234"));
    map.put("skilled", Field.create("1234"));
    map.put("null", Field.create(Field.Type.STRING, null));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 6);
    Assert.assertTrue(result.containsKey("beginner"));
    Assert.assertEquals(-12234, result.get("beginner").getValue());
    Assert.assertTrue(result.containsKey("intermediate"));
    Assert.assertEquals(-1, result.get("intermediate").getValue());
    Assert.assertTrue(result.containsKey("advanced"));
    Assert.assertEquals(12, result.get("advanced").getValue());
    Assert.assertTrue(result.containsKey("expert"));
    Assert.assertEquals(1234, result.get("expert").getValue());
    Assert.assertTrue(result.containsKey("skilled"));
    Assert.assertEquals(1234, result.get("skilled").getValue());
    Assert.assertTrue(result.containsKey("null"));
    Assert.assertEquals(null, result.get("null").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testRenameMultipleListElementsWithConstIdxExpr() throws StageException {
  FieldRenamerConfig renameConfig1 = new FieldRenamerConfig();
  renameConfig1.fromFieldExpression = "/listOfInts[0]";
  renameConfig1.toFieldExpression = "/nonExisting0";

  FieldRenamerConfig renameConfig2 = new FieldRenamerConfig();
  renameConfig2.fromFieldExpression = "/listOfInts[1]";
  renameConfig2.toFieldExpression = "/nonExisting1";

  FieldRenamerConfig renameConfig3 = new FieldRenamerConfig();
  renameConfig3.fromFieldExpression = "/listOfInts[2]";
  renameConfig3.toFieldExpression = "/nonExisting2";

  FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
  errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.TO_ERROR;
  errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
  errorHandler.existingToFieldHandling = ExistingToFieldHandling.REPLACE;

  //Reverse order in configuration so as to preserve array indices
  FieldRenamerProcessor processor =
      new FieldRenamerProcessor(ImmutableList.of(renameConfig3, renameConfig2, renameConfig1),  errorHandler);

  // Test non-existent source with existing target field
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("listOfInts",
        Field.create(
            ImmutableList.of(
                Field.create(Field.Type.INTEGER, 1),
                Field.create(Field.Type.INTEGER, 2),
                Field.create(Field.Type.INTEGER, 3)
            )
        )
    );
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Map<String, Field> result = output.getRecords().get("a").get(0).get().getValueAsMap();
    Assert.assertTrue(result.containsKey("listOfInts"));
    Assert.assertTrue(result.get("listOfInts").getValueAsList().isEmpty());
    Assert.assertTrue(result.containsKey("nonExisting0"));
    Assert.assertTrue(result.containsKey("nonExisting1"));
    Assert.assertTrue(result.containsKey("nonExisting2"));
    Assert.assertEquals(1, result.get("nonExisting0").getValueAsInteger());
    Assert.assertEquals(2, result.get("nonExisting1").getValueAsInteger());
    Assert.assertEquals(3, result.get("nonExisting2").getValueAsInteger());
  } finally {
    runner.runDestroy();
  }
}
 

static Record createRecord() {
  Record record = RecordCreator.create();
  Map<String, Field> map = new HashMap<>();
  record.set(Field.create(map));
  return record;
}