Java Code Examples for com.streamsets.pipeline.api.Field#getValueAsMap()

private Record createSchemaChangeRecordWithMissingFields(String missingField) throws StageException{
  Record r = RecordCreator.create();
  Field f = HiveMetastoreUtil.newSchemaMetadataFieldBuilder(
      "default",
      "sample",
      generateColumnTypeInfo(),
      generatePartitionTypeInfo(),
      true,
      HiveTestUtil.WAREHOUSE_DIR +"/sample",
      "",
      HMPDataFormat.AVRO
  );
  Map<String, Field> fieldMap = f.getValueAsMap();
  fieldMap.remove(missingField);
  r.set(Field.create(fieldMap));
  return r;
}
 

@Test
public void testKeepNonExistingFiled() throws StageException {
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
    .addConfiguration("fields", ImmutableList.of("/city"))
    .addConfiguration("filterOperation", FilterOperation.KEEP)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("name", Field.create("a"));
    map.put("age", Field.create("b"));
    map.put("streetAddress", Field.create("c"));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 0);
  } finally {
    runner.runDestroy();
  }
}
 

@Override
protected DecimalTypeInfo generateJdbcTypeInfoFromMetadataField(JdbcType type, Field JdbcTypeField, JdbcSchemaWriter schemaWriter)
    throws JdbcStageCheckedException {
  Map<String, Field> extraInfo = JdbcTypeField.getValueAsMap();
  if(!extraInfo.containsKey(SCALE)) {
    throw new JdbcStageCheckedException(JdbcErrors.JDBC_301, Utils.format("Missing {} in field {}", SCALE, JdbcTypeField));
  }
  if(!extraInfo.containsKey(PRECISION)) {
    throw new JdbcStageCheckedException(JdbcErrors.JDBC_301, Utils.format("Missing {} in field {}", PRECISION, JdbcTypeField));
  }
  int scale = extraInfo.get(SCALE).getValueAsInteger();
  int precision = extraInfo.get(PRECISION).getValueAsInteger();
  return new DecimalTypeInfo(schemaWriter, precision, scale);
}
 

@Test
public void testFieldMaskProcessorVariableLength() throws StageException {

  FieldMaskConfig nameMaskConfig = new FieldMaskConfig();
  nameMaskConfig.fields = ImmutableList.of("/name", "/age", "/ssn", "/phone");
  nameMaskConfig.maskType = MaskType.VARIABLE_LENGTH;
  nameMaskConfig.mask = null;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldMaskDProcessor.class)
      .addConfiguration("fieldMaskConfigs", ImmutableList.of(nameMaskConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("name", Field.create("streamsetsinc"));
    map.put("age", Field.create("12"));
    map.put("ssn", Field.create("123-45-6789"));
    map.put("phone", Field.create(Field.Type.STRING, null));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 4);
    Assert.assertTrue(result.containsKey("name"));
    Assert.assertEquals("xxxxxxxxxxxxx", result.get("name").getValue());
    Assert.assertTrue(result.containsKey("age"));
    Assert.assertEquals("xx", result.get("age").getValue());
    Assert.assertTrue(result.containsKey("ssn"));
    Assert.assertEquals("xxxxxxxxxxx", result.get("ssn").getValue());
    Assert.assertTrue(result.containsKey("phone"));
    Assert.assertEquals(null, result.get("phone").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testReplaceNullByteArrayFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/byteArrayField");
  stringFieldReplacement.newValue = "streamsets";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("byteArrayField", Field.create(Field.Type.BYTE_ARRAY, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("byteArrayField"));
    Assert.assertTrue(Arrays.equals("streamsets".getBytes(), result.get("byteArrayField").getValueAsByteArray()));
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testByteField() throws StageException {
  HasherConfig hasherConfig = createInPlaceHasherProcessor(ImmutableList.of("/age"), hashType);

  FieldHasherProcessor processor = new FieldHasherProcessor(hasherConfig, OnStagePreConditionFailure.CONTINUE);

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldHasherDProcessor.class, processor)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("age", Field.create(Field.Type.BYTE, 125));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("age"));
    Assert.assertEquals(computeHash(Field.Type.BYTE, "125", hashType),
        result.get("age").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testReplaceNullLongFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/longField");
  stringFieldReplacement.newValue = "523456789345";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("longField", Field.create(Field.Type.LONG, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("longField"));
    Assert.assertEquals(523456789345L, result.get("longField").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testMultiStringMD5NoSeparator() throws StageException {
  // new way - after SDC-6540.
  HasherConfig hasherConfig = TestFieldHasherProcessor.createTargetFieldHasherProcessor(
      ImmutableList.of("/x", "/y", "/z"),
      HashType.MD5,
      "/hash",
      ""
  );
  hasherConfig.useSeparator = false;
  FieldHasherProcessor processor = new FieldHasherProcessor(hasherConfig, OnStagePreConditionFailure.CONTINUE);

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldHasherDProcessor.class, processor)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Record record = createRec(DATA1, DATA2, DATA3);

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);

    Map<String, Field> result = field.getValueAsMap();
    verifyResult(result);

    Assert.assertEquals(DigestUtils.md5Hex(DATA1+DATA2+DATA3), result.get("hash").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Override
protected DecimalTypeInfo generateHiveTypeInfoFromMetadataField(HiveType type, String comment, Field hiveTypeField)
    throws HiveStageCheckedException {
  Map<String, Field> extraInfo = hiveTypeField.getValueAsMap();
  if(!extraInfo.containsKey(SCALE)) {
    throw new HiveStageCheckedException(Errors.HIVE_01, Utils.format("Missing {} in field {}", SCALE, hiveTypeField));
  }
  if(!extraInfo.containsKey(PRECISION)) {
    throw new HiveStageCheckedException(Errors.HIVE_01, Utils.format("Missing {} in field {}", PRECISION, hiveTypeField));
  }
  int scale = extraInfo.get(SCALE).getValueAsInteger();
  int precision = extraInfo.get(PRECISION).getValueAsInteger();
  return new DecimalTypeInfo(comment, precision, scale);
}
 

@Test
public void testCreateMapPrimitiveField() throws Exception {
  String schema = "{\"type\":\"map\",\"values\":\"int\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);

  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, ImmutableMap.of(new Utf8("Hari"), 1, new Utf8("Kiran"), 2), false);
  Assert.assertEquals(Field.Type.MAP, field.getType());
  Map<String, Field> valueAsMap = field.getValueAsMap();

  Assert.assertTrue(valueAsMap.containsKey("Hari"));
  Field hari = valueAsMap.get("Hari");
  Assert.assertEquals(Field.Type.INTEGER, hari.getType());
  Assert.assertEquals(1, hari.getValueAsInteger());

  Assert.assertTrue(valueAsMap.containsKey("Kiran"));
  hari = valueAsMap.get("Kiran");
  Assert.assertEquals(Field.Type.INTEGER, hari.getType());
  Assert.assertEquals(2, hari.getValueAsInteger());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof Map<?, ?>);

  Map<String, Integer> map = (Map<String, Integer>) avroObject;
  Assert.assertTrue(map.containsKey("Hari"));
  Assert.assertEquals(1, (int) map.get("Hari"));
  Assert.assertTrue(map.containsKey("Kiran"));
  Assert.assertEquals(2, (int) map.get("Kiran"));

  //Check invalid type - String to Map
  makeBadType(Field.create("notMicroseconds"), record, avroSchema);
}
 

@Test
public void testRemoveNonExistingField() throws StageException {
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
    .addConfiguration("fields", ImmutableList.of("/city"))
    .addConfiguration("filterOperation", FilterOperation.REMOVE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("name", Field.create("a"));
    map.put("age", Field.create("b"));
    map.put("streetAddress", Field.create("c"));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 3);
    Assert.assertTrue(result.containsKey("name"));
    Assert.assertTrue(result.containsKey("age"));
    Assert.assertTrue(result.containsKey("streetAddress"));
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testStringToChar() throws StageException {
  FieldTypeConverterConfig fieldTypeConverterConfig =
      new FieldTypeConverterConfig();
  fieldTypeConverterConfig.fields = ImmutableList.of("/beginner", "/intermediate", "/null");
  fieldTypeConverterConfig.targetType = Field.Type.CHAR;
  fieldTypeConverterConfig.dataLocale = "en";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
      .addConfiguration("convertBy", ConvertBy.BY_FIELD)
      .addConfiguration("fieldTypeConverterConfigs", ImmutableList.of(fieldTypeConverterConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("beginner", Field.create("a"));
    map.put("intermediate", Field.create("yes"));
    map.put("null", Field.create(Field.Type.STRING, null));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 3);
    Assert.assertTrue(result.containsKey("beginner"));
    Assert.assertEquals('a', result.get("beginner").getValue());
    Assert.assertTrue(result.containsKey("intermediate"));
    Assert.assertEquals('y', result.get("intermediate").getValue());
    Assert.assertTrue(result.containsKey("null"));
    Assert.assertEquals(null, result.get("null").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testReplaceNullDoubleFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/doubleField");
  stringFieldReplacement.newValue = "12345.6789";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("doubleField", Field.create(Field.Type.DOUBLE, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("doubleField"));
    Assert.assertEquals(12345.6789, result.get("doubleField").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testReplaceNullCharFields() throws StageException {

  FieldValueReplacerConfig stringFieldReplacement = new FieldValueReplacerConfig();
  stringFieldReplacement.fields = ImmutableList.of("/charField");
  stringFieldReplacement.newValue = "c";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldValueReplacerDProcessor.class)
    .addConfiguration("nullReplacerConditionalConfigs", null)
    .addConfiguration("fieldsToReplaceIfNull", ImmutableList.of(stringFieldReplacement))
      .addConfiguration("fieldsToConditionallyReplace", null)
    .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
    .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("charField", Field.create(Field.Type.CHAR, null));
          Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("charField"));
    Assert.assertEquals('c', result.get("charField").getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testLongToDateTimeString() throws Exception {
  FieldTypeConverterConfig dtConfig =
      new FieldTypeConverterConfig();
  dtConfig.fields = ImmutableList.of("/dateString");
  dtConfig.targetType = Field.Type.STRING;
  dtConfig.dataLocale = "en";
  dtConfig.treatInputFieldAsDate = true;
  dtConfig.dateFormat = DateFormat.OTHER;
  dtConfig.otherDateFormat = "yyyy-MM-dd";

  FieldTypeConverterConfig stringConfig =
      new FieldTypeConverterConfig();
  stringConfig.fields = ImmutableList.of("/LongString");
  stringConfig.targetType = Field.Type.STRING;
  stringConfig.treatInputFieldAsDate = false;
  stringConfig.dataLocale = "en";
  stringConfig.dateFormat = DateFormat.OTHER;
  stringConfig.otherDateFormat = "yyyy-MM-dd HH:mm:ss";


  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
      .addConfiguration("convertBy", ConvertBy.BY_FIELD)
      .addConfiguration("fieldTypeConverterConfigs", ImmutableList.of(stringConfig, dtConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("LongString", Field.create(1431763314761L));
    map.put("dateString", Field.create(1431763314761L));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertEquals("1431763314761", result.get("LongString").getValueAsString());
    Assert.assertEquals("2015-05-16", result.get("dateString").getValueAsString());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testMapMergeOverwriteSuccess() throws StageException {
  // If source and target contain fields with the same name, overwrite determines success
  FieldMergerConfig mergeConfig = new FieldMergerConfig();
  mergeConfig.fromField = "/source";
  mergeConfig.toField = "/target";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldMergerDProcessor.class)
      .addConfiguration("mergeMapping", ImmutableList.of(mergeConfig))
      .addConfiguration("onStagePreConditionFailure", OnStagePreConditionFailure.CONTINUE)
      .addConfiguration("overwriteExisting", true)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    String overwrittenKey = "c";
    String overwrittenValue = "cval_overwritten";
    Map<String, Field> sourceMap = ImmutableMap.of(
        "a", Field.create("aval"),
        overwrittenKey, Field.create(overwrittenValue));
    Map<String, Field> targetMap = ImmutableMap.of(
        overwrittenKey, Field.create("cval"),
        "d", Field.create("dval"));

    Map<String, Field> map = new LinkedHashMap<>();
    map.put("source", Field.create(Field.Type.MAP, sourceMap));
    map.put("target", Field.create(Field.Type.MAP, targetMap));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 1);
    Assert.assertTrue(result.containsKey("target"));
    Assert.assertTrue(!result.containsKey("source"));
    Map<String, Field> targetVal = result.get("target").getValueAsMap();
    Assert.assertEquals(3, targetVal.keySet().size());
    Assert.assertEquals(overwrittenValue, targetVal.get(overwrittenKey).getValueAsString());
  } finally {
    runner.runDestroy();
  }
}
 

/**
 * Map mode
 */
private void writeFieldToDocumentMap(Document doc, Field field, String name) throws IOException {
  if (field.getValue() == null) {
    // On the Map mode just set the null on the document using the name.
    doc.setNull(name);
  } else {
    switch (field.getType()) {
      case FILE_REF:
        throw new IOException("Cannot serialize FileRef fields.");
      case MAP:
      case LIST_MAP:
        Document newDoc = loader.createNewEmptyDocument();
        Map<String, Field> map = field.getValueAsMap();
        for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
          String fieldName = fieldEntry.getKey();
          Field newField = fieldEntry.getValue();
          // recursive call in map mode.
          writeFieldToDocumentMap(newDoc, newField, fieldName);
        }
        // Map the new doc
        doc.set(name, newDoc);
        break;
      case LIST:
        List<Field> listOfFields = field.getValueAsList();
        List<Object> objsList = new ArrayList<>();
        for (Field f : listOfFields) {
          // recursive call in a list mode.
          writeFieldToDocumentList(f, objsList);
        }
        doc.setArray(name, objsList.toArray());
        break;
      case BOOLEAN:
        doc.set(name, field.getValueAsBoolean());
        break;
      case CHAR:
        doc.set(name, String.valueOf(field.getValueAsChar()));
        break;
      case BYTE:
        doc.set(name, new byte[]{field.getValueAsByte()});
        break;
      case SHORT:
        doc.set(name, field.getValueAsShort());
        break;
      case INTEGER:
        doc.set(name, field.getValueAsInteger());
        break;
      case LONG:
        doc.set(name, field.getValueAsLong());
        break;
      case FLOAT:
        doc.set(name, field.getValueAsFloat());
        break;
      case DOUBLE:
        doc.set(name, field.getValueAsDouble());
        break;
      case DATE:
        doc.set(name, field.getValueAsDate().getTime());
        break;
      case DATETIME:
        doc.set(name, field.getValueAsDatetime().getTime());
        break;
      case TIME:
        doc.set(name, field.getValueAsTime().getTime());
        break;
      case DECIMAL:
        doc.set(name, field.getValueAsDecimal());
        break;
      case STRING:
      case ZONED_DATETIME:
        doc.set(name, field.getValueAsString());
        break;
      case BYTE_ARRAY:
        doc.set(name, field.getValueAsByteArray());
        break;
      default:
        throw new IllegalStateException(String.format("Unrecognized field type (%s) in field: %s",
            field.getType().name(),
            field.toString()
        ));
    }
  }
}
 

@Test
public void testSimpleMatch() throws Exception {
  List<String> outputFields = ImmutableList.of("Stream", "Google", "Hadoop");

  ProcessorRunner runner = new ProcessorRunner.Builder(FuzzyFieldDProcessor.class)
      .addConfiguration("rootFieldPaths", ImmutableList.of("/"))
      .addConfiguration("outputFieldNames", outputFields)
      .addConfiguration("matchThreshold", 60)
      .addConfiguration("allCandidates", true)
      .addConfiguration("inPlace", false)
      .addConfiguration("preserveUnmatchedFields", true)
      .addOutputLane("a").build();

  runner.runInit();

  try {
    List<Field> csvWithHeader = new ArrayList<>();
    Map<String, Field> f1 = new HashMap<>();
    f1.put("header", Field.create("Straem"));
    f1.put("value", Field.create("val1"));
    csvWithHeader.add(Field.create(f1));
    Map<String, Field> f2 = new HashMap<>();
    f2.put("header", Field.create("Google"));
    f2.put("value", Field.create("val2"));
    csvWithHeader.add(Field.create(f2));
    Map<String, Field> f3 = new HashMap<>();
    f3.put("header", Field.create("Hadopp"));
    f3.put("value", Field.create("val3"));
    csvWithHeader.add(Field.create(f3));
    Record record = RecordCreator.create("s", "s:1");

    record.set(Field.create(csvWithHeader));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(0, runner.getErrorRecords().size());
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();

    Assert.assertTrue(result.get("Stream").getValue() instanceof List);
    Assert.assertTrue(result.get("Google").getValue() instanceof List);
    Assert.assertTrue(result.get("Hadoop").getValue() instanceof List);

    List<Field> streamResult = result.get("Stream").getValueAsList();
    Assert.assertEquals(1, streamResult.size());
    Map<String, Field> streamResult0 = streamResult.get(0).getValueAsMap();
    Assert.assertEquals(67, streamResult0.get("score").getValue());
    Assert.assertEquals("Straem", streamResult0.get("header").getValue());
    Assert.assertEquals("val1", streamResult0.get("value").getValue());

    List<Field> googleResult = result.get("Google").getValueAsList();
    Assert.assertEquals(1, googleResult.size());
    Map<String, Field> googleResult0 = googleResult.get(0).getValueAsMap();
    Assert.assertEquals(100, googleResult0.get("score").getValue());
    Assert.assertEquals("Google", googleResult0.get("header").getValue());
    Assert.assertEquals("val2", googleResult0.get("value").getValue());

    List<Field> hadoopResult = result.get("Hadoop").getValueAsList();
    Assert.assertEquals(1, hadoopResult.size());
    Map<String, Field> hadoopResult0 = hadoopResult.get(0).getValueAsMap();
    Assert.assertEquals(84, hadoopResult0.get("score").getValue());
    Assert.assertEquals("Hadopp", hadoopResult0.get("header").getValue());
    Assert.assertEquals("val3", hadoopResult0.get("value").getValue());

  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testSnakeCaseMatchInPlace() throws Exception {
  List<String> outputFields = ImmutableList.of("sale_price", "invoice_Price", "date");

  ProcessorRunner runner = new ProcessorRunner.Builder(FuzzyFieldDProcessor.class)
      .addConfiguration("rootFieldPaths", ImmutableList.of("/"))
      .addConfiguration("outputFieldNames", outputFields)
      .addConfiguration("matchThreshold", 60)
      .addConfiguration("allCandidates", false)
      .addConfiguration("inPlace", true)
      .addConfiguration("preserveUnmatchedFields", true)
      .addOutputLane("a").build();

  runner.runInit();

  try {
    List<Field> csvWithHeader = new ArrayList<>();
    Map<String, Field> f1 = new HashMap<>();
    f1.put("header", Field.create("Inv. Price"));
    f1.put("value", Field.create("100"));
    csvWithHeader.add(Field.create(f1));
    Map<String, Field> f2 = new HashMap<>();
    f2.put("header", Field.create("Sale"));
    f2.put("value", Field.create("1000"));
    csvWithHeader.add(Field.create(f2));
    Map<String, Field> f3 = new HashMap<>();
    f3.put("header", Field.create("Sold Date"));
    f3.put("value", Field.create("2015-01-01"));
    csvWithHeader.add(Field.create(f3));
    Record record = RecordCreator.create("s", "s:1");

    record.set(Field.create(csvWithHeader));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(0, runner.getErrorRecords().size());
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();

    Assert.assertEquals("1000", result.get("sale_price").getValueAsString());
    Assert.assertEquals("100", result.get("invoice_Price").getValueAsString());
    Assert.assertEquals("2015-01-01", result.get("date").getValueAsString());
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testStringToDecimalEnglishLocale() throws StageException {
  FieldTypeConverterConfig fieldTypeConverterConfig =
      new FieldTypeConverterConfig();
  fieldTypeConverterConfig.fields = ImmutableList.of("/beginner", "/intermediate", "/advanced", "/expert",
      "/skilled", "/null");
  fieldTypeConverterConfig.targetType = Field.Type.DECIMAL;
  fieldTypeConverterConfig.dataLocale = "en";
  fieldTypeConverterConfig.scale = -1;
  fieldTypeConverterConfig.decimalScaleRoundingStrategy = DecimalScaleRoundingStrategy.ROUND_UNNECESSARY;

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
      .addConfiguration("convertBy", ConvertBy.BY_FIELD)
      .addConfiguration("fieldTypeConverterConfigs", ImmutableList.of(fieldTypeConverterConfig))
      .addOutputLane("a").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("expert", Field.create("1.234"));
    map.put("advanced", Field.create("1,234"));
    map.put("beginner", Field.create("1,234.56789"));
    map.put("intermediate", Field.create("1.234,56789"));
    map.put("skilled", Field.create("-1.23E-12"));
    map.put("null", Field.create(Field.Type.STRING, null));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof Map);
    Map<String, Field> result = field.getValueAsMap();
    Assert.assertTrue(result.size() == 6);
    Assert.assertTrue(result.containsKey("beginner"));
    Assert.assertEquals(BigDecimal.valueOf(1234.56789), result.get("beginner").getValue());
    Assert.assertEquals("9", result.get("beginner").getAttribute(HeaderAttributeConstants.ATTR_PRECISION));
    Assert.assertEquals("5", result.get("beginner").getAttribute(HeaderAttributeConstants.ATTR_SCALE));
    Assert.assertTrue(result.containsKey("intermediate"));
    Assert.assertEquals(BigDecimal.valueOf(1.234), result.get("intermediate").getValue());
    Assert.assertTrue(result.containsKey("null"));
    Assert.assertEquals(null, result.get("null").getValue());
    Assert.assertEquals(BigDecimal.valueOf(1.234), result.get("expert").getValue());
    Assert.assertTrue(result.containsKey("expert"));
    Assert.assertEquals(BigDecimal.valueOf(1234), result.get("advanced").getValue());
    Assert.assertTrue(result.containsKey("advanced"));
    Assert.assertTrue(result.containsKey("skilled"));
    Assert.assertEquals(BigDecimal.valueOf(-1.23E-12), result.get("skilled").getValue());
  } finally {
    runner.runDestroy();
  }
}