Java Code Examples for com.streamsets.pipeline.api.Field#setAttribute()

@Test
public void testGetNamespaces() throws IOException {
  Writer writer = Mockito.mock(Writer.class);
  XmlCharDataGenerator generator = new XmlCharDataGenerator(writer, true, ImmutableList.of(SCHEMA), true);

  Map<String, String> existingNamespaces = ImmutableMap.of("ns1", "NS1");

  Field field = Field.create(0);
  field.setAttribute("xmlns:ns1", "fNS1");
  field.setAttribute("xmlns:ns2", "NS2");
  field.setAttribute("a", "A");

  Map<String, String> expected = ImmutableMap.of("ns1", "fNS1", "ns2", "NS2");

  Map<String, String> got = generator.getNamespaces(existingNamespaces, field);

  Assert.assertEquals(expected, got);
}
 

@NotNull
private Record makeListRecord(Map<String, Pair<Field.Type, Object>> fieldMap) {
  Record record = RecordCreator.create();
  Record.Header header = record.getHeader();
  ArrayList<Field> fields = new ArrayList<>();
  for (Map.Entry<String, Pair<Field.Type, Object>> entry : fieldMap.entrySet()) {
    String fieldName = entry.getKey();
    Field.Type fieldType = entry.getValue().getLeft();
    Field field = Field.create(fieldType, entry.getValue().getRight());
    if (fieldType == Field.Type.DECIMAL) {
      field.setAttribute(HeaderAttributeConstants.ATTR_SCALE, SCALE);
      field.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, PRECISION);
    }
    fields.add(field);
  }
  record.set(Field.create(fields));
  header.setAttribute("table", tableName);
  return record;
}
 

@Test
public void testFieldToXmlElementsFieldWithAdditionalNamespace() throws IOException, DataGeneratorException {
  Writer writer = Mockito.mock(Writer.class);
  XmlCharDataGenerator generator = new XmlCharDataGenerator(writer, true, ImmutableList.of(SCHEMA), true);
  Document document = generator.getDocumentBuilder().newDocument();

  String name = "nsa:element";
  Map<String, String> namespaces = ImmutableMap.of("", "", "ns", "NS");
  Field field = Field.create("data");
  field.setAttribute("xmlns:nsa", "NSA");
  List<Element> elements = generator.fieldToXmlElements(document, namespaces, name, field);
  Assert.assertEquals(1, elements.size());
  Element element = elements.get(0);
  Assert.assertEquals("nsa:element", element.getNodeName());
  Assert.assertEquals("NSA", element.getNamespaceURI());
  Assert.assertEquals(1, element.getChildNodes().getLength());
  Assert.assertEquals("data", element.getTextContent());
}
 

@Test
public void testRecordToXmlDocumentWithNS() throws IOException, DataGeneratorException {
  Writer writer = Mockito.mock(Writer.class);
  XmlCharDataGenerator generator = new XmlCharDataGenerator(writer, true, ImmutableList.of(SCHEMA), true);

  Record record = RecordCreator.create();
  Field root = Field.create(ImmutableMap.of("root", Field.create("data")));
  root.setAttribute("xmlns", "NS");
  record.set(root);

  Document document = generator.recordToXmlDocument(record);

  Assert.assertNotNull(document);
  Element element = document.getDocumentElement();
  Assert.assertEquals("root", element.getNodeName());
  Assert.assertEquals("NS", element.getNamespaceURI());
  Assert.assertEquals(1, element.getChildNodes().getLength());
  Assert.assertEquals("data", element.getTextContent());
}
 

@NotNull
private Record makeRecord(Map<String, Pair<Field.Type, Object>> fieldMap) {
  Record record = RecordCreator.create();
  Record.Header header = record.getHeader();
  LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
  for (Map.Entry<String, Pair<Field.Type, Object>> entry : fieldMap.entrySet()) {
    String fieldName = entry.getKey();
    Field.Type fieldType = entry.getValue().getLeft();
    Field field = Field.create(fieldType, entry.getValue().getRight());
    if (fieldType == Field.Type.DECIMAL) {
      field.setAttribute(HeaderAttributeConstants.ATTR_SCALE, SCALE);
      field.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, PRECISION);
    }
    fields.put(fieldName, field);
  }
  record.set(Field.create(fields));
  header.setAttribute("table", tableName);
  return record;
}
 

@Test
public void testScaleGreaterThanPrecision() throws Exception {
  JdbcMetadataDProcessor processor = getProcessor(h2ConnectionString);

  ProcessorRunner processorRunner = getProcessorRunner(processor);

  processorRunner.runInit();

  Record record = makeRecord(fieldMap2);
  Field field = record.get("/column5");
  field.setAttribute("precision", SCALE);
  field.setAttribute("scale", PRECISION);
  List<Record> recordList = ImmutableList.of(record);

  StageRunner.Output output = processorRunner.runProcess(recordList);
  Assert.assertEquals(0, output.getRecords().get("lane").size());

  Assert.assertEquals(1, processorRunner.getErrorRecords().size());
  Record errorRecord = processorRunner.getErrorRecords().get(0);
  Assert.assertEquals(JdbcErrors.JDBC_306.name(), errorRecord.getHeader().getErrorCode());
}
 

@NotNull
private Record makeRecord(Map<String, Pair<Field.Type, Object>> fieldMap) {
  Record record = RecordCreator.create();
  Record.Header header = record.getHeader();
  LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
  for (Map.Entry<String, Pair<Field.Type, Object>> entry : fieldMap.entrySet()) {
    String fieldName = entry.getKey();
    Field.Type fieldType = entry.getValue().getLeft();
    Field field = Field.create(fieldType, entry.getValue().getRight());
    if (fieldType == Field.Type.DECIMAL) {
      field.setAttribute(HeaderAttributeConstants.ATTR_SCALE, SCALE);
      field.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, PRECISION);
    }
    fields.put(fieldName, field);
  }
  record.set(Field.create(fields));
  header.setAttribute("table", tableName);
  return record;
}
 

@NotNull
private Record makeListRecord(Map<String, Pair<Field.Type, Object>> fieldMap) {
  Record record = RecordCreator.create();
  Record.Header header = record.getHeader();
  ArrayList<Field> fields = new ArrayList<>();
  for (Map.Entry<String, Pair<Field.Type, Object>> entry : fieldMap.entrySet()) {
    String fieldName = entry.getKey();
    Field.Type fieldType = entry.getValue().getLeft();
    Field field = Field.create(fieldType, entry.getValue().getRight());
    if (fieldType == Field.Type.DECIMAL) {
      field.setAttribute(HeaderAttributeConstants.ATTR_SCALE, SCALE);
      field.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, PRECISION);
    }
    fields.add(field);
  }
  record.set(Field.create(fields));
  header.setAttribute("table", tableName);
  return record;
}
 

public Field fromPrimitiveField(com.google.cloud.bigquery.Field field, FieldValue value) {
  Field.Type type = asRecordFieldType(field);
  Field f;
  if (value.isNull()) {
    f = Field.create(type, null);
  } else if (TEMPORAL_TYPES.contains(type)) {
    if (field.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) {
      // in general only TIMESTAMP should be a Unix epoch value.  However, to be certain will test for getTimeStampValue() and process as as string format if that fails
      f = Field.create(type, value.getTimestampValue() / 1000L); // micro to milli
    } else {
      // not a timestamp value.  Assume it's a date string format
      // Other BigQuery temporal types come as string representations.
      try {
        String dval = value.getStringValue();
        if (dval != null) {
          dval = stripMicrosec.matcher(dval).replaceAll("$1"); // strip out microseconds, for milli precision
        }
        f = Field.create(type, dateTimeFormatter.apply(field).parse(dval));
      } catch (ParseException ex) {
        // In case of failed date/time parsing, simply allow the value to proceed as a string
        LOG.error(String.format("Unable to convert BigQuery field type %s to field type %s", field.getType().toString(), type.toString()), ex);
        f = Field.create(Field.Type.STRING, value.getStringValue()); // allow it to proceed with a null value
        f.setAttribute("bq.parseException", String.format("%s using format %s", ex.getMessage(), dateTimeFormatter.apply(field).toPattern()) );
      }
    }
    if (f != null) f.setAttribute("bq.fullValue", value.getStringValue()); // add the parse error as a field header
  } else if (type == Field.Type.BYTE_ARRAY) {
    f = Field.create(type, value.getBytesValue());
  } else {
    f = Field.create(type, value.getValue());
  }
  return f;
}
 

@Test
public void testWrapRecordWithAttributes() throws IOException {
  List<Record> records = new ArrayList<>();
  RecordImpl r = new RecordImpl("stage", "source", null, null);
  Map<String, Field> fields = new HashMap<>();
  final Field first = Field.create(2);
  first.setAttribute("attr1-1", "one");
  first.setAttribute("attr1-2", "two");
  fields.put("first", first);
  final Field second = Field.create("second_value");
  second.setAttribute("attr2-1", "three");
  second.setAttribute("attr2-2", "four");
  fields.put("second", second);
  Field root = Field.create(fields);
  r.set(root);
  records.add(r);

  List<RecordJson> recordJsonList = BeanHelper.wrapRecords(records);
  System.out.println(ObjectMapperFactory.get().writeValueAsString(recordJsonList));
}
 

@Test
public void testScaleGreaterThanPrecision() throws Exception {
  JdbcMetadataDProcessor processor = getProcessor(h2ConnectionString);

  ProcessorRunner processorRunner = getProcessorRunner(processor);

  processorRunner.runInit();

  Record record = makeRecord(fieldMap2);
  Field field = record.get("/column5");
  field.setAttribute("precision", SCALE);
  field.setAttribute("scale", PRECISION);
  List<Record> recordList = ImmutableList.of(record);

  StageRunner.Output output = processorRunner.runProcess(recordList);
  Assert.assertEquals(0, output.getRecords().get("lane").size());

  Assert.assertEquals(1, processorRunner.getErrorRecords().size());
  Record errorRecord = processorRunner.getErrorRecords().get(0);
  Assert.assertEquals(JdbcErrors.JDBC_306.name(), errorRecord.getHeader().getErrorCode());
}
 

@Test
public void testGenerateDecimal() throws OnRecordErrorException {
  Field decimal = Field.create(Field.Type.DECIMAL, new BigDecimal("10.2"));
  decimal.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, "3");
  decimal.setAttribute(HeaderAttributeConstants.ATTR_SCALE, "1");

  Record record = RecordCreator.create();
  record.set(Field.create(Field.Type.LIST_MAP, ImmutableMap.of(
    "decimal", decimal
  )));

  generateAndValidateSchema(
    record,
    "{\"name\":\"decimal\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":3,\"scale\":1}}"
  );
}
 

@Override
public BaseFieldBuilder<? extends BaseFieldBuilder> end(String... attributes) {
  if (parentBuilder == null) {
    throw new IllegalStateException("Do not call end on the root builder; just call build when finished");
  }
  final Field field = Field.create(Field.Type.MAP, this.builder.build());
  for (Map.Entry<String, String> attr : buildAttributeMap(attributes).entrySet()) {
    field.setAttribute(attr.getKey(), attr.getValue());
  }
  parentBuilder.handleEndChildField(this.field, field);
  return parentBuilder;
}
 

public static <C extends Processor> void verifyAccessToSdcRecord(
    Class<C> clazz,
    Processor processor
) throws StageException {
  ProcessorRunner runner = new ProcessorRunner.Builder(clazz, processor)
      .addOutputLane("lane")
      .build();

  Record record = RecordCreator.create();
  Map<String, Field> map = new HashMap<>();
  Field value = Field.create("value");
  value.setAttribute("attr", "is-here");
  map.put("value", value);
  record.set(Field.create(map));

  runner.runInit();
  StageRunner.Output output;
  try{
    output = runner.runProcess(Collections.singletonList(record));
  } finally {
    runner.runDestroy();
  }
  List<Record> records = output.getRecords().get("lane");
  assertEquals(1, records.size());

  Record outputRecord = records.get(0);

  // Header attribute
  assertEquals("is-here", outputRecord.getHeader().getAttribute("attr"));
}
 

@Override
public BaseFieldBuilder<? extends BaseFieldBuilder> end(String... attributes) {
  final Field field = Field.create(Field.Type.LIST, fields);
  for (Map.Entry<String, String> attr : buildAttributeMap(attributes).entrySet()) {
    field.setAttribute(attr.getKey(), attr.getValue());
  }
  parentBuilder.handleEndChildField(this.field, field);
  return parentBuilder;
}
 

@Test
public void testCreateElement() throws DataGeneratorException, IOException {
  Writer writer = Mockito.mock(Writer.class);
  XmlCharDataGenerator generator = new XmlCharDataGenerator(writer, true, ImmutableList.of(SCHEMA), true);
  Document document = generator.getDocumentBuilder().newDocument();

  String name = "element";
  String namespace = "";
  Map<String, String> namespaces = ImmutableMap.of("", "", "ns", "NS");
  Field field = Field.create("data");

  Element element = generator.createElement(document, namespaces, name, namespace, field);
  Assert.assertEquals("element", element.getNodeName());
  Assert.assertEquals(null, element.getPrefix());
  Assert.assertEquals(null, element.getNamespaceURI());
  Assert.assertEquals(0, element.getChildNodes().getLength());
  Assert.assertEquals(0, element.getAttributes().getLength());

  name = "ns:element";
  namespace = "NS";
  element = generator.createElement(document, namespaces, name, namespace, field);
  Assert.assertEquals("ns:element", element.getNodeName());
  Assert.assertEquals("ns", element.getPrefix());
  Assert.assertEquals("NS", element.getNamespaceURI());
  Assert.assertEquals(0, element.getChildNodes().getLength());
  Assert.assertEquals(0, element.getAttributes().getLength());

  field.setAttribute("a", "A");
  field.setAttribute("ns:b", "B");
  field.setAttribute("xmlns", "NS1");
  field.setAttribute("xmlns:ns2", "NS2");
  element = generator.createElement(document, namespaces, name, namespace, field);
  Assert.assertEquals(2, element.getAttributes().getLength());
  Assert.assertEquals("a", element.getAttributes().item(0).getNodeName());
  Assert.assertEquals(null, element.getAttributes().item(0).getNamespaceURI());
  Assert.assertEquals("A", element.getAttributes().item(0).getNodeValue());
  Assert.assertEquals("ns:b", element.getAttributes().item(1).getNodeName());
  Assert.assertEquals("NS", element.getAttributes().item(1).getNamespaceURI());
  Assert.assertEquals("B", element.getAttributes().item(1).getNodeValue());
}
 

@Test
public void testGetAttributes() throws IOException {
  Writer writer = Mockito.mock(Writer.class);
  XmlCharDataGenerator generator = new XmlCharDataGenerator(writer, true, ImmutableList.of(SCHEMA), true);

  Field field = Field.create(0);
  Assert.assertEquals(Collections.emptyMap(), generator.getAttributes(field));

  field.setAttribute("xmlns", "uri");
  field.setAttribute("xmlns:foo", "uri:foo");
  Assert.assertEquals(Collections.emptyMap(), generator.getAttributes(field));

  field.setAttribute("a", "A");
  Assert.assertEquals(ImmutableMap.of("a", "A"), generator.getAttributes(field));
}
 

private Field convertStringToTargetType(
    Field field,
    Field.Type targetType,
    Locale dataLocale,
    String dateMask,
    int scale,
    DecimalScaleRoundingStrategy decimalScaleRoundingStrategy,
    DateTimeFormatter dateTimeFormatter
) throws ParseException {
  String stringValue = field.getValueAsString();
  switch(targetType) {
    case BOOLEAN:
      return Field.create(Field.Type.BOOLEAN, Boolean.valueOf(stringValue.trim()));
    case BYTE:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue).byteValue());
    case BYTE_ARRAY:
      return Field.create(stringValue.getBytes(StandardCharsets.UTF_8));
    case CHAR:
      return Field.create(stringValue.charAt(0));
    case DATE:
      java.text.DateFormat dateFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createDate(dateFormat.parse(stringValue.trim()));
    case DATETIME:
      java.text.DateFormat dateTimeFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createDatetime(dateTimeFormat.parse(stringValue.trim()));
    case TIME:
      java.text.DateFormat timeFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createTime(timeFormat.parse(stringValue.trim()));
    case ZONED_DATETIME:
      return Field.createZonedDateTime(ZonedDateTime.parse(stringValue.trim(), dateTimeFormatter));
    case DECIMAL:
      NumberFormat decimalFormat = NumberFormat.getInstance(dataLocale);
      DecimalFormat df = (DecimalFormat) decimalFormat;
      df.setParseBigDecimal(true);
      Number decimal = df.parse(stringValue.trim());
      BigDecimal bigDecimal = adjustScaleIfNeededForDecimalConversion(new BigDecimal(decimal.toString()), scale, decimalScaleRoundingStrategy);
      Field decimalField = Field.create(Field.Type.DECIMAL, bigDecimal);
      decimalField.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, String.valueOf(bigDecimal.precision()));
      decimalField.setAttribute(HeaderAttributeConstants.ATTR_SCALE, String.valueOf(bigDecimal.scale()));
      return decimalField;
    case DOUBLE:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).doubleValue());
    case FLOAT:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).floatValue());
    case INTEGER:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).intValue());
    case LONG:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).longValue());
    case SHORT:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).shortValue());
    case FILE_REF:
      throw new IllegalArgumentException(Utils.format("Cannot convert String value to type {}", targetType));
    default:
      return field;
  }
}
 

@Test
public void testFieldAttributeFunctions() throws Exception {
  ELEvaluator eval = new ELEvaluator("testFieldAttributeFunctions", elDefinitionExtractor, RecordEL.class);

  ELVariables vars = new ELVariables();

  Map<String, Field> fieldAMap = new HashMap<>();
  Field aFirstField = Field.create("1");
  aFirstField.setAttribute("attr1", "attrVal1");
  aFirstField.setAttribute("attr2", "attrVal2");
  fieldAMap.put("first", aFirstField);

  Field aSecondField = Field.create("2");
  aSecondField.setAttribute("attr3", "attrVal3");
  fieldAMap.put("second", aSecondField);

  Map<String, Field> rootFields = new HashMap<>();
  Field aField = Field.create(fieldAMap);
  aField.setAttribute("attr10", "attrVal10");
  rootFields.put("A", aField);

  Field bField = Field.create("b_value");
  bField.setAttribute("attr20", "attrVal20");
  bField.setAttribute("attr21", "attrVal21");
  rootFields.put("B", bField);

  Record record = Mockito.mock(Record.class);
  Mockito.when(record.get()).thenReturn(Field.create(rootFields));
  Mockito.when(record.get("/A")).thenReturn(aField);
  Mockito.when(record.get("/A/first")).thenReturn(aFirstField);
  Mockito.when(record.get("/A/second")).thenReturn(aSecondField);
  Mockito.when(record.get("/B")).thenReturn(bField);

  RecordEL.setRecordInContext(vars, record);

  assertEquals("attrVal1", eval.eval(vars, "${record:fieldAttribute('/A/first', 'attr1')}", String.class));
  assertEquals("attrVal2", eval.eval(vars, "${record:fieldAttribute('/A/first', 'attr2')}", String.class));
  assertEquals("", eval.eval(vars, "${record:fieldAttribute('/A/first', 'attr3')}", String.class));
  assertEquals(
      "default",
      eval.eval(vars, "${record:fieldAttributeOrDefault('/A/first', 'attr3', 'default')}",
          String.class)
  );
  assertEquals(
      "attrVal3",
      eval.eval(vars, "${record:fieldAttributeOrDefault('/A/second', 'attr3', 'default')}",
          String.class)
  );
  assertEquals("attrVal10", eval.eval(vars, "${record:fieldAttribute('/A', 'attr10')}", String.class));
  assertEquals("attrVal20", eval.eval(vars, "${record:fieldAttribute('/B', 'attr20')}", String.class));
  assertEquals("attrVal21", eval.eval(vars, "${record:fieldAttribute('/B', 'attr21')}", String.class));
  assertEquals(
      "default2",
      eval.eval(vars, "${record:fieldAttributeOrDefault('/XYZ', 'attr21', 'default2')}",
          String.class)
  );
}
 

private void testRecordWriterReader(RecordEncoding encoding) throws IOException {
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  RecordWriter writer = RecordWriterReaderFactory.createRecordWriter(encoding, os);

  RecordImpl record1 = new RecordImpl("stage", "source", new byte[] { 0, 1, 2}, "mode");
  record1.getHeader().setStagesPath("stagePath");
  record1.getHeader().setTrackingId("trackingId");
  Map<String, Field> map = new HashMap<>();
  final Field fieldA = Field.create(new BigDecimal(1));
  fieldA.setAttribute("a_attr1", "value1");
  fieldA.setAttribute("a_attr2", "value2");
  fieldA.setAttribute("a_attr3", "value3");
  map.put("a", fieldA);
  map.put("b", Field.create("Hello"));
  map.put("c", Field.create(new ArrayList<Field>()));
  record1.set(Field.create(map));
  writer.write(record1);

  RecordImpl record2 = new RecordImpl("stage2", "source2", null, null);
  record2.getHeader().setStagesPath("stagePath2");
  record2.getHeader().setTrackingId("trackingId2");
  record2.set(Field.create("Hello"));
  writer.write(record2);

  RecordImpl record3 = new RecordImpl("stage", "source", new byte[] { 0, 1, 2}, "mode");
  record3.getHeader().setStagesPath("stagePath3");
  record3.getHeader().setTrackingId("trackingId3");
  LinkedHashMap<String, Field> listMap = new LinkedHashMap<>();
  listMap.put("a", Field.create(new BigDecimal(1)));
  listMap.put("b", Field.create("Hello"));
  listMap.put("c", Field.create(new ArrayList<Field>()));

  //Test with special characters in listMap key to test JIRA SDC-1562
  final Field fooSpaceBarField = Field.create("foo space bar");
  fooSpaceBarField.setAttribute("fooSpaceBar_attr1", "fooBar1");
  listMap.put("foo bar", fooSpaceBarField);
  listMap.put("foo[bar", Field.create("foo open bracket bar"));
  listMap.put("foo]bar", Field.create("foo close bracket bar"));
  listMap.put("foo'bar", Field.create("foo single quote bar"));
  listMap.put("foo\"bar", Field.create("foo double quote bar"));
  listMap.put("foo -+^&*()#$@!~ bar", Field.create("foo special character bar"));
  listMap.put("foo/bar", Field.create("foo slash bar"));
  listMap.put("f/oo/'ba/\'r", Field.create("foo slash quote bar"));

  //nested listMap
  LinkedHashMap<String, Field> nestedListMap = new LinkedHashMap<>();
  nestedListMap.put("foo bar", Field.create("foo space bar"));
  nestedListMap.put("foo[bar", Field.create("foo open bracket bar"));
  nestedListMap.put("foo]bar", Field.create("foo close bracket bar"));
  nestedListMap.put("foo'bar", Field.create("foo single quote bar"));
  nestedListMap.put("foo\"bar", Field.create("foo double quote bar"));
  nestedListMap.put("foo -+^&*()#$@!~ bar", Field.create("foo special character bar"));
  final Field fooSlashBarField = Field.create("foo slash bar");
  fooSlashBarField.setAttribute("fooSlashBar_attr1", "fooSlashBar1");
  fooSlashBarField.setAttribute("fooSlashBar_attr2", "fooSlashBar2");
  nestedListMap.put("foo/bar", fooSlashBarField);

  listMap.put("nestedListMap", Field.createListMap(nestedListMap));
  record3.set(Field.createListMap(listMap));
  writer.write(record3);

  writer.close();
  byte[] bytes = os.toByteArray();
  Assert.assertEquals(encoding.getMagicNumber(), bytes[0]);
  InputStream is = new ByteArrayInputStream(bytes);
  RecordReader reader = RecordWriterReaderFactory.createRecordReader(is, 0, 1000);
  Assert.assertEquals(record1, reader.readRecord());
  Assert.assertEquals(record2, reader.readRecord());
  Assert.assertEquals(record3, reader.readRecord());
  Assert.assertNull(reader.readRecord());
  reader.close();
}