Java Code Examples for com.streamsets.pipeline.api.Field#getValueAsList()

private void doUpsertList(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException{
  if (value != null && value.getType() == Field.Type.LIST) {
    List<Field> values = value.getValueAsList();
    for (Field element : values) {
      if (element != null) {
        String val = element.getValueAsString();
        pipeline.lpush(key, val);
        tempRecords.add(new ErrorRecord(record, "List", key, val));
      }
    }
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), "value should be List");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be List"
        )
    );
  }
}
 

private void doUpsertSet(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException {
  if (value != null && value.getType() == Field.Type.LIST) {
    List<Field> values = value.getValueAsList();
    for (Field element : values) {
      if (element != null) {
        String val = element.getValueAsString();
        pipeline.sadd(key, val);
        tempRecords.add(new ErrorRecord(record, "Set", key, val));
      }
    }
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), "value should be List");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be List"
        )
    );
  }
}
 

private Map<String, Field> flattenRootField(Field rootField) throws StageException {
  Map<String, Field> fields;
  switch (rootField.getType()) {
    case LIST:
      // Flatten CSV input types.
      List<Field> fieldList = rootField.getValueAsList();
      fields = new HashMap<>();
      for (Field field : fieldList) {
        Map<String, Field> fieldWithHeader = field.getValueAsMap();
        if (!fieldWithHeader.containsKey(HEADER) || !fieldWithHeader.containsKey(VALUE)) {
          throw new StageException(Errors.FUZZY_00, rootField.getType());
        }
        fields.put(fieldWithHeader.get(HEADER).getValueAsString(), fieldWithHeader.get(VALUE));
      }
      break;
    case MAP:
      fields = rootField.getValueAsMap();
      break;
    default:
      throw new StageException(Errors.FUZZY_00, rootField.getType());
  }
  return fields;
}
 

@Test
public void testCreateListPrimitiveField() throws Exception {
  String schema = "{\"type\": \"array\", \"items\": \"string\"}";
  Schema avroSchema = new Schema.Parser().parse(schema);
  Record record = RecordCreator.create();
  Field field = AvroTypeUtil.avroToSdcField(record, avroSchema, Arrays.asList("Hari", "Kiran"), false);

  Assert.assertEquals(Field.Type.LIST, field.getType());
  List<Field> valueAsList = field.getValueAsList();
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(0).getType());
  Assert.assertEquals("Hari", valueAsList.get(0).getValueAsString());
  Assert.assertEquals(Field.Type.STRING, valueAsList.get(1).getType());
  Assert.assertEquals("Kiran", valueAsList.get(1).getValueAsString());

  record.set(field);
  Object avroObject = AvroTypeUtil.sdcRecordToAvro(record, avroSchema, new HashMap<String, Object>());
  Assert.assertTrue(avroObject instanceof List<?>);

  List<String> listString = (List<String>) avroObject;
  Assert.assertEquals("Hari", listString.get(0));
  Assert.assertEquals("Kiran", listString.get(1));

  //Check invalid type - String to List
  makeBadType(Field.create("notList"), record, avroSchema);
}
 

private static int getDelimitedIndex(String header, int startIndex) {
  int index = -1;
  Record record = getRecordInContext();
  if (record != null) {
    Field root = record.get();
    if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
      List<Field> list = root.getValueAsList();
      for (int i= startIndex; index == -1 && i < list.size(); i++) {
        Field element = list.get(i);
        if (element.getType() == Field.Type.MAP && element.getValue() != null) {
          Map<String, Field> map = element.getValueAsMap();
          if (map.containsKey("header")) {
            Field headerField = map.get("header");
            if (headerField.getType() == Field.Type.STRING && headerField.getValue() != null) {
              if (headerField.getValueAsString().equals(header)) {
                index = i;
              }
            }
          }
        }
      }
    }
  }
  return index;
}
 

@Test
public void testParsingMultipleValuesAsList() throws Exception {
  XmlParserConfig configs = new XmlParserConfig();
  configs.charset = "UTF-8";
  configs.fieldPathToParse = "/xml";
  configs.parsedFieldPath = "/data";
  configs.removeCtrlChars = false;
  configs.xmlRecordElement = "a";
  configs.multipleValuesBehavior = MultipleValuesBehavior.ALL_AS_LIST;

  XmlParserDProcessor processor = new XmlParserDProcessor();
  processor.configs = configs;

  ProcessorRunner runner = new ProcessorRunner.Builder(XmlParserDProcessor.class, processor)
      .addOutputLane("out").setOnRecordError(OnRecordError.TO_ERROR).build();
  Map<String, Field> map = new HashMap<>();
  map.put("xml", Field.create("<root><a>1</a><a>2</a><a>3</a></root>"));
  Record record = RecordCreator.create();
  record.set(Field.create(map));
  List<Record> input = new ArrayList<>();
  input.add(record);
  try {
    runner.runInit();
    StageRunner.Output output = runner.runProcess(input);
    Assert.assertTrue(output.getRecords().containsKey("out"));
    final List<Record> records = output.getRecords().get("out");
    Assert.assertEquals(1, records.size());
    final Field outputField = records.get(0).get("/data");
    Assert.assertNotNull(outputField);
    final List<Field> valueList = outputField.getValueAsList();
    Assert.assertNotNull(valueList);
    Assert.assertEquals(3, valueList.size());
    Assert.assertEquals("1", valueList.get(0).getValueAsMap().get("value").getValueAsString());
    Assert.assertEquals("2", valueList.get(1).getValueAsMap().get("value").getValueAsString());
    Assert.assertEquals("3", valueList.get(2).getValueAsMap().get("value").getValueAsString());

  } finally {
    runner.runDestroy();
  }
}
 

private void walkAndCheckField(String fieldName, Field expected, Field actual) {
  Assert.assertEquals(expected.getType(), actual.getType());
  Field.Type type = expected.getType();
  switch (type) {
    case MAP:
    case LIST_MAP:
      Map<String, Field> expectedFieldMap = expected.getValueAsMap();
      Map<String, Field> actualFieldMap = actual.getValueAsMap();
      Assert.assertTrue(getMismatchMessage(fieldName, type, "Expected Field has different keys from Actual Field"), Sets.difference(expectedFieldMap.keySet(), actualFieldMap.keySet()).isEmpty());
      Assert.assertTrue(getMismatchMessage(fieldName, type, "Actual Field has different keys from Expected Field"), Sets.difference(actualFieldMap.keySet(), expectedFieldMap.keySet()).isEmpty());
      for (String key : expectedFieldMap.keySet()) {
        walkAndCheckField(fieldName + "/" + key, expectedFieldMap.get(key), actualFieldMap.get(key));
      }
      break;
    case LIST:
      List<Field> expectedFieldList = expected.getValueAsList();
      List<Field> actualFieldList = actual.getValueAsList();
      Assert.assertEquals(getMismatchMessage(fieldName, type, "Size mistmatch"), expectedFieldList.size(), actualFieldList.size());
      for (int i=0; i < expectedFieldList.size(); i++) {
        walkAndCheckField(fieldName + "[" + i + "]", expectedFieldList.get(i), actualFieldList.get(i));
      }
      break;
    case BYTE_ARRAY:
      Assert.assertArrayEquals(getMismatchMessage(fieldName, type, "Byte Array value mismatch"), expected.getValueAsByteArray(), actual.getValueAsByteArray());
      break;
    default:
      Assert.assertEquals(getMismatchMessage(fieldName, type, "Field Value mismatch"),expected.getValue(), actual.getValue());
  }
}
 

private Field processByType(String matchingPath, Field rootField) throws StageException {
  switch (rootField.getType()) {
    case MAP:
    case LIST_MAP:
      if(rootField.getValue() == null) {
        return rootField;
      }
      for (Map.Entry<String, Field> entry : rootField.getValueAsMap().entrySet()) {
        entry.setValue(processByType(matchingPath + "/" + entry.getKey(), entry.getValue()));
      }
      break;
    case LIST:
      if(rootField.getValue() == null) {
        return rootField;
      }
      List<Field> fields = rootField.getValueAsList();
      for(int i = 0; i < fields.size(); i++) {
        fields.set(i, processByType(matchingPath + "[" + i + "]", fields.get(i)));
      }
      break;
    default:
      for(WholeTypeConverterConfig converterConfig : wholeTypeConverterConfigs) {
        if(converterConfig.sourceType == rootField.getType()) {
          rootField = convertField(matchingPath, rootField, converterConfig);
        }
      }
  }

  // Return original field
  return rootField;
}
 

private void visitFieldsInternal(
    RecordFieldImpl recordField,
    FieldVisitor visitor,
    String name,
    String path,
    Field currentField,
    Field parentField
) throws StageException {
  // For nested types, visit their children first
  switch (currentField.getType()) {
    case MAP:
    case LIST_MAP:
      for(Map.Entry<String, Field> entry : currentField.getValueAsMap().entrySet()) {
        visitFieldsInternal(
            recordField,
            visitor,
            entry.getKey(),
            path + "/" + escapeName(entry.getKey(), true),
            entry.getValue(),
            currentField
        );
      }
      break;
    case LIST:
      int index = 0;
      for(Field childField : currentField.getValueAsList()) {
        visitFieldsInternal(recordField, visitor, name, path + "[" + index + "]", childField, currentField);
        index++;
      }
      break;
    default:
  }

  // And always visit this field itself (whether it's terminal type of nested type doesn't matter)
  recordField.path = path;
  recordField.name = name;
  recordField.field = currentField;
  recordField.parentField = parentField;
  visitor.visit(recordField);
}
 

@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "dValueAt",
    description = "Returns the value of the specified header name")
public static String getDelimitedValueAt(
    @ElParam("index") int index) {
  String value = null;
  if (index >= 0) {
    Record record = getRecordInContext();
    if (record != null) {
      Field root = record.get();
      if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
        List<Field> list = root.getValueAsList();
        if (index < list.size()) {
          Field element = list.get(index);
          if (element.getType() == Field.Type.MAP && element.getValue() != null) {
            Map<String, Field> map = element.getValueAsMap();
            if (map.containsKey("value")) {
              Field valueField = map.get("value");
              if (valueField.getType() == Field.Type.STRING && valueField.getValue() != null) {
                value = valueField.getValueAsString();
              }
            }
          }
        }
      }
    }
  }
  return value;
}
 

@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "dValue",
    description = "Returns the value of the specified header name")
public static String getDelimitedValue(
    @ElParam("header") String header) {
  String value = null;
  Record record = getRecordInContext();
  if (record != null) {
    Field root = record.get();
    if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
      List<Field> list = root.getValueAsList();
      for (Field element : list) {
        if (element.getType() == Field.Type.MAP && element.getValue() != null) {
          Map<String, Field> map = element.getValueAsMap();
          if (map.containsKey("header")) {
            Field headerField = map.get("header");
            if (headerField.getType() == Field.Type.STRING && headerField.getValue() != null) {
              if (headerField.getValueAsString().equals(header)) {
                if (map.containsKey("value")) {
                  Field valueField = map.get("value");
                  if (valueField.getType() == Field.Type.STRING && valueField.getValue() != null) {
                    value = valueField.getValueAsString();
                    break;
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  return value;
}
 

private static void checkField(String fieldPath, Field expectedField, Field actualField) {
  String errorString = String.format("Error in Field Path: %s", fieldPath);
  Assert.assertEquals(errorString, expectedField.getType(), actualField.getType());
  errorString = errorString + " of type: " + expectedField.getType().name();
  switch (expectedField.getType()) {
    case MAP:
    case LIST_MAP:
      Map<String, Field> expectedFieldMap = expectedField.getValueAsMap();
      Map<String, Field> actualFieldMap = actualField.getValueAsMap();
      Assert.assertEquals(errorString, expectedFieldMap.keySet().size(), actualFieldMap.keySet().size());
      for (Map.Entry<String, Field> entry : actualFieldMap.entrySet()) {
        String actualFieldName = entry.getKey().toLowerCase();
        Assert.assertNotNull(errorString, expectedFieldMap.get(actualFieldName));
        checkField(fieldPath + "/" + actualFieldName, expectedFieldMap.get(actualFieldName), entry.getValue());
      }
      break;
    case LIST:
      List<Field> expectedFieldList = expectedField.getValueAsList();
      List<Field> actualFieldList = actualField.getValueAsList();
      Assert.assertEquals(errorString, expectedFieldList.size(), actualFieldList.size());
      for (int i = 0; i < expectedFieldList.size(); i++) {
        checkField(fieldPath + "[" + i + "]", expectedFieldList.get(i), actualFieldList.get(i));
      }
      break;
    case BYTE_ARRAY:
      Assert.assertArrayEquals(errorString, expectedField.getValueAsByteArray(), actualField.getValueAsByteArray());
      break;
    case ZONED_DATETIME:
      // H2 seems to throw away millis by default, so just make sure they are within 1 second
      final long millisDifference = expectedField.getValueAsZonedDateTime().toInstant().toEpochMilli()
          - actualField.getValueAsZonedDateTime().toInstant().toEpochMilli();
      assertThat(errorString, Math.abs(millisDifference), lessThanOrEqualTo(1000l));
      break;
    default:
      Assert.assertEquals(errorString, expectedField.getValue(), actualField.getValue());
  }
}
 

/**
 * Map mode
 */
private void writeFieldToDocumentMap(Document doc, Field field, String name) throws IOException {
  if (field.getValue() == null) {
    // On the Map mode just set the null on the document using the name.
    doc.setNull(name);
  } else {
    switch (field.getType()) {
      case FILE_REF:
        throw new IOException("Cannot serialize FileRef fields.");
      case MAP:
      case LIST_MAP:
        Document newDoc = loader.createNewEmptyDocument();
        Map<String, Field> map = field.getValueAsMap();
        for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
          String fieldName = fieldEntry.getKey();
          Field newField = fieldEntry.getValue();
          // recursive call in map mode.
          writeFieldToDocumentMap(newDoc, newField, fieldName);
        }
        // Map the new doc
        doc.set(name, newDoc);
        break;
      case LIST:
        List<Field> listOfFields = field.getValueAsList();
        List<Object> objsList = new ArrayList<>();
        for (Field f : listOfFields) {
          // recursive call in a list mode.
          writeFieldToDocumentList(f, objsList);
        }
        doc.setArray(name, objsList.toArray());
        break;
      case BOOLEAN:
        doc.set(name, field.getValueAsBoolean());
        break;
      case CHAR:
        doc.set(name, String.valueOf(field.getValueAsChar()));
        break;
      case BYTE:
        doc.set(name, new byte[]{field.getValueAsByte()});
        break;
      case SHORT:
        doc.set(name, field.getValueAsShort());
        break;
      case INTEGER:
        doc.set(name, field.getValueAsInteger());
        break;
      case LONG:
        doc.set(name, field.getValueAsLong());
        break;
      case FLOAT:
        doc.set(name, field.getValueAsFloat());
        break;
      case DOUBLE:
        doc.set(name, field.getValueAsDouble());
        break;
      case DATE:
        doc.set(name, field.getValueAsDate().getTime());
        break;
      case DATETIME:
        doc.set(name, field.getValueAsDatetime().getTime());
        break;
      case TIME:
        doc.set(name, field.getValueAsTime().getTime());
        break;
      case DECIMAL:
        doc.set(name, field.getValueAsDecimal());
        break;
      case STRING:
      case ZONED_DATETIME:
        doc.set(name, field.getValueAsString());
        break;
      case BYTE_ARRAY:
        doc.set(name, field.getValueAsByteArray());
        break;
      default:
        throw new IllegalStateException(String.format("Unrecognized field type (%s) in field: %s",
            field.getType().name(),
            field.toString()
        ));
    }
  }
}
 

private void writeFieldToJsonObject(Field field) throws IOException {
  if (field == null || field.getValue() == null) {
    generator.writeNull();
    return;
  }
  switch (field.getType()) {
    case FILE_REF:
      throw new IOException("Cannot serialize FileRef fields.");
    case MAP:
    case LIST_MAP:
      generator.writeStartObject();
      Map<String, Field> map = field.getValueAsMap();
      for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
        generator.writeFieldName(fieldEntry.getKey());
        writeFieldToJsonObject(fieldEntry.getValue());
      }
      generator.writeEndObject();
      break;
    case LIST:
      generator.writeStartArray();
      List<Field> list = field.getValueAsList();
      for (Field f : list) {
        writeFieldToJsonObject(f);
      }
      generator.writeEndArray();
      break;
    case BOOLEAN:
      generator.writeBoolean(field.getValueAsBoolean());
      break;
    case CHAR:
      generator.writeString(String.valueOf(field.getValueAsChar()));
      break;
    case BYTE:
      generator.writeBinary(new byte[] {field.getValueAsByte()});
      break;
    case SHORT:
      generator.writeNumber(field.getValueAsShort());
      break;
    case INTEGER:
      generator.writeNumber(field.getValueAsInteger());
      break;
    case LONG:
      generator.writeNumber(field.getValueAsLong());
      break;
    case FLOAT:
      generator.writeNumber(field.getValueAsFloat());
      break;
    case DOUBLE:
      generator.writeNumber(field.getValueAsDouble());
      break;
    case DATE:
      generator.writeNumber(field.getValueAsDate().getTime());
      break;
    case DATETIME:
      generator.writeNumber(field.getValueAsDatetime().getTime());
      break;
    case TIME:
      generator.writeNumber(field.getValueAsTime().getTime());
      break;
    case DECIMAL:
      generator.writeNumber(field.getValueAsDecimal());
      break;
    case STRING:
      generator.writeString(field.getValueAsString());
      break;
    case BYTE_ARRAY:
      generator.writeBinary(field.getValueAsByteArray());
      break;
    case ZONED_DATETIME:
      generator.writeString(field.getValueAsString());
      break;
    default:
      throw new IllegalStateException(String.format(
          "Unrecognized field type (%s) in field: %s",
          field.getType().name(),
          field.toString())
      );
  }
}
 

/**
 * List mode
 */
private void writeFieldToDocumentList(Field field, List list) throws IOException {
  if (field.getValue() == null) {
    list.add(null);
  } else {
    switch (field.getType()) {
      case FILE_REF:
        throw new IOException("Cannot serialize FileRef fields.");
      case MAP:
      case LIST_MAP:
        Document newDoc = loader.createNewEmptyDocument();
        Map<String, Field> map = field.getValueAsMap();
        for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
          String fieldName = fieldEntry.getKey();
          Field newField = fieldEntry.getValue();
          // recursive call in map mode.
          writeFieldToDocumentMap(newDoc, newField, fieldName);
        }
        // List mode
        list.add(newDoc);
        break;
      case LIST:
        List<Field> listOfFields = field.getValueAsList();
        List<Object> objsList = new ArrayList<>();
        for (Field f : listOfFields) {
          // recursive call in a list mode.
          writeFieldToDocumentList(f, objsList);
        }
        list.add(objsList);
        break;
      case BOOLEAN:
        list.add(field.getValueAsBoolean());
        break;
      case CHAR:
        list.add(String.valueOf(field.getValueAsChar()));
        break;
      case BYTE:
        list.add(new byte[]{field.getValueAsByte()});
        break;
      case SHORT:
        list.add(field.getValueAsShort());
        break;
      case INTEGER:
        list.add(field.getValueAsInteger());
        break;
      case LONG:
        list.add(field.getValueAsLong());
        break;
      case FLOAT:
        list.add(field.getValueAsFloat());
        break;
      case DOUBLE:
        list.add(field.getValueAsDouble());
        break;
      case DATE:
        list.add(field.getValueAsDate().getTime());
        break;
      case DATETIME:
        list.add(field.getValueAsDatetime().getTime());
        break;
      case TIME:
        list.add(field.getValueAsTime().getTime());
        break;
      case DECIMAL:
        list.add(field.getValueAsDecimal());
        break;
      case STRING:
      case ZONED_DATETIME:
        list.add(field.getValueAsString());
        break;
      case BYTE_ARRAY:
        list.add(field.getValueAsByteArray());
        break;
      default:
        throw new IllegalStateException(String.format(
            "Unrecognized field type (%s) in field: %s",
            field.getType().name(),
            field.toString())
        );
    }
  }
}
 

public List<Field> getChanges() {
  //Will return a 0...n length Field.LIST
  Field entry  = getEntryFromField("change");
  return entry.getValueAsList();
}
 

private static void handleRepeatedField(
    Record record,
    Field field,
    String fieldPath,
    Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
    Map<String, Object> defaultValueMap,
    Descriptors.FieldDescriptor f,
    DynamicMessage.Builder builder
) throws DataGeneratorException {
  List<Object> toReturn = new ArrayList<>();
  List<Field> valueAsList = field.getValueAsList();
  if(valueAsList != null) {
    // According to proto 2 and 3 language guide repeated fields can have 0 elements.
    // Also null is treated as empty in case of json mappings so I guess we can ignore if it is null.
    for (int i = 0; i < valueAsList.size(); i++) {
      if (f.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) {
        // repeated field of type message
        toReturn.add(
          sdcFieldToProtobufMsg(
            record,
            valueAsList.get(i),
            fieldPath + FORWARD_SLASH + f.getName() + "[" + i + "]",
            f.getMessageType(),
            messageTypeToExtensionMap,
            defaultValueMap
          )
        );
      } else {
        // repeated field of primitive types
        toReturn.add(
          getValue(
            f,
            valueAsList.get(i),
            record,
            fieldPath + FORWARD_SLASH + f.getName(),
            messageTypeToExtensionMap,
            defaultValueMap
          )
        );
      }
    }
  }
  builder.setField(f, toReturn);
}
 

@Test
public void testEmptyMapAndEmptyList() throws StageException {
  ExpressionProcessorConfig expressionProcessorConfig1 = new ExpressionProcessorConfig();
  expressionProcessorConfig1.expression = "${emptyMap()}";
  expressionProcessorConfig1.fieldToSet = "/d";

  ExpressionProcessorConfig expressionProcessorConfig2 = new ExpressionProcessorConfig();
  expressionProcessorConfig2.expression = "${emptyList()}";
  expressionProcessorConfig2.fieldToSet = "/d/e";

  ExpressionProcessorConfig expressionProcessorConfig3 = new ExpressionProcessorConfig();
  expressionProcessorConfig3.expression = "${emptyMap()}";
  expressionProcessorConfig3.fieldToSet = "/d/e[0]";

  ExpressionProcessorConfig expressionProcessorConfig4 = new ExpressionProcessorConfig();
  expressionProcessorConfig4.expression = "${record:value('/a')}";
  expressionProcessorConfig4.fieldToSet = "/d/e[0]/f";

  ProcessorRunner runner = new ProcessorRunner.Builder(ExpressionDProcessor.class)
    .addConfiguration("expressionProcessorConfigs", ImmutableList.of(
      expressionProcessorConfig1, expressionProcessorConfig2, expressionProcessorConfig3, expressionProcessorConfig4))
    .addOutputLane("lane").build();
  runner.runInit();

  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("a", Field.create("A"));
    map.put("b", Field.create("B"));
    map.put("c", Field.create("C"));
    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("lane").size());
    Field fieldD = output.getRecords().get("lane").get(0).get("/d");

    Assert.assertTrue(fieldD.getValue() instanceof Map);
    Map<String, Field> result = fieldD.getValueAsMap();
    Field fieldE = result.get("e");
    Assert.assertTrue(fieldE.getValue() instanceof List);
    List<Field> listField = fieldE.getValueAsList();
    Assert.assertTrue(listField.get(0).getValue() instanceof Map);
    Map<String, Field> fieldMap = listField.get(0).getValueAsMap();
    Assert.assertTrue(fieldMap.get("f").getValue() instanceof String);
    Assert.assertEquals(fieldMap.get("f").getValueAsString(), "A");
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testListOfList() throws StageException {
  WholeTypeConverterConfig converterConfig = new WholeTypeConverterConfig();
  converterConfig.sourceType = Field.Type.STRING;
  converterConfig.targetType = Field.Type.BOOLEAN;
  converterConfig.dataLocale = "en";

  ProcessorRunner runner = new ProcessorRunner.Builder(FieldTypeConverterDProcessor.class)
    .addConfiguration("convertBy", ConvertBy.BY_TYPE)
    .addConfiguration("wholeTypeConverterConfigs", ImmutableList.of(converterConfig))
    .addOutputLane("a").build();
  runner.runInit();

  try {
    List<Field> innerList = new LinkedList<>();
    innerList.add(Field.create("true"));
    innerList.add(Field.create("yes"));
    innerList.add(Field.create("122345566"));
    innerList.add(Field.create(Field.Type.STRING, null));
    innerList.add(Field.create(Field.Type.INTEGER, 10));
    innerList.add(Field.create(Field.Type.DECIMAL, BigDecimal.valueOf(1)));
    Field innerField = Field.create(innerList);

    List<Field> outerList = new LinkedList<>();
    outerList.add(innerField);

    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(outerList));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Field field = output.getRecords().get("a").get(0).get();
    Assert.assertTrue(field.getValue() instanceof List);
    List<Field> result = field.getValueAsList();
    Assert.assertTrue(result.size() == 1);

    field = result.get(0);

    Assert.assertTrue(field.getValue() instanceof List);
    result = field.getValueAsList();
    Assert.assertTrue(result.size() == 6);

    Assert.assertEquals(true, result.get(0).getValue());

    Assert.assertEquals(false, result.get(1).getValue());

    Assert.assertEquals(false, result.get(2).getValue());

    Assert.assertEquals(Field.Type.BOOLEAN, result.get(3).getType());
    Assert.assertEquals(null, result.get(3).getValue());

    Assert.assertEquals(10, result.get(4).getValue());

    Assert.assertEquals(BigDecimal.valueOf(1), result.get(5).getValue());
  } finally {
    runner.runDestroy();
  }
}
 

@SuppressWarnings("unchecked")
private static <T> void extractInnerMapFromTheList(
    Record metadataRecord,
    String listFieldName,
    String innerPairFirstFieldName,
    String innerPairSecondFieldName,
    boolean isSecondFieldHiveType,
    LinkedHashMap<String, T> returnValMap,
    HiveStageCheckedException exception
) throws HiveStageCheckedException{
  boolean throwException = false;
  try {
    if (metadataRecord.has(SEP + listFieldName)) {
      Field columnField = metadataRecord.get(SEP + listFieldName);
      List<Field> columnList = columnField.getValueAsList();
      if (columnList != null) {
        for (Field listElementField : columnList) {
          if (listElementField.getType() != Field.Type.MAP && listElementField.getType() != Field.Type.LIST_MAP) {
            throwException = true;
            break;
          }
          LinkedHashMap<String, Field> innerPair = listElementField.getValueAsListMap();
          String innerPairFirstField = innerPair.get(innerPairFirstFieldName).getValueAsString();
          T retVal;
          if (isSecondFieldHiveType) {
            Field hiveTypeInfoField = innerPair.get(innerPairSecondFieldName);
            HiveType hiveType = HiveType.getHiveTypeFromString(
                hiveTypeInfoField.getValueAsMap().get(HiveMetastoreUtil.TYPE).getValueAsString()
            );
            retVal = (T) (hiveType.getSupport().generateHiveTypeInfoFromMetadataField(hiveTypeInfoField));
          } else {
            retVal = (T) innerPair.get(innerPairSecondFieldName).getValueAsString();
          }
          returnValMap.put(innerPairFirstField, retVal);
        }
      }
    } else {
      // we allow partition to be empty for non-partitioned table
      if (!listFieldName.equals(PARTITION_FIELD))
        throwException = true;
    }
  } catch(Exception e) {
    LOG.error("Can't parse metadata record", e);
    throwException = true;
    exception.initCause(e);
  }
  if (throwException) {
    throw exception;
  }
}