Java Code Examples for org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo#getAllStructFieldTypeInfos()

private static Schema convert(LinkedList<String> path, String name,
                              StructTypeInfo type,
                              Collection<String[]> required) {
  List<String> names = type.getAllStructFieldNames();
  List<TypeInfo> types = type.getAllStructFieldTypeInfos();
  Preconditions.checkArgument(names.size() == types.size(),
      "Cannot convert struct: %s names != %s types",
      names.size(), types.size());

  List<Schema.Field> fields = Lists.newArrayList();
  for (int i = 0; i < names.size(); i += 1) {
    path.addLast(name);
    fields.add(convertField(path, names.get(i), types.get(i), required));
    path.removeLast();
  }

  Schema recordSchema = Schema.createRecord(name, doc(type), null, false);
  recordSchema.setFields(fields);

  return recordSchema;
}
 

private Object deserializeStruct(StructTypeInfo type, Object data)
        throws SerDeException {
    if (!(data instanceof Map)) {
        throw new SerDeException("Value not of type map");
    }
    //noinspection unchecked
    Map<String, Object> map = (Map<String, Object>) data;

    List<String> fieldNames = type.getAllStructFieldNames();
    List<TypeInfo> fieldTypes = type.getAllStructFieldTypeInfos();

    // When deserializing a struct the returned value is a list of values in the same order as the field names.

    List<Object> values = Lists.newArrayListWithCapacity(fieldNames.size());
    for (int i=0; i < fieldNames.size(); i++) {
        Object rawValue = getRawValueOrNullIfAbsent(fieldNames.get(i), map);
        Object value = deserialize(fieldTypes.get(i), rawValue);
        values.add(value);
    }

    return values;
}
 

/**
 * Parses a JSON object and its fields. The Hive metadata is used to
 * determine how to parse the object fields.
 *
 * @param field
 *            - The JSON object to parse
 * @param fieldTypeInfo
 *            - Metadata about the Hive column
 * @return - A map representing the object and its fields
 */
@SuppressWarnings("unchecked")
private Object parseStruct(final Object field,
		final StructTypeInfo fieldTypeInfo) {
	final Map<Object, Object> map = (Map<Object, Object>) field;
	final ArrayList<TypeInfo> structTypes = fieldTypeInfo
			.getAllStructFieldTypeInfos();
	final ArrayList<String> structNames = fieldTypeInfo
			.getAllStructFieldNames();
	final List<Object> structRow = new ArrayList<Object>(structTypes.size());
	for (int i = 0; i < structNames.size(); i++) {
		structRow.add(parseField(map.get(structNames.get(i)),
				structTypes.get(i)));
	}
	return structRow;
}
 

public ArrayWritableObjectInspector(final StructTypeInfo rowTypeInfo) {

        typeInfo = rowTypeInfo;
        fieldNames = rowTypeInfo.getAllStructFieldNames();
        fieldInfos = rowTypeInfo.getAllStructFieldTypeInfos();
        fields = new ArrayList<StructField>(fieldNames.size());
        fieldsByName = new HashMap<String, StructFieldImpl>();

        for (int i = 0; i < fieldNames.size(); ++i) {
            final String name = fieldNames.get(i);
            final TypeInfo fieldInfo = fieldInfos.get(i);

            final StructFieldImpl field = new StructFieldImpl(name, getObjectInspector(fieldInfo), i);
            fields.add(field);
            fieldsByName.put(name, field);
        }
    }
 

private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeException {
  switch (ti.getCategory()) {
  case PRIMITIVE:
    PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
    return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
  case STRUCT:
    StructTypeInfo sti = (StructTypeInfo) ti;
    List<ObjectInspector> ois = new ArrayList<ObjectInspector>(sti.getAllStructFieldTypeInfos().size());
    for (TypeInfo typeInfo : sti.getAllStructFieldTypeInfos()) {
      ois.add(createObjectInspectorWorker(typeInfo));
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois);
  case LIST:
    ListTypeInfo lti = (ListTypeInfo) ti;
    TypeInfo listElementTypeInfo = lti.getListElementTypeInfo();
    return ObjectInspectorFactory.getStandardListObjectInspector(createObjectInspectorWorker(listElementTypeInfo));
  default:
    throw new SerDeException("No Hive categories matched for [" + ti + "]");
  }
}
 

public ArrayWritableObjectInspector(final StructTypeInfo rowTypeInfo) {

    typeInfo = rowTypeInfo;
    fieldNames = rowTypeInfo.getAllStructFieldNames();
    fieldInfos = rowTypeInfo.getAllStructFieldTypeInfos();
    fields = new ArrayList<StructField>(fieldNames.size());
    fieldsByName = new HashMap<String, StructFieldImpl>();

    for (int i = 0; i < fieldNames.size(); ++i) {
      final String name = fieldNames.get(i);
      final TypeInfo fieldInfo = fieldInfos.get(i);

      final StructFieldImpl field = new StructFieldImpl(name, getObjectInspector(fieldInfo), i);
      fields.add(field);
      fieldsByName.put(name, field);
    }
  }
 

/**
 * Convert Hive data type to a Flink data type.
 *
 * @param hiveType a Hive data type
 * @return the corresponding Flink data type
 */
public static DataType toFlinkType(TypeInfo hiveType) {
	checkNotNull(hiveType, "hiveType cannot be null");

	switch (hiveType.getCategory()) {
		case PRIMITIVE:
			return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
		case LIST:
			ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
			return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
			return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;

			List<String> names = structTypeInfo.getAllStructFieldNames();
			List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();

			DataTypes.Field[] fields = new DataTypes.Field[names.size()];

			for (int i = 0; i < fields.length; i++) {
				fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
			}

			return DataTypes.ROW(fields);
		default:
			throw new UnsupportedOperationException(
				String.format("Flink doesn't support Hive data type %s yet.", hiveType));
	}
}
 

public OrcLazyRowObjectInspector(StructTypeInfo info) {
  super(info.getAllStructFieldNames().size());
  ArrayList<String> fieldNames = info.getAllStructFieldNames();
  ArrayList<TypeInfo> fieldTypes = info.getAllStructFieldTypeInfos();
  for(int i=0; i < fieldNames.size(); ++i) {
    fields.add(new Field(fieldNames.get(i),
        OrcLazyObjectInspectorUtils.createLazyObjectInspector(fieldTypes.get(i)), i));
  }
}
 

public OrcLazyStructObjectInspector(StructTypeInfo info) {
  ArrayList<String> fieldNames = info.getAllStructFieldNames();
  ArrayList<TypeInfo> fieldTypes = info.getAllStructFieldTypeInfos();
  fields = new ArrayList<StructField>(fieldNames.size());
  for(int i=0; i < fieldNames.size(); ++i) {
    fields.add(new Field(fieldNames.get(i),
        OrcLazyObjectInspectorUtils.createWritableObjectInspector(fieldTypes.get(i)), i));
  }
}
 

PigStructInspector(StructTypeInfo info) {
    ArrayList<String> fieldNames = info.getAllStructFieldNames();
    ArrayList<TypeInfo> fieldTypes = info.getAllStructFieldTypeInfos();
    fields = new ArrayList<StructField>(fieldNames.size());
    for (int i = 0; i < fieldNames.size(); ++i) {
        fields.add(new Field(fieldNames.get(i),
                createObjectInspector(fieldTypes.get(i)), i));
    }
}
 

private static ObjectInspector getObjectInspector(TypeInfo type) {
	switch (type.getCategory()) {

		case PRIMITIVE:
			PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
			return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
		case LIST:
			ListTypeInfo listType = (ListTypeInfo) type;
			return ObjectInspectorFactory.getStandardListObjectInspector(
					getObjectInspector(listType.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapType = (MapTypeInfo) type;
			return ObjectInspectorFactory.getStandardMapObjectInspector(
					getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structType = (StructTypeInfo) type;
			List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();

			List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
			for (TypeInfo fieldType : fieldTypes) {
				fieldInspectors.add(getObjectInspector(fieldType));
			}

			return ObjectInspectorFactory.getStandardStructObjectInspector(
					structType.getAllStructFieldNames(), fieldInspectors);
		default:
			throw new CatalogException("Unsupported Hive type category " + type.getCategory());
	}
}
 

/**
 * Parses a JSON object and its fields. The Hive metadata is used to
 * determine how to parse the object fields.
 *
 * @param field         - The JSON object to parse
 * @param fieldTypeInfo - Metadata about the Hive column
 * @return - A map representing the object and its fields
 */
private Object parseStruct(Object field, StructTypeInfo fieldTypeInfo) {
	Map<Object, Object> map = (Map<Object, Object>) field;
	ArrayList<TypeInfo> structTypes = fieldTypeInfo.getAllStructFieldTypeInfos();
	ArrayList<String> structNames = fieldTypeInfo.getAllStructFieldNames();

	List<Object> structRow = new ArrayList<Object>(structTypes.size());
	if (map != null) {
		for (int i = 0; i < structNames.size(); i++) {
			structRow.add(parseField(map.get(structNames.get(i)), structTypes.get(i)));
		}
	}
	return structRow;
}
 

private Object deserializeStruct(String columnName, StructTypeInfo columnType) {
	// No equivalent Java type for the backing structure, need to recurse
	// and build a list
	ArrayList<TypeInfo> innerFieldTypes = (ArrayList<TypeInfo>) columnType
			.getAllStructFieldTypeInfos();
	List<Object> innerObjectRow = new ArrayList<Object>(
			innerFieldTypes.size());
	List<String> innerColumnNames = columnType.getAllStructFieldNames();
	rowElements.add("");
	fieldNo++;
	return workerBase(innerObjectRow, innerFieldTypes.size(),
			innerColumnNames, innerFieldTypes);

}
 

@Override
public boolean supportsHiveType(TypeInfo typeInfo) {
  try {
    switch (typeInfo.getCategory()) {
      case MAP:
        MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
        if (!mapTypeInfo.getMapKeyTypeInfo().equals(TypeInfoFactory.stringTypeInfo)) {
          return false;
        }

        TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo();
        HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(valueTypeInfo);
        return true;

      case STRUCT:
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        for (TypeInfo fieldTypeInfo : structTypeInfo.getAllStructFieldTypeInfos()) {
          HiveDynamoDBTypeFactory.getTypeObjectFromHiveType(fieldTypeInfo);
        }
        return true;

      default:
        return false;
    }
  } catch (IllegalArgumentException e) {
    return false;
  }
}
 

private static ObjectInspector getObjectInspector(TypeInfo type) {
	switch (type.getCategory()) {

		case PRIMITIVE:
			PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
			return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
		case LIST:
			ListTypeInfo listType = (ListTypeInfo) type;
			return ObjectInspectorFactory.getStandardListObjectInspector(
					getObjectInspector(listType.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapType = (MapTypeInfo) type;
			return ObjectInspectorFactory.getStandardMapObjectInspector(
					getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structType = (StructTypeInfo) type;
			List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();

			List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
			for (TypeInfo fieldType : fieldTypes) {
				fieldInspectors.add(getObjectInspector(fieldType));
			}

			return ObjectInspectorFactory.getStandardStructObjectInspector(
					structType.getAllStructFieldNames(), fieldInspectors);
		default:
			throw new CatalogException("Unsupported Hive type category " + type.getCategory());
	}
}
 

/**
 * Convert Hive data type to a Flink data type.
 *
 * @param hiveType a Hive data type
 * @return the corresponding Flink data type
 */
public static DataType toFlinkType(TypeInfo hiveType) {
	checkNotNull(hiveType, "hiveType cannot be null");

	switch (hiveType.getCategory()) {
		case PRIMITIVE:
			return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
		case LIST:
			ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
			return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
			return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;

			List<String> names = structTypeInfo.getAllStructFieldNames();
			List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();

			DataTypes.Field[] fields = new DataTypes.Field[names.size()];

			for (int i = 0; i < fields.length; i++) {
				fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
			}

			return DataTypes.ROW(fields);
		default:
			throw new UnsupportedOperationException(
				String.format("Flink doesn't support Hive data type %s yet.", hiveType));
	}
}
 

/**
 * Returns the standard java object inspector
 * 
 * @param typeInfo
 *            the type info
 * @param xmlProcessor
 *            the XML processor
 * @return the standard java object inspector
 */
public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo(TypeInfo typeInfo, XmlProcessor xmlProcessor) {
    switch (typeInfo.getCategory()) {
        case PRIMITIVE: {
            return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
        }
        case LIST: {
            ObjectInspector listElementObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo).getListElementTypeInfo(),
                xmlProcessor);
            return new XmlListObjectInspector(listElementObjectInspector, xmlProcessor);
        }
        case MAP: {
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            ObjectInspector mapKeyObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo(),
                xmlProcessor);
            ObjectInspector mapValueObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo(),
                xmlProcessor);
            return new XmlMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector, xmlProcessor);
        }
        case STRUCT: {
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            List<String> structFieldNames = structTypeInfo.getAllStructFieldNames();
            List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
            List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
            for (int fieldIndex = 0; fieldIndex < fieldTypeInfos.size(); ++fieldIndex) {
                structFieldObjectInspectors.add(getStandardJavaObjectInspectorFromTypeInfo(fieldTypeInfos.get(fieldIndex), xmlProcessor));
            }
            return getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, xmlProcessor);
        }
        default: {
            throw new IllegalStateException();
        }
    }
}
 

private static GroupType convertStructType(String name, StructTypeInfo typeInfo, Repetition repetition)
{
    List<String> columnNames = typeInfo.getAllStructFieldNames();
    List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos();
    return new GroupType(repetition, name, convertTypes(columnNames, columnTypes));
}
 

private static GroupType convertStructType(final String name, final StructTypeInfo typeInfo) {
  final List<String> columnNames = typeInfo.getAllStructFieldNames();
  final List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos();
  return new GroupType(Repetition.OPTIONAL, name, convertTypes(columnNames, columnTypes));

}
 

private static GroupType convertStructType(String name, StructTypeInfo typeInfo)
{
    List<String> columnNames = typeInfo.getAllStructFieldNames();
    List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos();
    return new GroupType(Repetition.OPTIONAL, name, convertTypes(columnNames, columnTypes));
}