org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo Java Examples

private Object worker(String columnName, TypeInfo columnType){
	
	switch(columnType.getCategory()) {
		
		case STRUCT:
			return deserializeStruct(columnName, (StructTypeInfo) columnType);
		case UNION:
			return deserializeUnion(columnName,(UnionTypeInfo) columnType);
		case LIST:
		return deserializeList(columnName, (ListTypeInfo) columnType);
		case MAP:
		throw new RuntimeException("map type is not possible for cobol layout" + columnType.getCategory());
		case PRIMITIVE:
		return deserializePrimitive(columnName, (PrimitiveTypeInfo) columnType);
		default:
		throw new RuntimeException("Unknown TypeInfo: " + columnType.getCategory());
	}
}
 

public MDSMapObjectInspector( final MapTypeInfo typeInfo ){
  TypeInfo keyTypeInfo = typeInfo.getMapKeyTypeInfo();
  if( keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && ( (PrimitiveTypeInfo)keyTypeInfo ).getPrimitiveCategory() == PrimitiveCategory.STRING ){
    keyObjectInspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
  }
  else{
    throw new RuntimeException( "Map key type is string only." );
  }

  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getMapValueTypeInfo() ); 

  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getMapValueTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 

public UnionField( final UnionTypeInfo typeInfo ){
  byte tag = (byte)0;
  
  for( TypeInfo childTypeInfo : typeInfo.getAllUnionObjectTypeInfos() ){
    IGetUnionObject field = craeteGetUnionObject( tag , childTypeInfo );
    columnTypeMap.put( field.getColumnType() , field );
    tag++;
  }
}
 

public OrcUnionObjectInspector(UnionTypeInfo info) {
  List<TypeInfo> unionChildren = info.getAllUnionObjectTypeInfos();
  this.children = new ArrayList<ObjectInspector>(unionChildren.size());
  for(TypeInfo child: info.getAllUnionObjectTypeInfos()) {
    this.children.add(OrcLazyObjectInspectorUtils.createWritableObjectInspector(child));
  }
}
 

public OrcLazyUnionObjectInspector(UnionTypeInfo info) {
  List<TypeInfo> unionChildren = info.getAllUnionObjectTypeInfos();
  this.children = new ArrayList<ObjectInspector>(unionChildren.size());
  for(TypeInfo child: info.getAllUnionObjectTypeInfos()) {
    this.children.add(OrcLazyObjectInspectorUtils.createWritableObjectInspector(child));
  }
}
 

private Object deserializeUnion(UnionTypeInfo type, Object data)
        throws SerDeException {
    // Try each union type in order until one matches.
    for (byte i=0; i < type.getAllUnionObjectTypeInfos().size(); i++) {
        try {
            Object value = deserialize(type.getAllUnionObjectTypeInfos().get(i), data);
            return new StandardUnionObjectInspector.StandardUnion(i, value);
        } catch (SerDeException e) {
            // Skip it and try the next
        }
    }

    throw new SerDeException("No suitable type found");
}
 

/**
 * Deserializes a raw value to the provided type.
 */
private Object deserialize(TypeInfo type, Object rawValue)
        throws SerDeException {
    Object value = null;

    if (rawValue != null) {
        switch (type.getCategory()) {
            case PRIMITIVE:
                value = deserializePrimitive((PrimitiveTypeInfo) type, rawValue);
                break;
            case STRUCT:
                value = deserializeStruct((StructTypeInfo) type, rawValue);
                break;
            case MAP:
                value = deserializeMap((MapTypeInfo) type, rawValue);
                break;
            case LIST:
                value = deserializeList((ListTypeInfo) type, rawValue);
                break;
            case UNION:
                value = deserializeUnion((UnionTypeInfo) type, rawValue);
                break;
        }
    }

    return value;
}
 

public MDSListObjectInspector( final ListTypeInfo typeInfo ){
  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getListElementTypeInfo() ); 
  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getListElementTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 

public MDSUnionStructField( final int fieldId , final String fieldName , final StandardUnionObjectInspector inspector , final UnionTypeInfo unionTypeInfo ){
  this.fieldId = fieldId;
  this.fieldName = fieldName;
  this.inspector = inspector;
  childColumnAndIndex = new ColumnAndIndex();
  childColumnAndIndex.columnIndex = -1;
  unionField = new UnionField( unionTypeInfo );
}
 

public static boolean isSupportedType(TypeInfo typeInfo, StorageFormat storageFormat)
{
    switch (typeInfo.getCategory()) {
        case PRIMITIVE:
            return getPrimitiveType((PrimitiveTypeInfo) typeInfo) != null;
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            return isSupportedType(mapTypeInfo.getMapKeyTypeInfo(), storageFormat) && isSupportedType(mapTypeInfo.getMapValueTypeInfo(), storageFormat);
        case LIST:
            ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
            return isSupportedType(listTypeInfo.getListElementTypeInfo(), storageFormat);
        case STRUCT:
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            return structTypeInfo.getAllStructFieldTypeInfos().stream()
                    .allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
        case UNION:
            // This feature (reading uniontypes as structs) has only been verified against Avro and ORC tables. Here's a discussion:
            //   1. Avro tables are supported and verified.
            //   2. ORC tables are supported and verified.
            //   3. The Parquet format doesn't support uniontypes itself so there's no need to add support for it in Presto.
            //   4. TODO: RCFile tables are not supported yet.
            //   5. TODO: The support for Avro is done in SerDeUtils so it's possible that formats other than Avro are also supported. But verification is needed.
            if (storageFormat.getSerDe().equalsIgnoreCase(AVRO.getSerDe()) || storageFormat.getSerDe().equalsIgnoreCase(ORC.getSerDe())) {
                UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
                return unionTypeInfo.getAllUnionObjectTypeInfos().stream()
                        .allMatch(fieldTypeInfo -> isSupportedType(fieldTypeInfo, storageFormat));
            }
    }
    return false;
}
 

public UnionGetField( final UnionTypeInfo unionTypeInfo ){
  unionField = new UnionField( unionTypeInfo );
}
 

public UnionGetField( final UnionTypeInfo unionTypeInfo ){
  unionField = new UnionField( unionTypeInfo );
}
 

public static ObjectInspector craeteObjectInspectorFromTypeInfo( final TypeInfo typeInfo ){
  switch ( typeInfo.getCategory() ){
    case STRUCT:
      return new MDSStructObjectInspector( (StructTypeInfo)typeInfo );
    case MAP:
      return new MDSMapObjectInspector( (MapTypeInfo)typeInfo );
    case LIST:
      return new MDSListObjectInspector( (ListTypeInfo)typeInfo );
    case UNION:
      UnionTypeInfo unionTypeInfo = (UnionTypeInfo)typeInfo;
      List<ObjectInspector> unionList = new ArrayList<ObjectInspector>();
      for( TypeInfo childTypeInfo : unionTypeInfo.getAllUnionObjectTypeInfos() ){
        unionList.add( craeteObjectInspectorFromTypeInfo( childTypeInfo ) );
      }
      return ObjectInspectorFactory.getStandardUnionObjectInspector( unionList );
    case PRIMITIVE:
      PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo)typeInfo;
      switch( primitiveTypeInfo.getPrimitiveCategory() ){
        case STRING:
          return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
        case BINARY:
          return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
        case BOOLEAN:
          return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
        case BYTE:
          return PrimitiveObjectInspectorFactory.writableByteObjectInspector;
        case DOUBLE:
          return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
        case FLOAT:
          return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
        case INT:
          return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
        case LONG:
          return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
        case SHORT:
          return PrimitiveObjectInspectorFactory.writableShortObjectInspector;

        case DATE:
        case DECIMAL:
        case TIMESTAMP:
        case VOID:
        default:
        throw new UnsupportedOperationException( "Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory() );
      }
    default:
      throw new UnsupportedOperationException( "Unknown category " + typeInfo.getCategory() );
  }
}
 

private Object deserializeUnion(String columnName,UnionTypeInfo columnType) throws RuntimeException {
	return null;
}
 

public static ObjectInspector createLazyObjectInspector(TypeInfo info) {
  switch (info.getCategory()) {
    case PRIMITIVE:
      switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
        case FLOAT:
          return FLOAT_OBJECT_INSPECTOR;
        case DOUBLE:
          return DOUBLE_OBJECT_INSPECTOR;
        case BOOLEAN:
          return BOOLEAN_OBJECT_INSPECTOR;
        case BYTE:
          return BYTE_OBJECT_INSPECTOR;
        case SHORT:
          return SHORT_OBJECT_INSPECTOR;
        case INT:
          return INT_OBJECT_INSPECTOR;
        case LONG:
          return LONG_OBJECT_INSPECTOR;
        case BINARY:
          return BINARY_OBJECT_INSPECTOR;
        case STRING:
          return STRING_OBJECT_INSPECTOR;
        case TIMESTAMP:
          return TIMESTAMP_OBJECT_INSPECTOR;
        default:
          throw new IllegalArgumentException("Unknown primitive type " +
            ((PrimitiveTypeInfo) info).getPrimitiveCategory());
      }
    case STRUCT:
      return new OrcLazyStructObjectInspector((StructTypeInfo) info);
    case UNION:
      return new OrcLazyUnionObjectInspector((UnionTypeInfo) info);
    case MAP:
      return new OrcLazyMapObjectInspector((MapTypeInfo) info);
    case LIST:
      return new OrcLazyListObjectInspector((ListTypeInfo) info);
    default:
      throw new IllegalArgumentException("Unknown type " +
        info.getCategory());
  }
}
 

public static ObjectInspector createWritableObjectInspector(TypeInfo info) {
  switch (info.getCategory()) {
    case PRIMITIVE:
      switch (((PrimitiveTypeInfo) info).getPrimitiveCategory()) {
        case FLOAT:
          return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
        case DOUBLE:
          return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
        case BOOLEAN:
          return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
        case BYTE:
          return PrimitiveObjectInspectorFactory.writableByteObjectInspector;
        case SHORT:
          return PrimitiveObjectInspectorFactory.writableShortObjectInspector;
        case INT:
          return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
        case LONG:
          return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
        case BINARY:
          return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
        case STRING:
          return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
        case TIMESTAMP:
          return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
        default:
          throw new IllegalArgumentException("Unknown primitive type " +
            ((PrimitiveTypeInfo) info).getPrimitiveCategory());
      }
    case STRUCT:
      return new OrcStruct.OrcStructInspector((StructTypeInfo) info);
    case UNION:
      return new OrcUnion.OrcUnionObjectInspector((UnionTypeInfo) info);
    case MAP:
      return new OrcStruct.OrcMapObjectInspector((MapTypeInfo) info);
    case LIST:
      return new OrcStruct.OrcListObjectInspector((ListTypeInfo) info);
    default:
      throw new IllegalArgumentException("Unknown type " +
        info.getCategory());
  }
}
 

@VisibleForTesting
static Schema convert(LinkedList<String> path, String name,
                      TypeInfo type, Collection<String[]> required) {
  switch (type.getCategory()) {
    case PRIMITIVE:
      if (type.getClass() == charClass || type.getClass() == varcharClass) {
        // this is required because type name includes length
        return Schema.create(Schema.Type.STRING);
      }

      String typeInfoName = type.getTypeName();
      Preconditions.checkArgument(TYPEINFO_TO_TYPE.containsKey(typeInfoName),
          "Cannot convert unsupported type: %s", typeInfoName);
      return Schema.create(TYPEINFO_TO_TYPE.get(typeInfoName));

    case LIST:
      return Schema.createArray(optional(convert(path, name,
          ((ListTypeInfo) type).getListElementTypeInfo(), required)));

    case MAP:
      MapTypeInfo mapType = (MapTypeInfo) type;
      Preconditions.checkArgument(
          "string".equals(mapType.getMapKeyTypeInfo().toString()),
          "Non-String map key type: %s", mapType.getMapKeyTypeInfo());

      return Schema.createMap(optional(convert(path, name,
          mapType.getMapValueTypeInfo(), required)));

    case STRUCT:
      return convert(path, name, (StructTypeInfo) type, required);

    case UNION:
      List<TypeInfo> unionTypes = ((UnionTypeInfo) type)
          .getAllUnionObjectTypeInfos();

      // add NULL so all union types are optional
      List<Schema> types = Lists.newArrayList(NULL);
      for (int i = 0; i < unionTypes.size(); i += 1) {
        // types within unions cannot be required
        types.add(convert(
            path, name + "_" + i, unionTypes.get(i), NO_REQUIRED_FIELDS));
      }

      return Schema.createUnion(types);

    default:
      throw new IllegalArgumentException(
          "Unknown TypeInfo category: " + type.getCategory());
  }
}