Java Code Examples for org.apache.parquet.schema.Type#asGroupType()

The following examples show how to use org.apache.parquet.schema.Type#asGroupType() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AvroRecordConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                          Schema avroSchema, GenericData model) {
  super(parent);
  GroupType parquetGroup = parquetSchema.asGroupType();
  this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

  int parquetIndex = 0;
  for (int index = 0; index < avroSchema.getTypes().size(); index++) {
    Schema memberSchema = avroSchema.getTypes().get(index);
    if (!memberSchema.getType().equals(Schema.Type.NULL)) {
      Type memberType = parquetGroup.getType(parquetIndex);
      memberConverters[parquetIndex] = newConverter(memberSchema, memberType, model, new ParentValueContainer() {
        @Override
        public void add(Object value) {
          Preconditions.checkArgument(
              AvroUnionConverter.this.memberValue == null,
              "Union is resolving to more than one type");
          memberValue = value;
        }
      });
      parquetIndex++; // Note for nulls the parquetIndex id not increased
    }
  }
}
 
Example 2
Source File: AvroIndexedRecordConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                          Schema avroSchema, GenericData model) {
  this.parent = parent;
  GroupType parquetGroup = parquetSchema.asGroupType();
  this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

  int parquetIndex = 0;
  for (int index = 0; index < avroSchema.getTypes().size(); index++) {
    Schema memberSchema = avroSchema.getTypes().get(index);
    if (!memberSchema.getType().equals(Schema.Type.NULL)) {
      Type memberType = parquetGroup.getType(parquetIndex);
      memberConverters[parquetIndex] = newConverter(memberSchema, memberType, model, new ParentValueContainer() {
        @Override
        public void add(Object value) {
          Preconditions.checkArgument(memberValue==null, "Union is resolving to more than one type");
          memberValue = value;
        }
      });
      parquetIndex++; // Note for nulls the parquetIndex id not increased
    }
  }
}
 
Example 3
Source File: AvroRecordConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public AvroArrayConverter(ParentValueContainer parent, GroupType type,
                          Schema avroSchema, GenericData model,
                          Class<?> arrayClass) {
  this.parent = parent;
  this.avroSchema = avroSchema;

  Preconditions.checkArgument(arrayClass.isArray(),
      "Cannot convert non-array: " + arrayClass.getName());
  this.elementClass = arrayClass.getComponentType();

  ParentValueContainer setter = createSetterAndContainer();
  Schema elementSchema = this.avroSchema.getElementType();
  Type repeatedType = type.getType(0);

  // always determine whether the repeated type is the element type by
  // matching it against the element schema.
  if (isElementType(repeatedType, elementSchema)) {
    // the element type is the repeated type (and required)
    converter = newConverter(elementSchema, repeatedType, model, elementClass, setter);
  } else {
    // the element is wrapped in a synthetic group and may be optional
    converter = new ArrayElementConverter(
        repeatedType.asGroupType(), elementSchema, model, setter);
  }
}
 
Example 4
Source File: ParquetRecordWriter.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
/**
 * Changes the list inner '$data$' vector name to 'element' in the schema
 */
private Type renameChildTypeToElement(Type childType) {
  if (childType.isPrimitive()) {
    PrimitiveType childPrimitiveType = childType.asPrimitiveType();
    return new PrimitiveType(childType.getRepetition(),
      childPrimitiveType.getPrimitiveTypeName(),
      childPrimitiveType.getTypeLength(),
      "element",
      childPrimitiveType.getOriginalType(),
      childPrimitiveType.getDecimalMetadata(),
      childPrimitiveType.getId());
  } else {
    GroupType childGroupType = childType.asGroupType();
    Type.ID id = childGroupType.getId();
    GroupType groupType = new GroupType(childType.getRepetition(),
      "element",
      childType.getOriginalType(),
      childGroupType.getFields());
    if (id != null) {
      groupType = groupType.withId(id.hashCode());
    }
    return groupType;
  }
}
 
Example 5
Source File: AvroRecordConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public AvroCollectionConverter(ParentValueContainer parent, GroupType type,
                               Schema avroSchema, GenericData model,
                               Class<?> containerClass) {
  this.parent = parent;
  this.avroSchema = avroSchema;
  this.containerClass = containerClass;
  Schema elementSchema = AvroSchemaConverter.getNonNull(avroSchema.getElementType());
  Type repeatedType = type.getType(0);
  // always determine whether the repeated type is the element type by
  // matching it against the element schema.
  if (isElementType(repeatedType, elementSchema)) {
    // the element type is the repeated type (and required)
    converter = newConverter(elementSchema, repeatedType, model, new ParentValueContainer() {
      @Override
      @SuppressWarnings("unchecked")
      public void add(Object value) {
        container.add(value);
      }
    });
  } else {
    // the element is wrapped in a synthetic group and may be optional
    converter = new ElementConverter(repeatedType.asGroupType(), elementSchema, model);
  }
}
 
Example 6
Source File: SimpleGroupConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
SimpleGroupConverter(SimpleGroupConverter parent, int index, GroupType schema) {
  this.parent = parent;
  this.index = index;

  converters = new Converter[schema.getFieldCount()];

  for (int i = 0; i < converters.length; i++) {
    final Type type = schema.getType(i);
    if (type.isPrimitive()) {
      converters[i] = new SimplePrimitiveConverter(this, i);
    } else {
      converters[i] = new SimpleGroupConverter(this, i, type.asGroupType());
    }

  }
}
 
Example 7
Source File: ThriftRecordConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private Converter newConverter(List<TProtocol> events, Type type, ThriftField field) {
  switch (field.getType().getType()) {
  case LIST:
    return new ListConverter(events, type.asGroupType(), field);
  case SET:
    return new SetConverter(events, type.asGroupType(), field);
  case MAP:
    return new MapConverter(events, type.asGroupType(), field);
  case STRUCT:
    return new StructConverter(events, type.asGroupType(), field);
  case STRING:
    return new FieldStringConverter(events, field);
  case ENUM:
    return new FieldEnumConverter(events, field);
  default:
    return new FieldPrimitiveConverter(events, field);
  }
}
 
Example 8
Source File: HiveGroupConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
protected static Converter getConverterFromDescription(final Type type, final int index,
    final HiveGroupConverter parent) {
  if (type == null) {
    return null;
  }
  if (type.isPrimitive()) {
    return ETypeConverter.getNewConverter(type.asPrimitiveType().getPrimitiveTypeName().javaType,
        index, parent);
  } else {
    if (type.asGroupType().getRepetition() == Repetition.REPEATED) {
      return new ArrayWritableGroupConverter(type.asGroupType(), parent, index);
    } else {
      return new DataWritableGroupConverter(type.asGroupType(), parent, index);
    }
  }
}
 
Example 9
Source File: ProtoMessageConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private Converter newScalarConverter(ParentValueContainer pvc, Message.Builder parentBuilder, Descriptors.FieldDescriptor fieldDescriptor, Type parquetType) {

    JavaType javaType = fieldDescriptor.getJavaType();

    switch (javaType) {
      case STRING: return new ProtoStringConverter(pvc);
      case FLOAT: return new ProtoFloatConverter(pvc);
      case DOUBLE: return new ProtoDoubleConverter(pvc);
      case BOOLEAN: return new ProtoBooleanConverter(pvc);
      case BYTE_STRING: return new ProtoBinaryConverter(pvc);
      case ENUM: return new ProtoEnumConverter(pvc, fieldDescriptor);
      case INT: return new ProtoIntConverter(pvc);
      case LONG: return new ProtoLongConverter(pvc);
      case MESSAGE: {
        Message.Builder subBuilder = parentBuilder.newBuilderForField(fieldDescriptor);
        return new ProtoMessageConverter(pvc, subBuilder, parquetType.asGroupType());
      }
    }

    throw new UnsupportedOperationException(String.format("Cannot convert type: %s" +
            " (Parquet type: %s) ", javaType, parquetType));
  }
 
Example 10
Source File: ProtoMessageConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public ListConverter(Message.Builder parentBuilder, Descriptors.FieldDescriptor fieldDescriptor, Type parquetType) {
  LogicalTypeAnnotation logicalTypeAnnotation = parquetType.getLogicalTypeAnnotation();
  if (!(logicalTypeAnnotation instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) || parquetType.isPrimitive()) {
    throw new ParquetDecodingException("Expected LIST wrapper. Found: " + logicalTypeAnnotation + " instead.");
  }

  GroupType rootWrapperType = parquetType.asGroupType();
  if (!rootWrapperType.containsField("list") || rootWrapperType.getType("list").isPrimitive()) {
    throw new ParquetDecodingException("Expected repeated 'list' group inside LIST wrapperr but got: " + rootWrapperType);
  }

  GroupType listType = rootWrapperType.getType("list").asGroupType();
  if (!listType.containsField("element")) {
    throw new ParquetDecodingException("Expected 'element' inside repeated list group but got: " + listType);
  }

  Type elementType = listType.getType("element");
  converter = newMessageConverter(parentBuilder, fieldDescriptor, elementType);
}
 
Example 11
Source File: ProtoWriteSupport.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private GroupType getGroupType(Type type) {
  LogicalTypeAnnotation logicalTypeAnnotation = type.getLogicalTypeAnnotation();
  if (logicalTypeAnnotation == null) {
    return type.asGroupType();
  }
  return logicalTypeAnnotation.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<GroupType>() {
    @Override
    public Optional<GroupType> visit(LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) {
      return ofNullable(type.asGroupType().getType("list").asGroupType().getType("element").asGroupType());
    }

    @Override
    public Optional<GroupType> visit(LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) {
      return ofNullable(type.asGroupType().getType("key_value").asGroupType().getType("value").asGroupType());
    }
  }).orElse(type.asGroupType());
}
 
Example 12
Source File: AvroIndexedRecordConverter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public AvroArrayConverter(ParentValueContainer parent, GroupType type,
    Schema avroSchema, GenericData model) {
  this.parent = parent;
  this.avroSchema = avroSchema;
  Schema elementSchema = AvroSchemaConverter
      .getNonNull(avroSchema.getElementType());
  Type repeatedType = type.getType(0);
  // always determine whether the repeated type is the element type by
  // matching it against the element schema.
  if (AvroRecordConverter.isElementType(repeatedType, elementSchema)) {
    // the element type is the repeated type (and required)
    converter = newConverter(elementSchema, repeatedType, model, new ParentValueContainer() {
      @Override
      @SuppressWarnings("unchecked")
      public void add(Object value) {
        array.add(value);
      }
    });
  } else {
    // the element is wrapped in a synthetic group and may be optional
    converter = new ElementConverter(repeatedType.asGroupType(), elementSchema, model);
  }
}
 
Example 13
Source File: ParquetTypeVisitor.java    From iceberg with Apache License 2.0 5 votes vote down vote up
public static <T> T visit(Type type, ParquetTypeVisitor<T> visitor) {
  if (type instanceof MessageType) {
    return visitor.message((MessageType) type,
        visitFields(type.asGroupType(), visitor));

  } else if (type.isPrimitive()) {
    return visitor.primitive(type.asPrimitiveType());

  } else {
    // if not a primitive, the typeId must be a group
    GroupType group = type.asGroupType();
    OriginalType annotation = group.getOriginalType();
    if (annotation != null) {
      switch (annotation) {
        case LIST:
          return visitList(group, visitor);

        case MAP:
          return visitMap(group, visitor);

        default:
      }
    }

    return visitor.struct(group, visitFields(group, visitor));
  }
}
 
Example 14
Source File: SimpleRecordConverter.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private Converter createConverter(Type field) {
  LogicalTypeAnnotation ltype = field.getLogicalTypeAnnotation();

  if (field.isPrimitive()) {
    if (ltype != null) {
      return ltype.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<Converter>() {
        @Override
        public Optional<Converter> visit(LogicalTypeAnnotation.StringLogicalTypeAnnotation stringLogicalType) {
          return of(new StringConverter(field.getName()));
        }

        @Override
        public Optional<Converter> visit(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalType) {
          int scale = decimalLogicalType.getScale();
          return of(new DecimalConverter(field.getName(), scale));
        }
      }).orElse(new SimplePrimitiveConverter(field.getName()));
    }
    return new SimplePrimitiveConverter(field.getName());
  }

  GroupType groupType = field.asGroupType();
  if (ltype != null) {
    return ltype.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor<Converter>() {
      @Override
      public Optional<Converter> visit(LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) {
        return of(new SimpleMapRecordConverter(groupType, field.getName(), SimpleRecordConverter.this));
      }

      @Override
      public Optional<Converter> visit(LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) {
        return of(new SimpleListRecordConverter(groupType, field.getName(), SimpleRecordConverter.this));
      }
    }).orElse(new SimpleRecordConverter(groupType, field.getName(), this));
  }
  return new SimpleRecordConverter(groupType, field.getName(), this);
}
 
Example 15
Source File: ColumnReadStoreImpl.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private PrimitiveConverter getPrimitiveConverter(ColumnDescriptor path) {
  Type currentType = schema;
  Converter currentConverter = recordConverter;
  for (String fieldName : path.getPath()) {
    final GroupType groupType = currentType.asGroupType();
    int fieldIndex = groupType.getFieldIndex(fieldName);
    currentType = groupType.getType(fieldName);
    currentConverter = currentConverter.asGroupConverter().getConverter(fieldIndex);
  }
  PrimitiveConverter converter = currentConverter.asPrimitiveConverter();
  return converter;
}
 
Example 16
Source File: AvroRecordConverter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
private static Converter newConverter(Schema schema, Type type,
    GenericData model, Class<?> knownClass, ParentValueContainer setter) {
  LogicalType logicalType = schema.getLogicalType();
  Conversion<?> conversion;

  if (knownClass != null) {
    conversion = model.getConversionByClass(knownClass, logicalType);
  } else {
    conversion = model.getConversionFor(logicalType);
  }

  ParentValueContainer parent = ParentValueContainer
      .getConversionContainer(setter, conversion, schema);

  switch (schema.getType()) {
  case BOOLEAN:
    return new AvroConverters.FieldBooleanConverter(parent);
  case INT:
    Class<?> intDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (intDatumClass == null) {
      return new AvroConverters.FieldIntegerConverter(parent);
    }
    if (intDatumClass == byte.class || intDatumClass == Byte.class) {
      return new AvroConverters.FieldByteConverter(parent);
    }
    if (intDatumClass == char.class || intDatumClass == Character.class) {
      return new AvroConverters.FieldCharConverter(parent);
    }
    if (intDatumClass == short.class || intDatumClass == Short.class) {
      return new AvroConverters.FieldShortConverter(parent);
    }
    return new AvroConverters.FieldIntegerConverter(parent);
  case LONG:
    return new AvroConverters.FieldLongConverter(parent);
  case FLOAT:
    return new AvroConverters.FieldFloatConverter(parent);
  case DOUBLE:
    return new AvroConverters.FieldDoubleConverter(parent);
  case BYTES:
    Class<?> byteDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (byteDatumClass == null) {
      return new AvroConverters.FieldByteBufferConverter(parent);
    }
    if (byteDatumClass.isArray() && byteDatumClass.getComponentType() == byte.class) {
      return new AvroConverters.FieldByteArrayConverter(parent);
    }
    return new AvroConverters.FieldByteBufferConverter(parent);
  case STRING:
    if (logicalType != null && logicalType.getName().equals(LogicalTypes.uuid().getName())) {
      return new AvroConverters.FieldUUIDConverter(parent, type.asPrimitiveType());
    }
    return newStringConverter(schema, model, parent);
  case RECORD:
    return new AvroRecordConverter(parent, type.asGroupType(), schema, model);
  case ENUM:
    return new AvroConverters.FieldEnumConverter(parent, schema, model);
  case ARRAY:
    Class<?> arrayDatumClass = getDatumClass(conversion, knownClass, schema, model);
    if (arrayDatumClass != null && arrayDatumClass.isArray()) {
      return new AvroArrayConverter(parent, type.asGroupType(), schema, model,
          arrayDatumClass);
    }
    return new AvroCollectionConverter(parent, type.asGroupType(), schema,
        model, arrayDatumClass);
  case MAP:
    return new MapConverter(parent, type.asGroupType(), schema, model);
  case UNION:
    return new AvroUnionConverter(parent, type, schema, model);
  case FIXED:
    return new AvroConverters.FieldFixedConverter(parent, schema, model);
  default:
    throw new UnsupportedOperationException(String.format(
        "Cannot convert Avro type: %s to Parquet type: %s", schema, type));
  }
}
 
Example 17
Source File: AvroIndexedRecordConverter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
private static Converter newConverter(Schema schema, Type type,
    GenericData model, ParentValueContainer setter) {

  LogicalType logicalType = schema.getLogicalType();
  // the expected type is always null because it is determined by the parent
  // datum class, which never helps for generic. when logical types are added
  // to specific, this should pass the expected type here.
  Conversion<?> conversion = model.getConversionFor(logicalType);
  ParentValueContainer parent = ParentValueContainer
      .getConversionContainer(setter, conversion, schema);

  switch (schema.getType()) {
  case ARRAY:
    return new AvroArrayConverter(parent, type.asGroupType(), schema, model);
  case BOOLEAN:
    return new AvroConverters.FieldBooleanConverter(parent);
  case BYTES:
    return new AvroConverters.FieldByteBufferConverter(parent);
  case DOUBLE:
    return new AvroConverters.FieldDoubleConverter(parent);
  case ENUM:
    return new FieldEnumConverter(parent, schema, model);
  case FIXED:
    return new FieldFixedConverter(parent, schema, model);
  case FLOAT:
    return new AvroConverters.FieldFloatConverter(parent);
  case INT:
    return new AvroConverters.FieldIntegerConverter(parent);
  case LONG:
    return new AvroConverters.FieldLongConverter(parent);
  case MAP:
    return new MapConverter(parent, type.asGroupType(), schema, model);
  case RECORD:
    return new AvroIndexedRecordConverter(parent, type.asGroupType(), schema, model);
  case STRING:
    return new AvroConverters.FieldStringConverter(parent);
  case UNION:
    return new AvroUnionConverter(parent, type, schema, model);
  case NULL: // fall through
  default:
    throw new UnsupportedOperationException(String.format("Cannot convert Avro type: %s" +
        " (Parquet type: %s) ", schema, type));
  }
}
 
Example 18
Source File: TupleConverter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
static Converter newConverter(FieldSchema pigField, Type type, final ParentValueContainer parent, boolean elephantBirdCompatible, boolean columnIndexAccess) {
  try {
    switch (pigField.type) {
    case DataType.BAG:
      return new BagConverter(type.asGroupType(), pigField, parent, elephantBirdCompatible, columnIndexAccess);
    case DataType.MAP:
      return new MapConverter(type.asGroupType(), pigField, parent, elephantBirdCompatible, columnIndexAccess);
    case DataType.TUPLE:
      return new TupleConverter(type.asGroupType(), pigField.schema, elephantBirdCompatible, columnIndexAccess) {
        @Override
        public void end() {
          super.end();
          parent.add(this.currentTuple);
        }
      };
    case DataType.CHARARRAY:
        //If the orignal type isn't a string, we don't want to use the dictionary because
        //a custom implementation will be needed for each type.  Just default to no dictionary.
      return new FieldStringConverter(parent, type.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation);
    case DataType.BYTEARRAY:
      return new FieldByteArrayConverter(parent);
    case DataType.INTEGER:
      return new FieldIntegerConverter(parent);
    case DataType.BOOLEAN:
      if (elephantBirdCompatible) {
        return new FieldIntegerConverter(parent);
      } else {
        return new FieldBooleanConverter(parent);
      }
    case DataType.FLOAT:
      return new FieldFloatConverter(parent);
    case DataType.DOUBLE:
      return new FieldDoubleConverter(parent);
    case DataType.LONG:
      return new FieldLongConverter(parent);
    case DataType.BIGDECIMAL:
      return new FieldBigDecimalConverter(type, parent);
    default:
      throw new TupleConversionException("unsupported pig type: " + pigField);
    }
  } catch (FrontendException | RuntimeException e) {
    throw new TupleConversionException(
        "error while preparing converter for:\n" + pigField + "\n" + type, e);
  }
}
 
Example 19
Source File: ParquetTypeVisitor.java    From iceberg with Apache License 2.0 4 votes vote down vote up
public static <T> T visit(Type type, ParquetTypeVisitor<T> visitor) {
  if (type instanceof MessageType) {
    return visitor.message((MessageType) type,
        visitFields(type.asGroupType(), visitor));

  } else if (type.isPrimitive()) {
    return visitor.primitive(type.asPrimitiveType());

  } else {
    // if not a primitive, the typeId must be a group
    GroupType group = type.asGroupType();
    OriginalType annotation = group.getOriginalType();
    if (annotation != null) {
      switch (annotation) {
        case LIST:
          Preconditions.checkArgument(!group.isRepetition(REPEATED),
              "Invalid list: top-level group is repeated: " + group);
          Preconditions.checkArgument(group.getFieldCount() == 1,
              "Invalid list: does not contain single repeated field: " + group);

          GroupType repeatedElement = group.getFields().get(0).asGroupType();
          Preconditions.checkArgument(repeatedElement.isRepetition(REPEATED),
              "Invalid list: inner group is not repeated");
          Preconditions.checkArgument(repeatedElement.getFieldCount() <= 1,
              "Invalid list: repeated group is not a single field: " + group);

          visitor.fieldNames.push(repeatedElement.getName());
          try {
            T elementResult = null;
            if (repeatedElement.getFieldCount() > 0) {
              elementResult = visitField(repeatedElement.getType(0), visitor);
            }

            return visitor.list(group, elementResult);

          } finally {
            visitor.fieldNames.pop();
          }

        case MAP:
          Preconditions.checkArgument(!group.isRepetition(REPEATED),
              "Invalid map: top-level group is repeated: " + group);
          Preconditions.checkArgument(group.getFieldCount() == 1,
              "Invalid map: does not contain single repeated field: " + group);

          GroupType repeatedKeyValue = group.getType(0).asGroupType();
          Preconditions.checkArgument(repeatedKeyValue.isRepetition(REPEATED),
              "Invalid map: inner group is not repeated");
          Preconditions.checkArgument(repeatedKeyValue.getFieldCount() <= 2,
              "Invalid map: repeated group does not have 2 fields");

          visitor.fieldNames.push(repeatedKeyValue.getName());
          try {
            T keyResult = null;
            T valueResult = null;
            switch (repeatedKeyValue.getFieldCount()) {
              case 2:
                // if there are 2 fields, both key and value are projected
                keyResult = visitField(repeatedKeyValue.getType(0), visitor);
                valueResult = visitField(repeatedKeyValue.getType(1), visitor);
              case 1:
                // if there is just one, use the name to determine what it is
                Type keyOrValue = repeatedKeyValue.getType(0);
                if (keyOrValue.getName().equalsIgnoreCase("key")) {
                  keyResult = visitField(keyOrValue, visitor);
                  // value result remains null
                } else {
                  valueResult = visitField(keyOrValue, visitor);
                  // key result remains null
                }
              default:
                // both results will remain null
            }

            return visitor.map(group, keyResult, valueResult);

          } finally {
            visitor.fieldNames.pop();
          }

        default:
      }
    }

    return visitor.struct(group, visitFields(group, visitor));
  }
}
 
Example 20
Source File: ParquetTypeVisitor.java    From presto with Apache License 2.0 4 votes vote down vote up
public static <T> T visit(Type type, ParquetTypeVisitor<T> visitor)
{
    if (type instanceof MessageType) {
        return visitor.message((MessageType) type, visitFields(type.asGroupType(), visitor));
    }
    else if (type.isPrimitive()) {
        return visitor.primitive(type.asPrimitiveType());
    }
    else {
        // if not a primitive, the typeId must be a group
        GroupType group = type.asGroupType();
        OriginalType annotation = group.getOriginalType();
        if (annotation == LIST) {
            checkArgument(!group.isRepetition(REPEATED),
                    "Invalid list: top-level group is repeated: " + group);
            checkArgument(group.getFieldCount() == 1,
                    "Invalid list: does not contain single repeated field: " + group);

            GroupType repeatedElement = group.getFields().get(0).asGroupType();
            checkArgument(repeatedElement.isRepetition(REPEATED),
                    "Invalid list: inner group is not repeated");
            checkArgument(repeatedElement.getFieldCount() <= 1,
                    "Invalid list: repeated group is not a single field: " + group);

            visitor.fieldNames.push(repeatedElement.getName());
            try {
                T elementResult = null;
                if (repeatedElement.getFieldCount() > 0) {
                    elementResult = visitField(repeatedElement.getType(0), visitor);
                }

                return visitor.list(group, elementResult);
            }
            finally {
                visitor.fieldNames.pop();
            }
        }
        else if (annotation == MAP) {
            checkArgument(!group.isRepetition(REPEATED),
                    "Invalid map: top-level group is repeated: " + group);
            checkArgument(group.getFieldCount() == 1,
                    "Invalid map: does not contain single repeated field: " + group);

            GroupType repeatedKeyValue = group.getType(0).asGroupType();
            checkArgument(repeatedKeyValue.isRepetition(REPEATED),
                    "Invalid map: inner group is not repeated");
            checkArgument(repeatedKeyValue.getFieldCount() <= 2,
                    "Invalid map: repeated group does not have 2 fields");

            visitor.fieldNames.push(repeatedKeyValue.getName());
            try {
                T keyResult = null;
                T valueResult = null;
                if (repeatedKeyValue.getFieldCount() == 2) {
                    keyResult = visitField(repeatedKeyValue.getType(0), visitor);
                    valueResult = visitField(repeatedKeyValue.getType(1), visitor);
                }
                else if (repeatedKeyValue.getFieldCount() == 1) {
                    Type keyOrValue = repeatedKeyValue.getType(0);
                    if (keyOrValue.getName().equalsIgnoreCase("key")) {
                        keyResult = visitField(keyOrValue, visitor);
                        // value result remains null
                    }
                    else {
                        valueResult = visitField(keyOrValue, visitor);
                        // key result remains null
                    }
                }
                return visitor.map(group, keyResult, valueResult);
            }
            finally {
                visitor.fieldNames.pop();
            }
        }
        return visitor.struct(group, visitFields(group, visitor));
    }
}