Java Code Examples for org.apache.parquet.format.SchemaElement#isSetField_id()

The following examples show how to use org.apache.parquet.format.SchemaElement#isSetField_id() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MetadataReader.java    From presto with Apache License 2.0 5 votes vote down vote up 
private static void readTypeSchema(Types.GroupBuilder<?> builder, Iterator<SchemaElement> schemaIterator, int typeCount)
{
    for (int i = 0; i < typeCount; i++) {
        SchemaElement element = schemaIterator.next();
        Types.Builder<?, ?> typeBuilder;
        if (element.type == null) {
            typeBuilder = builder.group(Repetition.valueOf(element.repetition_type.name()));
            readTypeSchema((Types.GroupBuilder<?>) typeBuilder, schemaIterator, element.num_children);
        }
        else {
            Types.PrimitiveBuilder<?> primitiveBuilder = builder.primitive(getTypeName(element.type), Repetition.valueOf(element.repetition_type.name()));
            if (element.isSetType_length()) {
                primitiveBuilder.length(element.type_length);
            }
            if (element.isSetPrecision()) {
                primitiveBuilder.precision(element.precision);
            }
            if (element.isSetScale()) {
                primitiveBuilder.scale(element.scale);
            }
            typeBuilder = primitiveBuilder;
        }

        if (element.isSetConverted_type()) {
            typeBuilder.as(getOriginalType(element.converted_type));
        }
        if (element.isSetField_id()) {
            typeBuilder.id(element.field_id);
        }
        typeBuilder.named(element.name.toLowerCase(Locale.ENGLISH));
    }
}
Example 2
Source File: ParquetMetadataConverter.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up 
MessageType fromParquetSchema(List<SchemaElement> schema, List<ColumnOrder> columnOrders) {
  Iterator<SchemaElement> iterator = schema.iterator();
  SchemaElement root = iterator.next();
  Types.MessageTypeBuilder builder = Types.buildMessage();
  if (root.isSetField_id()) {
    builder.id(root.field_id);
  }
  buildChildren(builder, iterator, root.getNum_children(), columnOrders, 0);
  return builder.named(root.name);
}
Example 3
Source File: ParquetMetadataConverter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
private void buildChildren(Types.GroupBuilder builder,
                           Iterator<SchemaElement> schema,
                           int childrenCount,
                           List<ColumnOrder> columnOrders,
                           int columnCount) {
  for (int i = 0; i < childrenCount; i++) {
    SchemaElement schemaElement = schema.next();

    // Create Parquet Type.
    Types.Builder childBuilder;
    if (schemaElement.type != null) {
      Types.PrimitiveBuilder primitiveBuilder = builder.primitive(
          getPrimitive(schemaElement.type),
          fromParquetRepetition(schemaElement.repetition_type));
      if (schemaElement.isSetType_length()) {
        primitiveBuilder.length(schemaElement.type_length);
      }
      if (schemaElement.isSetPrecision()) {
        primitiveBuilder.precision(schemaElement.precision);
      }
      if (schemaElement.isSetScale()) {
        primitiveBuilder.scale(schemaElement.scale);
      }
      if (columnOrders != null) {
        org.apache.parquet.schema.ColumnOrder columnOrder = fromParquetColumnOrder(columnOrders.get(columnCount));
        // As per parquet format 2.4.0 no UNDEFINED order is supported. So, set undefined column order for the types
        // where ordering is not supported.
        if (columnOrder.getColumnOrderName() == ColumnOrderName.TYPE_DEFINED_ORDER
            && (schemaElement.type == Type.INT96 || schemaElement.converted_type == ConvertedType.INTERVAL)) {
          columnOrder = org.apache.parquet.schema.ColumnOrder.undefined();
        }
        primitiveBuilder.columnOrder(columnOrder);
      }
      childBuilder = primitiveBuilder;

    } else {
      childBuilder = builder.group(fromParquetRepetition(schemaElement.repetition_type));
      buildChildren((Types.GroupBuilder) childBuilder, schema, schemaElement.num_children, columnOrders, columnCount);
    }

    if (schemaElement.isSetLogicalType()) {
      childBuilder.as(getLogicalTypeAnnotation(schemaElement.logicalType));
    }
    if (schemaElement.isSetConverted_type()) {
      OriginalType originalType = getLogicalTypeAnnotation(schemaElement.converted_type, schemaElement).toOriginalType();
      OriginalType newOriginalType = (schemaElement.isSetLogicalType() && getLogicalTypeAnnotation(schemaElement.logicalType) != null) ?
         getLogicalTypeAnnotation(schemaElement.logicalType).toOriginalType() : null;
      if (!originalType.equals(newOriginalType)) {
        if (newOriginalType != null) {
          LOG.warn("Converted type and logical type metadata mismatch (convertedType: {}, logical type: {}). Using value in converted type.",
            schemaElement.converted_type, schemaElement.logicalType);
        }
        childBuilder.as(originalType);
      }
    }
    if (schemaElement.isSetField_id()) {
      childBuilder.id(schemaElement.field_id);
    }

    childBuilder.named(schemaElement.name);
    ++columnCount;
  }
}