Java Code Examples for org.apache.parquet.schema.PrimitiveType#getOriginalType()

/**
 * Changes the list inner '$data$' vector name to 'element' in the schema
 */
private Type renameChildTypeToElement(Type childType) {
  if (childType.isPrimitive()) {
    PrimitiveType childPrimitiveType = childType.asPrimitiveType();
    return new PrimitiveType(childType.getRepetition(),
      childPrimitiveType.getPrimitiveTypeName(),
      childPrimitiveType.getTypeLength(),
      "element",
      childPrimitiveType.getOriginalType(),
      childPrimitiveType.getDecimalMetadata(),
      childPrimitiveType.getId());
  } else {
    GroupType childGroupType = childType.asGroupType();
    Type.ID id = childGroupType.getId();
    GroupType groupType = new GroupType(childType.getRepetition(),
      "element",
      childType.getOriginalType(),
      childGroupType.getFields());
    if (id != null) {
      groupType = groupType.withId(id.hashCode());
    }
    return groupType;
  }
}
 

private static void showDetails(PrettyPrintWriter out, PrimitiveType type, int depth, MessageType container, List<String> cpath) {
  String name = Strings.repeat(".", depth) + type.getName();
  OriginalType otype = type.getOriginalType();
  Repetition rep = type.getRepetition();
  PrimitiveTypeName ptype = type.getPrimitiveTypeName();

  out.format("%s: %s %s", name, rep, ptype);
  if (otype != null) out.format(" O:%s", otype);

  if (container != null) {
    cpath.add(type.getName());
    String[] paths = cpath.toArray(new String[0]);
    cpath.remove(cpath.size() - 1);

    ColumnDescriptor desc = container.getColumnDescription(paths);

    int defl = desc.getMaxDefinitionLevel();
    int repl = desc.getMaxRepetitionLevel();
    out.format(" R:%d D:%d", repl, defl);
  }
  out.println();
}
 

public static boolean isIntType(PrimitiveType primitiveType) {
  if (primitiveType.getOriginalType() != null) {
    switch (primitiveType.getOriginalType()) {
      case INT_8:
      case INT_16:
      case INT_32:
      case DATE:
        return true;
      default:
        return false;
    }
  }
  return primitiveType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT32;
}
 

static Function<Object, Object> converterFromParquet(PrimitiveType type) {
  if (type.getOriginalType() != null) {
    switch (type.getOriginalType()) {
      case UTF8:
        // decode to CharSequence to avoid copying into a new String
        return binary -> StandardCharsets.UTF_8.decode(((Binary) binary).toByteBuffer());
      case DECIMAL:
        int scale = type.getDecimalMetadata().getScale();
        switch (type.getPrimitiveTypeName()) {
          case INT32:
          case INT64:
            return num -> BigDecimal.valueOf(((Number) num).longValue(), scale);
          case FIXED_LEN_BYTE_ARRAY:
          case BINARY:
            return bin -> new BigDecimal(new BigInteger(((Binary) bin).getBytes()), scale);
          default:
            throw new IllegalArgumentException(
                "Unsupported primitive type for decimal: " + type.getPrimitiveTypeName());
        }
      default:
    }
  }

  switch (type.getPrimitiveTypeName()) {
    case FIXED_LEN_BYTE_ARRAY:
    case BINARY:
      return binary -> ByteBuffer.wrap(((Binary) binary).getBytes());
    default:
  }

  return obj -> obj;
}
 

static Function<Object, Object> converterFromParquet(PrimitiveType type) {
  if (type.getOriginalType() != null) {
    switch (type.getOriginalType()) {
      case UTF8:
        // decode to CharSequence to avoid copying into a new String
        return binary -> Charsets.UTF_8.decode(((Binary) binary).toByteBuffer());
      case DECIMAL:
        int scale = type.getDecimalMetadata().getScale();
        switch (type.getPrimitiveTypeName()) {
          case INT32:
          case INT64:
            return num -> BigDecimal.valueOf(((Number) num).longValue(), scale);
          case FIXED_LEN_BYTE_ARRAY:
          case BINARY:
            return bin -> new BigDecimal(new BigInteger(((Binary) bin).getBytes()), scale);
          default:
            throw new IllegalArgumentException(
                "Unsupported primitive type for decimal: " + type.getPrimitiveTypeName());
        }
      default:
    }
  }

  switch (type.getPrimitiveTypeName()) {
    case FIXED_LEN_BYTE_ARRAY:
    case BINARY:
      return binary -> ByteBuffer.wrap(((Binary) binary).getBytes());
    default:
  }

  return obj -> obj;
}
 

private static void showDetails(PrettyPrintWriter out, PrimitiveType type, int depth, MessageType container, List<String> cpath, boolean showOriginalTypes) {
  String name = Strings.repeat(".", depth) + type.getName();
  Repetition rep = type.getRepetition();
  PrimitiveTypeName ptype = type.getPrimitiveTypeName();

  out.format("%s: %s %s", name, rep, ptype);
  if (showOriginalTypes) {
    OriginalType otype;
    try {
      otype = type.getOriginalType();
    } catch (Exception e) {
      otype = null;
    }
    if (otype != null) out.format(" O:%s", otype);
  } else {
    LogicalTypeAnnotation ltype = type.getLogicalTypeAnnotation();
    if (ltype != null) out.format(" L:%s", ltype);
  }

  if (container != null) {
    cpath.add(type.getName());
    String[] paths = cpath.toArray(new String[0]);
    cpath.remove(cpath.size() - 1);

    ColumnDescriptor desc = container.getColumnDescription(paths);

    int defl = desc.getMaxDefinitionLevel();
    int repl = desc.getMaxRepetitionLevel();
    out.format(" R:%d D:%d", repl, defl);
  }
  out.println();
}
 

@NotNull
private static ArrowVectorAccessor getDictionaryVectorAccessor(
    Dictionary dictionary,
    ColumnDescriptor desc,
    FieldVector vector, PrimitiveType primitive) {
  Preconditions.checkState(vector instanceof IntVector, "Dictionary ids should be stored in IntVectors only");
  if (primitive.getOriginalType() != null) {
    switch (desc.getPrimitiveType().getOriginalType()) {
      case ENUM:
      case JSON:
      case UTF8:
      case BSON:
        return new DictionaryStringAccessor((IntVector) vector, dictionary);
      case INT_64:
      case TIMESTAMP_MILLIS:
      case TIMESTAMP_MICROS:
        return new DictionaryLongAccessor((IntVector) vector, dictionary);
      case DECIMAL:
        switch (primitive.getPrimitiveTypeName()) {
          case BINARY:
          case FIXED_LEN_BYTE_ARRAY:
            return new DictionaryDecimalBinaryAccessor(
                (IntVector) vector,
                dictionary);
          case INT64:
            return new DictionaryDecimalLongAccessor(
                (IntVector) vector,
                dictionary);
          case INT32:
            return new DictionaryDecimalIntAccessor(
                (IntVector) vector,
                dictionary);
          default:
            throw new UnsupportedOperationException(
                "Unsupported base type for decimal: " + primitive.getPrimitiveTypeName());
        }
      default:
        throw new UnsupportedOperationException(
            "Unsupported logical type: " + primitive.getOriginalType());
    }
  } else {
    switch (primitive.getPrimitiveTypeName()) {
      case FIXED_LEN_BYTE_ARRAY:
      case BINARY:
        return new DictionaryBinaryAccessor((IntVector) vector, dictionary);
      case FLOAT:
        return new DictionaryFloatAccessor((IntVector) vector, dictionary);
      case INT64:
        return new DictionaryLongAccessor((IntVector) vector, dictionary);
      case DOUBLE:
        return new DictionaryDoubleAccessor((IntVector) vector, dictionary);
      default:
        throw new UnsupportedOperationException("Unsupported type: " + primitive);
    }
  }
}