org.apache.arrow.vector.types.pojo.ArrowType.Decimal Java Examples

@Override
public Void visit(Decimal decimal) {
  writeString(sqlTypeNameVisitor.visit(decimal));

  try {
    generator.writeFieldName("precision");
    generator.writeNumber(decimal.getPrecision());

    generator.writeFieldName("scale");
    generator.writeNumber(decimal.getScale());
  } catch (IOException e) {
    // no op
  }

  return null;
}
 

@Test
public void primitiveTwoWay() {
  BatchSchema schema = BatchSchema.newBuilder()
    .addField(CompleteType.BIT.toField("boolean"))
    .addField(CompleteType.INT.toField("int"))
    .addField(CompleteType.BIGINT.toField("long"))
    .addField(CompleteType.FLOAT.toField("float"))
    .addField(CompleteType.DOUBLE.toField("double"))
    .addField(CompleteType.DATE.toField("date"))
    .addField(CompleteType.TIMESTAMP.toField("time"))
    .addField(CompleteType.VARCHAR.toField("string"))
    .addField(CompleteType.VARBINARY.toField("binary"))
    .addField(new CompleteType(new Decimal(38, 16)).toField("decimal_38_16"))
    .build();

  org.apache.iceberg.Schema icebergSchema = schemaConverter.toIceberg(schema);
  BatchSchema result = schemaConverter.fromIceberg(icebergSchema);
  assertEquals(result, schema);
}
 

@Override
public CompleteType getOutputType(CompleteType baseReturn, List<LogicalExpression> args) {
  int scale = 0;
  int integral = 0;

  // Get the max scale and integral part from the inputs
  for (LogicalExpression e : args) {
    Decimal arg = e.getCompleteType().getType(Decimal.class);
    scale = Math.max(scale, arg.getScale());
    integral = Math.max(integral, arg.getPrecision() - scale);
  }
  if (scale + integral > 38) {
    throw UserException.functionError().message(
      "derived precision for union of arguments exceeds 38", args
    ).build(logger);
  }

  return CompleteType.fromDecimalPrecisionScale(integral + scale, scale);
}
 

static ArrowType.Decimal getDecimalOutputTypeForRound(int arg1Precision, int arg1Scale,
                                                      int destinationScale) {
  int finalScale = Math.min(arg1Scale, destinationScale);
  if (finalScale < 0) {
    // -ve scale has special semantics in round/truncate fns.
    finalScale = 0;
  }
  Preconditions.checkState(arg1Scale >= finalScale,
    "Final scale " + finalScale+" cannot be greater than " +
      "original scale of argument : " + arg1Scale);

  int finalPrecision = arg1Precision - (arg1Scale - finalScale);
  if (finalScale < arg1Scale) {
    finalPrecision++;
  }

  return new ArrowType.Decimal(finalPrecision, finalScale);
}
 

static ArrowType.Decimal getDecimalOutputTypeForTruncate(int arg1Precision, int arg1Scale,
                                                         int destinationScale) {
  int finalScale = Math.min(arg1Scale, destinationScale);
  if (finalScale < 0) {
    // -ve scale has special semantics in round/truncate fns.
    finalScale = 0;
  }
  Preconditions.checkState(arg1Scale >= finalScale,
    "Final scale " + finalScale+" cannot be greater than " +
    "original scale of argument : " + arg1Scale);

  int finalPrecision = arg1Precision - (arg1Scale - finalScale);
  if (finalPrecision == 0) {
    finalPrecision = 1;
  }
  return new ArrowType.Decimal(finalPrecision, finalScale);
}
 

public static CompleteType fromDecimalPrecisionScale(int precision, int scale){
  // TODO: lots of ARP failures with this check.
  //Preconditions.checkArgument(scale >= 0, "invalid scale " + scale +
  //  "must be >= 0");
  //Preconditions.checkArgument(precision > 0 && precision >= scale,
  //  "invalid precision " + precision + ", must be > 0 and >= scale " + scale);
  return new CompleteType(new ArrowType.Decimal(precision, scale));
}
 

public boolean isNumeric() {
  switch(type.getTypeID()){
    case Decimal:
    case FloatingPoint:
    case Int:
      return true;
    default:
      return false;
  }
}
 

@Test
public void primitiveBasic() {
  org.apache.iceberg.Schema icebergSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "boolean", Types.BooleanType.get()),
    NestedField.optional(2, "int", Types.IntegerType.get()),
    NestedField.optional(3, "long", Types.LongType.get()),
    NestedField.optional(4, "float", Types.FloatType.get()),
    NestedField.optional(5, "double", Types.DoubleType.get()),
    NestedField.optional(6, "decimal_38_16", Types.DecimalType.of(38, 16)),
    NestedField.optional(7, "string", Types.StringType.get()),
    NestedField.optional(8, "binary", Types.BinaryType.get()),
    NestedField.optional(9, "date", Types.DateType.get()),
    NestedField.optional(10, "time", Types.TimeType.get()),
    NestedField.optional(11, "fixed_32", Types.FixedType.ofLength(32)),
    NestedField.optional(12, "timestamp", Types.TimestampType.withZone())
  );

  BatchSchema schema = BatchSchema.newBuilder()
    .addField(CompleteType.BIT.toField("boolean"))
    .addField(CompleteType.INT.toField("int"))
    .addField(CompleteType.BIGINT.toField("long"))
    .addField(CompleteType.FLOAT.toField("float"))
    .addField(CompleteType.DOUBLE.toField("double"))
    .addField(new CompleteType(new Decimal(38, 16)).toField("decimal_38_16"))
    .addField(CompleteType.VARCHAR.toField("string"))
    .addField(CompleteType.VARBINARY.toField("binary"))
    .addField(CompleteType.DATE.toField("date"))
    .addField(CompleteType.TIME.toField("time"))
    .addField(new CompleteType(new ArrowType.FixedSizeBinary(32)).toField("fixed_32"))
    .addField(CompleteType.TIMESTAMP.toField("timestamp"))
    .build();

  assertEquals(schema, schemaConverter.fromIceberg(icebergSchema));
  assertEquals(icebergSchema.toString(), schemaConverter.toIceberg(schema).toString());
}
 

@Override
public CompleteType getOutputType(CompleteType baseReturn, List<LogicalExpression> args) {
  int scale = 0;
  int precision = 0;

  // Get the max scale and precision from the inputs
  for (LogicalExpression e : args) {
    Decimal arg = e.getCompleteType().getType(Decimal.class);
    scale = Math.max(scale, arg.getScale());
    precision = Math.max(precision, arg.getPrecision());
  }

  return CompleteType.fromDecimalPrecisionScale(precision, scale);
}
 

public static Field getArrowFieldFromHivePrimitiveType(String name, TypeInfo typeInfo) {
  switch (typeInfo.getCategory()) {
  case PRIMITIVE:
    PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
    switch (pTypeInfo.getPrimitiveCategory()) {
    case BOOLEAN:

      return new Field(name, true, new Bool(), null);
    case BYTE:
      return new Field(name, true, new Int(32, true), null);
    case SHORT:
      return new Field(name, true, new Int(32, true), null);

    case INT:
      return new Field(name, true, new Int(32, true), null);

    case LONG:
      return new Field(name, true, new Int(64, true), null);

    case FLOAT:
      return new Field(name, true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);

    case DOUBLE:
      return new Field(name, true, new FloatingPoint(FloatingPointPrecision.DOUBLE), null);

    case DATE:
      return new Field(name, true, new Date(DateUnit.MILLISECOND), null);

    case TIMESTAMP:
      return new Field(name, true, new Timestamp(TimeUnit.MILLISECOND, null), null);

    case BINARY:
      return new Field(name, true, new Binary(), null);
    case DECIMAL: {
      DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) pTypeInfo;
      return new Field(name, true, new Decimal(decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale()), null);
    }

    case STRING:
    case VARCHAR:
    case CHAR: {
      return new Field(name, true, new Utf8(), null);
    }
    case UNKNOWN:
    case VOID:
    default:
      // fall through.
    }
  default:
  }

  return null;
}
 

public static ArrowType toArrowType(UserProtos.ResultColumnMetadata type) {
  String typeName = type.getDataType();
  switch (typeName) {
    case "NULL":
      return new Null();
    case "MAP":
      return new ArrowType.Map(false); //todo inner type?
    case "ARRAY":
      return new ArrowType.List(); //todo inner type?
    case "UNION":
      throw new UnsupportedOperationException("have not implemented unions");
      //return new Union(); //todo inner type?
    case "TINYINT":
      return new Int(8, true);
    case "SMALLINT":
      return new Int(16, true);
    case "INTEGER":
      return new Int(32, true);
    case "BIGINT":
      return new Int(64, true);
    case "FLOAT":
      return new FloatingPoint(FloatingPointPrecision.SINGLE);
    case "DOUBLE":
      return new FloatingPoint(FloatingPointPrecision.DOUBLE);
    case "CHARACTER VARYING":
      return new Utf8();
    case "BINARY VARYING":
      return new Binary();
    case "BOOLEAN":
      return new Bool();
    case "DECIMAL":
      return new Decimal(type.getPrecision(), type.getScale());
    case "DATE":
      return new Date(DateUnit.MILLISECOND);
    case "TIME":
      return new Time(TimeUnit.MICROSECOND, 64);
    case "TIMESTAMP":
      return new Timestamp(TimeUnit.MICROSECOND, "UTC");
    case "INTERVAL DAY TO SECOND":
      return new Interval(IntervalUnit.DAY_TIME);
    case "INTERVAL YEAR TO MONTH":
      return new Interval(IntervalUnit.YEAR_MONTH);
    case "BINARY":
      return new ArrowType.FixedSizeBinary(50);
    default:
      throw new IllegalStateException("unable to find arrow type for " + typeName);
  }
}
 

public static CompleteType fromIcebergPrimitiveType(PrimitiveType type) {
  switch (type.typeId()) {
    case BOOLEAN:
      return CompleteType.BIT;
    case INTEGER:
      return CompleteType.INT;
    case LONG:
      return CompleteType.BIGINT;
    case FLOAT:
      return CompleteType.FLOAT;
    case DOUBLE:
      return CompleteType.DOUBLE;
    case STRING:
      return CompleteType.VARCHAR;
    case BINARY:
      return CompleteType.VARBINARY;
    case UUID:
      return new CompleteType(new FixedSizeBinary(16));
    case DATE:
      return CompleteType.DATE;
    case TIME:
      // TODO: When we support Time and Timestamp MICROS, this needs to be changed  to use
      // the existing schema definition for older tables, and to use MICROS for newer tables
      return CompleteType.TIME;
    case TIMESTAMP:
      {
        if (((TimestampType) type.asPrimitiveType()).shouldAdjustToUTC()) {
          return CompleteType.TIMESTAMP;
        } else {
          throw new UnsupportedOperationException("iceberg timestamp type without zone not supported");
        }
      }
    case FIXED:
      return new CompleteType(new FixedSizeBinary(((FixedType)type).length()));
    case DECIMAL:
      DecimalType decimalType = (DecimalType)type;
      return new CompleteType(new Decimal(decimalType.precision(), decimalType.scale()));
    default:
      throw new UnsupportedOperationException("Unsupported iceberg type : " + type);
  }
}
 

public void setup(OutputMutator output){
  vector = (DecimalVector)output.getVector(name);
  if (vector == null) {
    vector = output.addField(new Field(name, true, new Decimal(precision, scale), null), DecimalVector.class);
  }
}
 

public static int scale(LogicalExpression e){
  return e.getCompleteType().getType(Decimal.class).getScale();
}
 

public static int prec(LogicalExpression e){
  return e.getCompleteType().getType(Decimal.class).getPrecision();
}
 

public static CompleteType getOutputType(OperationType operationType, List<LogicalExpression> args) {
  return new CompleteType(DecimalTypeUtil.getResultTypeForOperation(operationType,
    args.get(0).getCompleteType().getType(ArrowType.Decimal.class),
    args.get(1).getCompleteType().getType(ArrowType.Decimal.class)));
}
 

public CompleteType getOutputType(CompleteType baseReturn, List<LogicalExpression> args) {
  ArrowType.Decimal type =  getDecimalOutputTypeForTruncate(prec(args.get(0)),
    scale(args.get(0)), 0);
  return CompleteType.fromDecimalPrecisionScale(type.getPrecision(), type.getScale());
}
 

public CompleteType getOutputType(CompleteType baseReturn, List<LogicalExpression> args) {
  ArrowType.Decimal type =  getDecimalOutputTypeForRound(prec(args.get(0)),
    scale(args.get(0)), 0);
  return CompleteType.fromDecimalPrecisionScale(type.getPrecision(), type.getScale());
}
 

private boolean castEqual(CompleteType from, MajorType to) {
  MinorType fromMinor = from.toMinorType();
  if(from.toMinorType() != to.getMinorType()){
    return false;
  }

  switch (fromMinor) {
  case FLOAT4:
  case FLOAT8:
  case INT:
  case BIGINT:
  case BIT:
  case TINYINT:
  case SMALLINT:
  case UINT1:
  case UINT2:
  case UINT4:
  case UINT8:
  case TIME:
  case TIMESTAMP:
  case TIMESTAMPTZ:
  case DATE:
  case INTERVAL:
  case INTERVALDAY:
  case INTERVALYEAR:
    // nothing else matters.
    return true;
  case DECIMAL:
  case DECIMAL9:
  case DECIMAL18:
  case DECIMAL28DENSE:
  case DECIMAL28SPARSE:
  case DECIMAL38DENSE:
  case DECIMAL38SPARSE:
    Decimal fromDecimal = from.getType(Decimal.class);
    return to.getScale() == fromDecimal.getScale() && to.getPrecision() == fromDecimal.getPrecision();
  case VARBINARY:
  case VARCHAR:
    // since we don't track length, if there is an explicit cast to one of these types, we have to do it.
    // TODO: come up with a way of determining if the cast is explicit.
    return false;

  default:
    errorCollector.addGeneralError(String.format("Casting rules are unknown for type %s.", from));
    return false;
  }
}
 

public static void read(CompleteType ct, JExpression vector, JBlock eval, HoldingContainer out, JCodeModel model,
                        JExpression indexVariable) {

  MinorType type = ct.toMinorType();

    eval.assign(out.getIsSet(), vector.invoke("isSet").arg(indexVariable));
    eval = eval._if(out.getIsSet().eq(JExpr.lit(1)))._then();
    switch (type) {
    case BIGINT:
    case FLOAT4:
    case FLOAT8:
    case INT:
    case MONEY:
    case SMALLINT:
    case TINYINT:
    case UINT1:
    case UINT2:
    case UINT4:
    case UINT8:
    case INTERVALYEAR:
    case DATE:
    case TIME:
    case TIMESTAMP:
    case BIT:
      eval.assign(out.getValue(), vector.invoke("get").arg(indexVariable));
      return;
    case DECIMAL:
      eval.assign(out.getHolder().ref("scale"), JExpr.lit(ct.getType(Decimal.class).getScale()));
      eval.assign(out.getHolder().ref("precision"), JExpr.lit(ct.getType(Decimal.class).getPrecision()));
      eval.assign(out.getHolder().ref("start"), JExpr.lit(TypeHelper.getSize(getArrowMinorType(type))).mul(indexVariable));
      eval.assign(out.getHolder().ref("buffer"), vector.invoke("getDataBuffer"));
      return;
    case INTERVALDAY: {
      JVar start = eval.decl(model.INT, "start", JExpr.lit(TypeHelper.getSize(getArrowMinorType(type))).mul(indexVariable));
      eval.assign(out.getHolder().ref("days"), vector.invoke("getDataBuffer").invoke("getInt").arg(start));
      eval.assign(out.getHolder().ref("milliseconds"), vector.invoke("getDataBuffer").invoke("getInt").arg(start.plus(JExpr.lit(4))));
      return;
    }
    case VARBINARY:
    case VARCHAR:
       eval.assign(out.getHolder().ref("buffer"), vector.invoke("getDataBuffer"));
       JVar se = eval.decl(model.LONG, "startEnd", vector.invoke("getStartEnd").arg(indexVariable));
       eval.assign(out.getHolder().ref("start"), JExpr.cast(model._ref(int.class), se));
       eval.assign(out.getHolder().ref("end"), JExpr.cast(model._ref(int.class), se.shr(JExpr.lit(32))));
      return;

    }

  // fallback.
  eval.add(vector.invoke("get").arg(indexVariable).arg(out.getHolder()));
}
 

public static ArrowType getArrowTypeForMajorType(MajorType majorType) {
  if (majorType.getMinorType() == TypeProtos.MinorType.DECIMAL) {
    return new Decimal(majorType.getPrecision(), majorType.getScale());
  }
  return getArrowMinorType(majorType.getMinorType()).getType();
}
 

public static MajorType getMajorTypeForArrowType(ArrowType arrowType, List<Field> children) {
  MajorType.Builder builder = MajorType.newBuilder()
    .setMinorType(getMinorTypeFromArrowMinorType(getMinorTypeForArrowType(arrowType)))
    .setMode(DataMode.OPTIONAL);
  ArrowTypeID fieldType = arrowType.getTypeID();
  switch(fieldType) {
    case Decimal:
      builder.setPrecision(((Decimal) arrowType).getPrecision()).setScale(((Decimal) arrowType).getScale());
      break;

    case Utf8:
    case Binary:
      builder.setPrecision(CompleteType.DEFAULT_VARCHAR_PRECISION);
      break;

    case Timestamp:
      TimeUnit unit = ((Timestamp) arrowType).getUnit();
      switch(unit) {
        // Only MILLISECONDS is supported, but future-proofing
        case SECOND:
          builder.setPrecision(0);
          break;
        case MILLISECOND:
          builder.setPrecision(3);
          break;
        case MICROSECOND:
          builder.setPrecision(6);
          break;
        case NANOSECOND:
          builder.setPrecision(9);
          break;
        default:
          throw new AssertionError("Arrow TimeUnit " + unit + "not supported");
      }
      break;

    case Union:
      for (Field child : children) {
        builder.addSubType(getMinorTypeFromArrowMinorType(getMinorTypeForArrowType(child.getType())));
      }
      break;

    default:
      // Nothing
  }
  return builder.build();
}
 

@Override
public String visit(Decimal paramDecimal) {
  return "DECIMAL";
}
 

@Override
public Integer visit(Decimal paramDecimal) {
  return 2 + paramDecimal.getPrecision();
}
 

public boolean isDecimal() {
  return type.getTypeID() == ArrowTypeID.Decimal;
}
 

@Override
public String visit(Decimal type) {
  return String.format("decimal(%d,%d)", type.getPrecision(), type.getScale());
}
 

@Override
public T visit(Decimal type) {
  return visitGeneric(type);
}