Java Code Examples for org.apache.arrow.vector.types.pojo.ArrowType#Decimal

/**
 * Converts from TPCDS columns to Apache Arrow fields.
 *
 * @param column The TPCDS column to conver.
 * @return The Apache Arrow field that corresponds to the TPCDS column.
 */
public static Field convertColumn(Column column)
{
    ColumnType type = column.getType();
    switch (type.getBase()) {
        case TIME:
        case IDENTIFIER:
            return FieldBuilder.newBuilder(column.getName(), Types.MinorType.BIGINT.getType()).build();
        case INTEGER:
            return FieldBuilder.newBuilder(column.getName(), Types.MinorType.INT.getType()).build();
        case DATE:
            return FieldBuilder.newBuilder(column.getName(), Types.MinorType.DATEDAY.getType()).build();
        case DECIMAL:
            ArrowType arrowType = new ArrowType.Decimal(type.getPrecision().get(), type.getScale().get());
            return FieldBuilder.newBuilder(column.getName(), arrowType).build();
        case CHAR:
        case VARCHAR:
            return FieldBuilder.newBuilder(column.getName(), Types.MinorType.VARCHAR.getType()).build();
    }
    throw new IllegalArgumentException("Unsupported TPC-DS type " + column.getName() + ":" + column.getType().getBase());
}
 

static ArrowType.Decimal getDecimalOutputTypeForTruncate(int arg1Precision, int arg1Scale,
                                                         int destinationScale) {
  int finalScale = Math.min(arg1Scale, destinationScale);
  if (finalScale < 0) {
    // -ve scale has special semantics in round/truncate fns.
    finalScale = 0;
  }
  Preconditions.checkState(arg1Scale >= finalScale,
    "Final scale " + finalScale+" cannot be greater than " +
    "original scale of argument : " + arg1Scale);

  int finalPrecision = arg1Precision - (arg1Scale - finalScale);
  if (finalPrecision == 0) {
    finalPrecision = 1;
  }
  return new ArrowType.Decimal(finalPrecision, finalScale);
}
 

static ArrowType.Decimal getDecimalOutputTypeForRound(int arg1Precision, int arg1Scale,
                                                      int destinationScale) {
  int finalScale = Math.min(arg1Scale, destinationScale);
  if (finalScale < 0) {
    // -ve scale has special semantics in round/truncate fns.
    finalScale = 0;
  }
  Preconditions.checkState(arg1Scale >= finalScale,
    "Final scale " + finalScale+" cannot be greater than " +
      "original scale of argument : " + arg1Scale);

  int finalPrecision = arg1Precision - (arg1Scale - finalScale);
  if (finalScale < arg1Scale) {
    finalPrecision++;
  }

  return new ArrowType.Decimal(finalPrecision, finalScale);
}
 

@Override
protected ArrowType defaultMethod(LogicalType logicalType) {
	if (logicalType instanceof LegacyTypeInformationType) {
		Class<?> typeClass = ((LegacyTypeInformationType) logicalType).getTypeInformation().getTypeClass();
		if (typeClass == BigDecimal.class) {
			// Because we can't get precision and scale from legacy BIG_DEC_TYPE_INFO,
			// we set the precision and scale to default value compatible with python.
			return new ArrowType.Decimal(38, 18);
		}
	}
	throw new UnsupportedOperationException(String.format(
		"Python vectorized UDF doesn't support logical type %s currently.", logicalType.asSummaryString()));
}
 

@Override
protected void doTypedSerialize(ArrowType arrowType, JsonGenerator jgen, SerializerProvider provider)
        throws IOException
{
    ArrowType.Decimal decimal = (ArrowType.Decimal) arrowType;
    jgen.writeNumberField(PRECISION_FIELD, decimal.getPrecision());
    jgen.writeNumberField(SCALE_FIELD, decimal.getScale());
}
 

@Override
protected ArrowType doTypedDeserialize(JsonParser jparser, DeserializationContext ctxt)
        throws IOException
{
    int precision = getNextIntField(jparser, PRECISION_FIELD);
    int scale = getNextIntField(jparser, SCALE_FIELD);
    return new ArrowType.Decimal(precision, scale);
}
 

public static CompleteType fromDecimalPrecisionScale(int precision, int scale){
  // TODO: lots of ARP failures with this check.
  //Preconditions.checkArgument(scale >= 0, "invalid scale " + scale +
  //  "must be >= 0");
  //Preconditions.checkArgument(precision > 0 && precision >= scale,
  //  "invalid precision " + precision + ", must be > 0 and >= scale " + scale);
  return new CompleteType(new ArrowType.Decimal(precision, scale));
}
 

@Override
public RelDataType deriveDecimalTruncateType(RelDataTypeFactory typeFactory, RelDataType type1,
  Integer scale2) {
  if (!SqlTypeUtil.isExactNumeric(type1) || !SqlTypeUtil.isDecimal(type1)) {
    return null;
  }

  ArrowType.Decimal finalPrecisionScale = OutputDerivation.getDecimalOutputTypeForTruncate(type1.getPrecision(),
    type1.getScale(), scale2);

  return typeFactory.createSqlType(SqlTypeName.DECIMAL, finalPrecisionScale.getPrecision(),
    finalPrecisionScale.getScale());
}
 

@Override
public RelDataType deriveDecimalRoundType(RelDataTypeFactory typeFactory, RelDataType type1,
  Integer scale2) {
  if (!SqlTypeUtil.isExactNumeric(type1) || !SqlTypeUtil.isDecimal(type1)) {
    return null;
  }

  ArrowType.Decimal finalPrecisionScale = OutputDerivation.getDecimalOutputTypeForRound(type1.getPrecision(),
    type1.getScale(), scale2);

  return typeFactory.createSqlType(SqlTypeName.DECIMAL, finalPrecisionScale.getPrecision(),
    finalPrecisionScale.getScale());
}
 

private RelDataType getDecimalReturnType(RelDataTypeFactory typeFactory, RelDataType type1, RelDataType type2, DecimalTypeUtil
  .OperationType operationType) {
  if (!SqlTypeUtil.isExactNumeric(type1) || !SqlTypeUtil.isExactNumeric(type2) || (!SqlTypeUtil
    .isDecimal(type1) && !SqlTypeUtil.isDecimal(type2))) {
    return null;
  } else {
    ArrowType.Decimal operand1 = new ArrowType.Decimal(type1.getPrecision(), type1.getScale());
    ArrowType.Decimal operand2 = new ArrowType.Decimal(type2.getPrecision(), type2.getScale());
    ArrowType.Decimal output = DecimalTypeUtil.getResultTypeForOperation(operationType, operand1,
      operand2);
    return typeFactory.createSqlType(SqlTypeName.DECIMAL, output.getPrecision(), output.getScale());
  }
}
 

private void verifyDecimalTypesAreSame(OutputMutator output, ParquetColumnResolver columnResolver) {
  for (ValueVector vector : output.getVectors()) {
    Field fieldInSchema = vector.getField();
    if (fieldInSchema.getType().getTypeID() == ArrowType.ArrowTypeID.Decimal) {
      ArrowType.Decimal typeInTable = (ArrowType.Decimal) fieldInSchema.getType();
      Type typeInParquet = null;
      // the field in arrow schema may not be present in hive schema
      try {
        typeInParquet  = schema.getType(columnResolver.getParquetColumnName(fieldInSchema.getName()));
      } catch (InvalidRecordException e) {
      }
      if (typeInParquet == null) {
        continue;
      }
      boolean schemaMisMatch = true;
      OriginalType originalType = typeInParquet.getOriginalType();
      if (originalType.equals(OriginalType.DECIMAL) ) {
        int precision = typeInParquet
          .asPrimitiveType().getDecimalMetadata().getPrecision();
        int scale = typeInParquet.asPrimitiveType().getDecimalMetadata().getScale();
        ArrowType decimalType = new ArrowType.Decimal(precision, scale);
        if (decimalType.equals(typeInTable)) {
          schemaMisMatch = false;
        }
      }
      if (schemaMisMatch) {
        throw UserException.schemaChangeError().message("Mixed types "+ fieldInSchema.getType()
          + " , " + typeInParquet + " is not supported.")
          .build(logger);
      }
    }
  }
}
 

private void verifyReturnType(OutputDerivation decimalZeroScaleRound,
                              ArrowType.Decimal expectedOutputType,
                              ArrayList<LogicalExpression> args) {
  CompleteType decimal10 = decimalZeroScaleRound.getOutputType(null , args);
  Assert.assertTrue(decimal10.isDecimal());
  Assert.assertTrue(expectedOutputType.equals(decimal10.getType()));
}
 

public CompleteType getOutputType(CompleteType baseReturn, List<LogicalExpression> args) {
  ArrowType.Decimal type =  getDecimalOutputTypeForRound(prec(args.get(0)),
    scale(args.get(0)), 0);
  return CompleteType.fromDecimalPrecisionScale(type.getPrecision(), type.getScale());
}
 

@Override
public ArrowType visit(DecimalType decimalType) {
	return new ArrowType.Decimal(decimalType.getPrecision(), decimalType.getScale());
}
 

@Override
ArrowType getType() {
  return new ArrowType.Decimal(precision, scale);
}
 

private ArrowType generifyDecimalType(ArrowType returnType) {
  if (returnType.getTypeID() == ArrowType.ArrowTypeID.Decimal) {
    returnType = new ArrowType.Decimal(0,0);
  }
  return returnType;
}
 

public CompleteType getOutputType(CompleteType baseReturn, List<LogicalExpression> args) {
  ArrowType.Decimal type =  getDecimalOutputTypeForTruncate(prec(args.get(0)),
    scale(args.get(0)), 0);
  return CompleteType.fromDecimalPrecisionScale(type.getPrecision(), type.getScale());
}
 

private Deserializer()
{
    super(ArrowType.class, ArrowType.Decimal.class);
}
 

private Serializer()
{
    super(ArrowType.class, ArrowType.Decimal.class);
}