org.apache.avro.LogicalType Java Examples

@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}
 

@Override
@SuppressWarnings("unchecked")
public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof ParquetDecimal) {
    ParquetDecimal decimal = (ParquetDecimal) logicalType;
    if (decimal.precision() <= 9) {
      return (Conversion<T>) intDecimalConversion;
    } else if (decimal.precision() <= 18) {
      return (Conversion<T>) longDecimalConversion;
    } else {
      return (Conversion<T>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<T>) uuidConversion;
  }
  return super.getConversionByClass(datumClass, logicalType);
}
 

private LogicalTypeAnnotation convertLogicalType(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  } else if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    return decimalType(decimal.getScale(), decimal.getPrecision());
  } else if (logicalType instanceof LogicalTypes.Date) {
    return dateType();
  } else if (logicalType instanceof LogicalTypes.TimeMillis) {
    return timeType(true, MILLIS);
  } else if (logicalType instanceof LogicalTypes.TimeMicros) {
    return timeType(true, MICROS);
  } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
    return timestampType(true, MILLIS);
  } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
    return timestampType(true, MICROS);
  } else if (logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
    return uuidType();
  }
  return null;
}
 

private <D> Object convert(Schema schema, LogicalType logicalType,
                           Conversion<D> conversion, Object datum) {
  if (conversion == null) {
    return datum;
  }
  Class<D> fromClass = conversion.getConvertedType();
  switch (schema.getType()) {
    case RECORD:  return conversion.toRecord(fromClass.cast(datum), schema, logicalType);
    case ENUM:    return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType);
    case ARRAY:   return conversion.toArray(fromClass.cast(datum), schema, logicalType);
    case MAP:     return conversion.toMap(fromClass.cast(datum), schema, logicalType);
    case FIXED:   return conversion.toFixed(fromClass.cast(datum), schema, logicalType);
    case STRING:  return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType);
    case BYTES:   return conversion.toBytes(fromClass.cast(datum), schema, logicalType);
    case INT:     return conversion.toInt(fromClass.cast(datum), schema, logicalType);
    case LONG:    return conversion.toLong(fromClass.cast(datum), schema, logicalType);
    case FLOAT:   return conversion.toFloat(fromClass.cast(datum), schema, logicalType);
    case DOUBLE:  return conversion.toDouble(fromClass.cast(datum), schema, logicalType);
    case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType);
  }
  return datum;
}
 

private Optional<Timestamp> readTimestamp(
    GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) {
  switch (avroType) {
    case LONG:
      if (LogicalTypes.timestampMillis().equals(logicalType)) {
        return Optional.ofNullable((Long) record.get(fieldName))
            .map(x -> Timestamp.ofTimeMicroseconds(1000L * x));
      }
      if (LogicalTypes.timestampMicros().equals(logicalType)) {
        return Optional.ofNullable((Long) record.get(fieldName))
            .map(Timestamp::ofTimeMicroseconds);
      }
      // Default to micro-seconds.
      return Optional.ofNullable((Long) record.get(fieldName)).map(Timestamp::ofTimeMicroseconds);
    case STRING:
      return Optional.ofNullable((Utf8) record.get(fieldName))
          .map(Utf8::toString)
          .map(Timestamp::parseTimestamp);
    default:
      throw new IllegalArgumentException("Cannot interpret " + avroType + " as TIMESTAMP");
  }
}
 

private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 

@Override
@SuppressWarnings("unchecked")
public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof ParquetDecimal) {
    ParquetDecimal decimal = (ParquetDecimal) logicalType;
    if (decimal.precision() <= 9) {
      return (Conversion<T>) intDecimalConversion;
    } else if (decimal.precision() <= 18) {
      return (Conversion<T>) longDecimalConversion;
    } else {
      return (Conversion<T>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<T>) uuidConversion;
  }
  return super.getConversionByClass(datumClass, logicalType);
}
 

@Override
public IndexedRecord fromRecord(IndexedRecord value, Schema schema, LogicalType type) {
  // read side
  long id = getId(value, schema);

  // keep track of this for later references
  references.put(id, value);

  // call any callbacks waiting to resolve this id
  List<Callback> callbacks = callbacksById.get(id);
  for (Callback callback : callbacks) {
    callback.set(value);
  }

  return value;
}
 

@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}
 

private Optional<Date> readDate(
    GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) {
  switch (avroType) {
    case INT:
      if (logicalType == null || !LogicalTypes.date().equals(logicalType)) {
        throw new IllegalArgumentException(
            "Cannot interpret Avrotype INT Logicaltype " + logicalType + " as DATE");
      }
      // Avro Date is number of days since Jan 1, 1970.
      // Have to convert to Java Date first before creating google.cloud.core.Date
      return Optional.ofNullable((Integer) record.get(fieldName))
          .map(x -> new java.util.Date((long) x * 24L * 3600L * 1000L))
          .map(Date::fromJavaUtilDate);
    case STRING:
      return Optional.ofNullable((Utf8) record.get(fieldName))
          .map(Utf8::toString)
          .map(Date::parseDate);
    default:
      throw new IllegalArgumentException("Cannot interpret " + avroType + " as DATE");
  }
}
 

private LogicalType convertOriginalTypeToLogicalType(OriginalType annotation, DecimalMetadata meta) {
  if (annotation == null) {
    return null;
  }
  switch (annotation) {
    case DECIMAL:
      return LogicalTypes.decimal(meta.getPrecision(), meta.getScale());
    case DATE:
      return LogicalTypes.date();
    case TIME_MILLIS:
      return LogicalTypes.timeMillis();
    case TIME_MICROS:
      return LogicalTypes.timeMicros();
    case TIMESTAMP_MILLIS:
      return LogicalTypes.timestampMillis();
    case TIMESTAMP_MICROS:
      return LogicalTypes.timestampMicros();
    default:
      return null;
  }
}
 

private OriginalType convertLogicalType(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  } else if (logicalType instanceof LogicalTypes.Decimal) {
    return OriginalType.DECIMAL;
  } else if (logicalType instanceof LogicalTypes.Date) {
    return OriginalType.DATE;
  } else if (logicalType instanceof LogicalTypes.TimeMillis) {
    return OriginalType.TIME_MILLIS;
  } else if (logicalType instanceof LogicalTypes.TimeMicros) {
    return OriginalType.TIME_MICROS;
  } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
    return OriginalType.TIMESTAMP_MILLIS;
  } else if (logicalType instanceof LogicalTypes.TimestampMicros) {
    return OriginalType.TIMESTAMP_MICROS;
  }
  return null;
}
 

private static List<Object> convertRepeatedField(
    Schema schema, TableFieldSchema fieldSchema, Object v) {
  Type arrayType = schema.getType();
  verify(
      arrayType == Type.ARRAY,
      "BigQuery REPEATED field %s should be Avro ARRAY, not %s",
      fieldSchema.getName(),
      arrayType);
  // REPEATED fields are represented as Avro arrays.
  if (v == null) {
    // Handle the case of an empty repeated field.
    return new ArrayList<>();
  }
  @SuppressWarnings("unchecked")
  List<Object> elements = (List<Object>) v;
  ArrayList<Object> values = new ArrayList<>();
  Type elementType = schema.getElementType().getType();
  LogicalType elementLogicalType = schema.getElementType().getLogicalType();
  for (Object element : elements) {
    values.add(convertRequiredField(elementType, elementLogicalType, fieldSchema, element));
  }
  return values;
}
 

private <D> Object convert(Schema schema, LogicalType logicalType,
    Conversion<D> conversion, Object datum) {
  if (conversion == null) {
    return datum;
  }
  Class<D> fromClass = conversion.getConvertedType();
  switch (schema.getType()) {
    case RECORD:  return conversion.toRecord(fromClass.cast(datum), schema, logicalType);
    case ENUM:    return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType);
    case ARRAY:   return conversion.toArray(fromClass.cast(datum), schema, logicalType);
    case MAP:     return conversion.toMap(fromClass.cast(datum), schema, logicalType);
    case FIXED:   return conversion.toFixed(fromClass.cast(datum), schema, logicalType);
    case STRING:  return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType);
    case BYTES:   return conversion.toBytes(fromClass.cast(datum), schema, logicalType);
    case INT:     return conversion.toInt(fromClass.cast(datum), schema, logicalType);
    case LONG:    return conversion.toLong(fromClass.cast(datum), schema, logicalType);
    case FLOAT:   return conversion.toFloat(fromClass.cast(datum), schema, logicalType);
    case DOUBLE:  return conversion.toDouble(fromClass.cast(datum), schema, logicalType);
    case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType);
    default: break;
  }
  return datum;
}
 

/**
 * Checks {@link TalendType#convertFromAvro(Schema)} throws {@link UnsupportedOperationException} with following message
 * "Unrecognized type",
 * when unknown logical type is passed
 */
@Test
public void testConvertFromAvroUnsupportedLogicalType() {
    thrown.expect(UnsupportedOperationException.class);
    thrown.expectMessage("Unrecognized type unsupported");
    LogicalType unsupported = new LogicalType("unsupported");
    Schema fieldSchema = unsupported.addToSchema(AvroUtils._string());
    TalendType.convertFromAvro(fieldSchema);
}
 

@Override
public GenericFixed toFixed(UUID value, Schema schema, LogicalType type) {
  ByteBuffer buffer = ByteBuffer.allocate(16);
  buffer.order(ByteOrder.BIG_ENDIAN);
  buffer.putLong(value.getMostSignificantBits());
  buffer.putLong(value.getLeastSignificantBits());
  return new GenericData.Fixed(schema, buffer.array());
}
 

private int convertFromDate(Schema schema, Date date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.date()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted / 86400000L);
	} else {
		throw new RuntimeException("Unsupported date type.");
	}
}
 

private long convertFromTimestamp(Schema schema, Timestamp date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.timestampMillis()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		return time + (long) LOCAL_TZ.getOffset(time);
	} else {
		throw new RuntimeException("Unsupported timestamp type.");
	}
}
 

private int convertFromTime(Schema schema, Time date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.timeMillis()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted % 86400000L);
	} else {
		throw new RuntimeException("Unsupported time type.");
	}
}
 

private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 

private int convertFromDate(Schema schema, Date date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.date()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted / 86400000L);
	} else {
		throw new RuntimeException("Unsupported date type.");
	}
}
 

private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 

/**
 * @param srcSchema THe schema to investigate.
 * @return The Date/Time output type that corresponds to the logical type of the schema, or null if none.
 */
public static TypeConverterOutputTypes getDateTimeTypeFromLogicalType(Schema srcSchema) {
    LogicalType srcLogicalType = AvroUtils.unwrapIfNullable(srcSchema).getLogicalType();
    if (srcLogicalType instanceof LogicalTypes.Date) {
        return TypeConverterOutputTypes.Date;
    } else if (srcLogicalType instanceof LogicalTypes.TimeMillis) {
        return TypeConverterOutputTypes.Time;
    } else if (srcLogicalType instanceof LogicalTypes.TimestampMillis) {
        return TypeConverterOutputTypes.DateTime;
    } else {
        return null;
    }
}
 

public static boolean isTimestamptz(Schema schema) {
  LogicalType logicalType = schema.getLogicalType();
  if (logicalType != null && logicalType instanceof LogicalTypes.TimestampMicros) {
    // timestamptz is adjusted to UTC
    Object value = schema.getObjectProp(ADJUST_TO_UTC_PROP);
    if (value instanceof Boolean) {
      return (Boolean) value;
    } else if (value instanceof String) {
      return Boolean.parseBoolean((String) value);
    }
  }

  return false;
}
 

public static boolean isTimestamptz(Schema schema) {
  LogicalType logicalType = schema.getLogicalType();
  if (logicalType != null && logicalType instanceof LogicalTypes.TimestampMicros) {
    // timestamptz is adjusted to UTC
    Object value = schema.getObjectProp(ADJUST_TO_UTC_PROP);
    if (value instanceof Boolean) {
      return (Boolean) value;
    } else if (value instanceof String) {
      return Boolean.parseBoolean((String) value);
    }
  }

  return false;
}
 

@Override
public UUID fromFixed(GenericFixed value, Schema schema, LogicalType type) {
  ByteBuffer buffer = ByteBuffer.wrap(value.bytes());
  buffer.order(ByteOrder.BIG_ENDIAN);
  long mostSigBits = buffer.getLong();
  long leastSigBits = buffer.getLong();
  return new UUID(mostSigBits, leastSigBits);
}
 

@Override
public GenericFixed toFixed(UUID value, Schema schema, LogicalType type) {
  ByteBuffer buffer = ByteBuffer.allocate(16);
  buffer.order(ByteOrder.BIG_ENDIAN);
  buffer.putLong(value.getMostSignificantBits());
  buffer.putLong(value.getLeastSignificantBits());
  return new GenericData.Fixed(schema, buffer.array());
}
 

public int convertToSQLType(Schema schema) {
    Schema.Type type = schema.getType(); // The standard Avro Type
    LogicalType logicalType = schema.getLogicalType(); // The logical type for Data by example
    String javaType = schema.getProp(SchemaConstants.JAVA_CLASS_FLAG);  // And the Talend java type if standard Avro type is Union

    if (logicalType == null && type == Schema.Type.UNION) {
        for (Schema s : schema.getTypes()) {
            logicalType = null;
            if (s.getType() != Schema.Type.NULL) {
                type = s.getType();
                javaType = s.getProp(SchemaConstants.JAVA_CLASS_FLAG);
                logicalType = s.getLogicalType();
                if (javaType == null && logicalType == null) {
                    type = s.getType(); // Get Avro type if JAVA_CLASS_FLAG is not defined
                }
                break;
            }
        }
    }

    int sqlType = Types.NULL;
    if (logicalType != null) {
        sqlType = convertAvroLogicialType(logicalType);
    } else if (javaType == null) {
        sqlType = convertRawAvroType(type);
    } else {
        sqlType = convertTalendAvroType(javaType);
    }

    if(this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.containsKey(sqlType)){
        sqlType = this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.get(sqlType);
    }

    return sqlType;
}
 

private static GenericRecord getGenericRecord() {

    LogicalType decimalType =
        LogicalTypes.decimal(Integer.MAX_VALUE)
            .addToSchema(org.apache.avro.Schema.create(Type.BYTES))
            .getLogicalType();
    ByteBuffer encodedDecimal =
        new Conversions.DecimalConversion().toBytes(BIG_DECIMAL, null, decimalType);

    return new GenericRecordBuilder(getAvroSchema())
        .set("bool", true)
        .set("int", 43)
        .set("long", 44L)
        .set("float", (float) 44.1)
        .set("double", (double) 44.2)
        .set("string", new Utf8("string"))
        .set("bytes", ByteBuffer.wrap(BYTE_ARRAY))
        .set("decimal", encodedDecimal)
        .set("timestampMillis", DATE_TIME.getMillis())
        .set("row", getSubGenericRecord("row"))
        .set("array", ImmutableList.of(getSubGenericRecord("array"), getSubGenericRecord("array")))
        .set(
            "map",
            ImmutableMap.of(
                new Utf8("k1"),
                getSubGenericRecord("map"),
                new Utf8("k2"),
                getSubGenericRecord("map")))
        .build();
  }
 

@VisibleForTesting
static Type convertType(Schema.Type avroType, LogicalType logicalType) {
    switch (avroType) {
        case BOOLEAN:
            return BooleanType.BOOLEAN;
        case INT:
            if (logicalType == LogicalTypes.timeMillis()) {
                return TIME;
            } else if (logicalType == LogicalTypes.date()) {
                return DATE;
            }
            return IntegerType.INTEGER;
        case LONG:
            if (logicalType == LogicalTypes.timestampMillis()) {
                return TIMESTAMP;
            }
            return BigintType.BIGINT;
        case FLOAT:
            return RealType.REAL;
        case DOUBLE:
            return DoubleType.DOUBLE;
        case BYTES:
            return VarbinaryType.VARBINARY;
        case STRING:
            return VarcharType.VARCHAR;
        case ENUM:
            return VarcharType.VARCHAR;
        default:
            log.error("Cannot convert type: %s", avroType);
            return null;
    }
}