Java Code Examples for org.apache.avro.LogicalTypes#fromSchema()

@Override
public void onVisit(Schema schema, Collection<String> breadcrumb) {
  try {
    LogicalTypes.fromSchema(schema);
  } catch (Exception e) {
    String path = breadcrumb.stream().collect(joining("/", "/", ""));
    throw new IllegalArgumentException("Invalid logical type declared at " + path, e);
  }
}
 

private com.google.cloud.spanner.Type inferType(Schema f, boolean supportArrays) {
  Schema.Type type = f.getType();
  LogicalType logicalType = LogicalTypes.fromSchema(f);

  switch (type) {
    case BOOLEAN:
      return Type.bool();
    case INT:
      return com.google.cloud.spanner.Type.int64();
    case LONG:
      if (LogicalTypes.timestampMillis().equals(logicalType)) {
        return com.google.cloud.spanner.Type.timestamp();
      }
      if (LogicalTypes.timestampMicros().equals(logicalType)) {
        return com.google.cloud.spanner.Type.timestamp();
      }
      return com.google.cloud.spanner.Type.int64();
    case FLOAT:
    case DOUBLE:
      return com.google.cloud.spanner.Type.float64();
    case STRING:
      return com.google.cloud.spanner.Type.string();
    case BYTES:
      return com.google.cloud.spanner.Type.bytes();
    case ARRAY:
      {
        if (supportArrays) {
          Schema element = f.getElementType();
          if (element.getType() == Schema.Type.UNION) {
            Schema unpacked = unpackNullable(element);
            if (unpacked == null) {
              throw new IllegalArgumentException("Cannot infer a type " + f);
            }
            element = unpacked;
          }
          try {
            return com.google.cloud.spanner.Type.array(inferType(element, false));
          } catch (IllegalArgumentException e) {
            throw new IllegalArgumentException("Cannot infer array type for field " + f);
          }
        }
        // Throw exception.
        break;
      }
  }
  throw new IllegalArgumentException("Cannot infer a type " + f);
}
 

/**
 * Convert the projected read from delta record into an array writable.
 */
public static Writable avroToArrayWritable(Object value, Schema schema) {

  if (value == null) {
    return null;
  }

  switch (schema.getType()) {
    case STRING:
      return new Text(value.toString());
    case BYTES:
      return new BytesWritable(((ByteBuffer)value).array());
    case INT:
      return new IntWritable((Integer) value);
    case LONG:
      return new LongWritable((Long) value);
    case FLOAT:
      return new FloatWritable((Float) value);
    case DOUBLE:
      return new DoubleWritable((Double) value);
    case BOOLEAN:
      return new BooleanWritable((Boolean) value);
    case NULL:
      return null;
    case RECORD:
      GenericRecord record = (GenericRecord) value;
      Writable[] recordValues = new Writable[schema.getFields().size()];
      int recordValueIndex = 0;
      for (Schema.Field field : schema.getFields()) {
        recordValues[recordValueIndex++] = avroToArrayWritable(record.get(field.name()), field.schema());
      }
      return new ArrayWritable(Writable.class, recordValues);
    case ENUM:
      return new Text(value.toString());
    case ARRAY:
      GenericArray arrayValue = (GenericArray) value;
      Writable[] arrayValues = new Writable[arrayValue.size()];
      int arrayValueIndex = 0;
      for (Object obj : arrayValue) {
        arrayValues[arrayValueIndex++] = avroToArrayWritable(obj, schema.getElementType());
      }
      // Hive 1.x will fail here, it requires values2 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, arrayValues);
    case MAP:
      Map mapValue = (Map) value;
      Writable[] mapValues = new Writable[mapValue.size()];
      int mapValueIndex = 0;
      for (Object entry : mapValue.entrySet()) {
        Map.Entry mapEntry = (Map.Entry) entry;
        Writable[] nestedMapValues = new Writable[2];
        nestedMapValues[0] = new Text(mapEntry.getKey().toString());
        nestedMapValues[1] = avroToArrayWritable(mapEntry.getValue(), schema.getValueType());
        mapValues[mapValueIndex++] = new ArrayWritable(Writable.class, nestedMapValues);
      }
      // Hive 1.x will fail here, it requires values3 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, mapValues);
    case UNION:
      List<Schema> types = schema.getTypes();
      if (types.size() != 2) {
        throw new IllegalArgumentException("Only support union with 2 fields");
      }
      Schema s1 = types.get(0);
      Schema s2 = types.get(1);
      if (s1.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s2);
      } else if (s2.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s1);
      } else {
        throw new IllegalArgumentException("Only support union with null");
      }
    case FIXED:
      if (schema.getLogicalType() != null && schema.getLogicalType().getName().equals("decimal")) {
        LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) LogicalTypes.fromSchema(schema);
        HiveDecimalWritable writable = new HiveDecimalWritable(((GenericFixed) value).bytes(),
            decimal.getScale());
        return HiveDecimalWritable.enforcePrecisionScale(writable,
            decimal.getPrecision(),
            decimal.getScale());
      }
      return new BytesWritable(((GenericFixed) value).bytes());
    default:
      return null;
  }
}
 

/** Converts AVRO schema to Beam field. */
private static Schema.FieldType toFieldType(TypeWithNullability type) {
  Schema.FieldType fieldType = null;
  org.apache.avro.Schema avroSchema = type.type;

  LogicalType logicalType = LogicalTypes.fromSchema(avroSchema);
  if (logicalType != null) {
    if (logicalType instanceof LogicalTypes.Decimal) {
      fieldType = FieldType.DECIMAL;
    } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
      // TODO: There is a desire to move Beam schema DATETIME to a micros representation. When
      // this is done, this logical type needs to be changed.
      fieldType = FieldType.DATETIME;
    } else if (logicalType instanceof LogicalTypes.Date) {
      fieldType = FieldType.DATETIME;
    }
  }

  if (fieldType == null) {
    switch (type.type.getType()) {
      case RECORD:
        fieldType = Schema.FieldType.row(toBeamSchema(avroSchema));
        break;

      case ENUM:
        fieldType = FieldType.logicalType(EnumerationType.create(type.type.getEnumSymbols()));
        break;

      case ARRAY:
        Schema.FieldType elementType =
            toFieldType(new TypeWithNullability(avroSchema.getElementType()));
        fieldType = Schema.FieldType.array(elementType);
        break;

      case MAP:
        fieldType =
            Schema.FieldType.map(
                Schema.FieldType.STRING,
                toFieldType(new TypeWithNullability(avroSchema.getValueType())));
        break;

      case FIXED:
        fieldType = FixedBytesField.fromAvroType(type.type).toBeamType();
        break;

      case STRING:
        fieldType = Schema.FieldType.STRING;
        break;

      case BYTES:
        fieldType = Schema.FieldType.BYTES;
        break;

      case INT:
        fieldType = Schema.FieldType.INT32;
        break;

      case LONG:
        fieldType = Schema.FieldType.INT64;
        break;

      case FLOAT:
        fieldType = Schema.FieldType.FLOAT;
        break;

      case DOUBLE:
        fieldType = Schema.FieldType.DOUBLE;
        break;

      case BOOLEAN:
        fieldType = Schema.FieldType.BOOLEAN;
        break;

      case UNION:
        throw new IllegalArgumentException("Union types not yet supported");

      case NULL:
        throw new IllegalArgumentException("Can't convert 'null' to FieldType");

      default:
        throw new AssertionError("Unexpected AVRO Schema.Type: " + avroSchema.getType());
    }
  }
  fieldType = fieldType.withNullable(type.nullable);
  return fieldType;
}
 

/**
 * Strict conversion from AVRO to Beam, strict because it doesn't do widening or narrowing during
 * conversion.
 *
 * @param value {@link GenericRecord} or any nested value
 * @param avroSchema schema for value
 * @param fieldType target beam field type
 * @return value converted for {@link Row}
 */
@SuppressWarnings("unchecked")
@Nullable
public static Object convertAvroFieldStrict(
    @Nullable Object value,
    @Nonnull org.apache.avro.Schema avroSchema,
    @Nonnull Schema.FieldType fieldType) {
  if (value == null) {
    return null;
  }

  TypeWithNullability type = new TypeWithNullability(avroSchema);
  LogicalType logicalType = LogicalTypes.fromSchema(type.type);
  if (logicalType != null) {
    if (logicalType instanceof LogicalTypes.Decimal) {
      ByteBuffer byteBuffer = (ByteBuffer) value;
      BigDecimal bigDecimal =
          new Conversions.DecimalConversion()
              .fromBytes(byteBuffer.duplicate(), type.type, logicalType);
      return convertDecimal(bigDecimal, fieldType);
    } else if (logicalType instanceof LogicalTypes.TimestampMillis) {
      if (value instanceof ReadableInstant) {
        return convertDateTimeStrict(((ReadableInstant) value).getMillis(), fieldType);
      } else {
        return convertDateTimeStrict((Long) value, fieldType);
      }
    } else if (logicalType instanceof LogicalTypes.Date) {
      if (value instanceof ReadableInstant) {
        int epochDays = Days.daysBetween(Instant.EPOCH, (ReadableInstant) value).getDays();
        return convertDateStrict(epochDays, fieldType);
      } else {
        return convertDateStrict((Integer) value, fieldType);
      }
    }
  }

  switch (type.type.getType()) {
    case FIXED:
      return convertFixedStrict((GenericFixed) value, fieldType);

    case BYTES:
      return convertBytesStrict((ByteBuffer) value, fieldType);

    case STRING:
      return convertStringStrict((CharSequence) value, fieldType);

    case INT:
      return convertIntStrict((Integer) value, fieldType);

    case LONG:
      return convertLongStrict((Long) value, fieldType);

    case FLOAT:
      return convertFloatStrict((Float) value, fieldType);

    case DOUBLE:
      return convertDoubleStrict((Double) value, fieldType);

    case BOOLEAN:
      return convertBooleanStrict((Boolean) value, fieldType);

    case RECORD:
      return convertRecordStrict((GenericRecord) value, fieldType);

    case ENUM:
      // enums are either Java enums, or GenericEnumSymbol,
      // they don't share common interface, but override toString()
      return convertEnumStrict(value, fieldType);

    case ARRAY:
      return convertArrayStrict((List<Object>) value, type.type.getElementType(), fieldType);

    case MAP:
      return convertMapStrict(
          (Map<CharSequence, Object>) value, type.type.getValueType(), fieldType);

    case UNION:
      throw new IllegalArgumentException("Union types not yet supported");

    case NULL:
      throw new IllegalArgumentException("Can't convert 'null' to non-nullable field");

    default:
      throw new AssertionError("Unexpected AVRO Schema.Type: " + type.type.getType());
  }
}