Java Code Examples for org.apache.beam.sdk.schemas.Schema#Field

/**
 * Strict conversion from AVRO to Beam, strict because it doesn't do widening or narrowing during
 * conversion. If Schema is not provided, one is inferred from the AVRO schema.
 */
public static Row toBeamRowStrict(GenericRecord record, @Nullable Schema schema) {
  if (schema == null) {
    schema = toBeamSchema(record.getSchema());
  }

  Row.Builder builder = Row.withSchema(schema);
  org.apache.avro.Schema avroSchema = record.getSchema();

  for (Schema.Field field : schema.getFields()) {
    Object value = record.get(field.getName());
    org.apache.avro.Schema fieldAvroSchema = avroSchema.getField(field.getName()).schema();
    builder.addValue(convertAvroFieldStrict(value, fieldAvroSchema, field.getType()));
  }

  return builder.build();
}
 

public static Row castRow(Row input, Schema inputSchema, Schema outputSchema) {
  if (input == null) {
    return null;
  }

  Row.Builder output = Row.withSchema(outputSchema);
  for (int i = 0; i < outputSchema.getFieldCount(); i++) {
    Schema.Field outputField = outputSchema.getField(i);

    int fromFieldIdx = inputSchema.indexOf(outputField.getName());
    Schema.Field inputField = inputSchema.getField(fromFieldIdx);

    Object inputValue = input.getValue(fromFieldIdx);
    Object outputValue = castValue(inputValue, inputField.getType(), outputField.getType());

    output.addValue(outputValue);
  }

  return output.build();
}
 

/** Get Avro Field from Beam Field. */
public static org.apache.avro.Schema.Field toAvroField(Schema.Field field, String namespace) {
  org.apache.avro.Schema fieldSchema =
      getFieldSchema(field.getType(), field.getName(), namespace);
  return new org.apache.avro.Schema.Field(
      field.getName(), fieldSchema, field.getDescription(), (Object) null);
}
 

private static Object[] rowToAvatica(Row row) {
  Schema schema = row.getSchema();
  Object[] convertedColumns = new Object[schema.getFields().size()];
  int i = 0;
  for (Schema.Field field : schema.getFields()) {
    convertedColumns[i] = fieldToAvatica(field.getType(), row.getBaseValue(i, Object.class));
    ++i;
  }
  return convertedColumns;
}
 

/**
 * Convert from a Beam Row to an AVRO GenericRecord. If a Schema is not provided, one is inferred
 * from the Beam schema on the row.
 */
public static GenericRecord toGenericRecord(
    Row row, @Nullable org.apache.avro.Schema avroSchema) {
  Schema beamSchema = row.getSchema();
  // Use the provided AVRO schema if present, otherwise infer an AVRO schema from the row
  // schema.
  if (avroSchema != null && avroSchema.getFields().size() != beamSchema.getFieldCount()) {
    throw new IllegalArgumentException(
        "AVRO schema doesn't match row schema. Row schema "
            + beamSchema
            + ". AVRO schema + "
            + avroSchema);
  }
  if (avroSchema == null) {
    avroSchema = toAvroSchema(beamSchema);
  }

  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  for (int i = 0; i < beamSchema.getFieldCount(); ++i) {
    Schema.Field field = beamSchema.getField(i);
    builder.set(
        field.getName(),
        genericFromBeamField(
            field.getType(), avroSchema.getField(field.getName()).schema(), row.getValue(i)));
  }
  return builder.build();
}
 

/** Converts a Beam Schema into an AVRO schema. */
public static org.apache.avro.Schema toAvroSchema(
    Schema beamSchema, @Nullable String name, @Nullable String namespace) {
  final String schemaName = Strings.isNullOrEmpty(name) ? "topLevelRecord" : name;
  final String schemaNamespace = namespace == null ? "" : namespace;
  String childNamespace =
      !"".equals(schemaNamespace) ? schemaNamespace + "." + schemaName : schemaName;
  List<org.apache.avro.Schema.Field> fields = Lists.newArrayList();
  for (Schema.Field field : beamSchema.getFields()) {
    org.apache.avro.Schema.Field recordField = toAvroField(field, childNamespace);
    fields.add(recordField);
  }
  return org.apache.avro.Schema.createRecord(schemaName, null, schemaNamespace, false, fields);
}
 

@Override
public Integer accept(
    Context context, Optional<Schema.Field> left, Optional<Schema.Field> right) {
  if (left.isPresent() && right.isPresent()) {
    return 1;
  } else {
    return 0;
  }
}
 

Convert(Schema.Field field) {
  Schema.Options options = field.getOptions();
  if (options.hasOption(SCHEMA_OPTION_META_NUMBER)) {
    this.number = options.getValue(SCHEMA_OPTION_META_NUMBER);
  } else {
    this.number = -1;
  }
}
 

/** Generates an insert statement based on {@link Schema.Field}. * */
static String generateStatement(String tableName, List<Schema.Field> fields) {
  String fieldNames =
      IntStream.range(0, fields.size())
          .mapToObj((index) -> fields.get(index).getName())
          .collect(Collectors.joining(", "));

  String valuePlaceholder =
      IntStream.range(0, fields.size())
          .mapToObj((index) -> "?")
          .collect(Collectors.joining(", "));

  return String.format("INSERT INTO %s(%s) VALUES(%s)", tableName, fieldNames, valuePlaceholder);
}
 

private List<Convert> createConverters(Schema schema) {
  List<Convert> fieldOverlays = new ArrayList<>();
  for (Schema.Field field : schema.getFields()) {
    fieldOverlays.add(createConverter(field));
  }
  return fieldOverlays;
}
 

BytesConvert(Schema.Field field) {
  super(field);
}
 

WrapperConvert(Schema.Field field, Convert valueConvert) {
  super(field);
  this.valueConvert = valueConvert;
}
 

TimestampConvert(Schema.Field field) {
  super(field);
}
 

public static Schema.Field toRecordField(Object[] args, int i) {
  return Schema.Field.of((String) args[i + 1], (FieldType) args[i]);
}
 

MapConvert(ProtoDynamicMessageSchema protoSchema, Schema.Field field) {
  super(field);
  Schema.FieldType fieldType = field.getType();
  key = protoSchema.createConverter(Schema.Field.of("KEY", fieldType.getMapKeyType()));
  value = protoSchema.createConverter(Schema.Field.of("VALUE", fieldType.getMapValueType()));
}
 

ArrayConvert(ProtoDynamicMessageSchema protoSchema, Schema.Field field) {
  super(field);
  Schema.FieldType collectionElementType = field.getType().getCollectionElementType();
  this.element = protoSchema.createConverter(Schema.Field.of("ELEMENT", collectionElementType));
}
 

public static Schema.Field toField(RelDataTypeField calciteField) {
  return toField(calciteField.getName(), calciteField.getType());
}
 

private FieldWithIndex(Schema.Field field, Integer index) {
  this.field = field;
  this.index = index;
}
 

static ProtoDynamicMessageSchema<?> forContext(Context context, Schema.Field field) {
  return new ProtoDynamicMessageSchema<>(context.getSubContext(field));
}
 

Schema.Field create(int index, ResultSetMetaData md) throws SQLException;