Java Code Examples for org.apache.avro.SchemaBuilder#record()
The following examples show how to use
org.apache.avro.SchemaBuilder#record() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: AvroUtils.java From envelope with Apache License 2.0 | 6 votes |
|
private static Schema schemaFor(StructType structType, String record, String namespace, int recordCount) {
// Increment after using the recordCount
record = (null == record) ? "record" + recordCount++ : record;
LOG.debug("Converting {} to Avro Record schema [{}:{}]", structType, record, namespace);
SchemaBuilder.RecordBuilder<Schema> schema = SchemaBuilder.record(record);
if (null != namespace) {
schema.namespace(namespace);
}
schema.doc("Auto-generated from Spark DataFrame");
SchemaBuilder.FieldAssembler<Schema> assembler = schema.fields();
StructField[] structFields = structType.fields();
for (StructField f : structFields) {
assembler.name(f.name()).type(typeFor(f.dataType(), f.nullable(), recordCount)).noDefault();
}
return assembler.endRecord();
}
Example 2
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 5 votes |
|
@Test
public void booleans() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("booleans");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Boolean columns.
.optionalBoolean("optional_boolean")
.requiredBoolean("required_boolean")
.optionalString("optional_string_boolean")
.requiredString("required_string_boolean");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_boolean` BOOL,"
+ "`required_boolean` BOOL NOT NULL,"
+ "`optional_string_boolean` BOOL,"
+ "`required_string_boolean` BOOL NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("required_boolean", true)
.set("optional_boolean", false)
.set("required_string_boolean", "FALSE")
.set("optional_string_boolean", "TRUE")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("required_boolean", false)
.set("optional_boolean", true)
.set("required_string_boolean", "true")
.set("optional_string_boolean", "f")
.build()));
}
Example 3
| Source File: ThirdeyeAvroUtils.java From incubator-pinot with Apache License 2.0 | 5 votes |
|
/**
* Constructs an avro schema from a pinot schema
* @param schema
* @return
*/
public static Schema constructAvroSchemaFromPinotSchema(org.apache.pinot.common.data.Schema schema) {
Schema avroSchema = null;
RecordBuilder<Schema> recordBuilder = SchemaBuilder.record("record");
FieldAssembler<Schema> fieldAssembler = recordBuilder.fields();
for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
String fieldName = fieldSpec.getName();
DataType dataType = fieldSpec.getDataType();
BaseFieldTypeBuilder<Schema> baseFieldTypeBuilder = fieldAssembler.name(fieldName).type().nullable();
switch (dataType) {
case BOOLEAN:
fieldAssembler = baseFieldTypeBuilder.booleanType().noDefault();
break;
case DOUBLE:
fieldAssembler = baseFieldTypeBuilder.doubleType().noDefault();
break;
case FLOAT:
fieldAssembler = baseFieldTypeBuilder.floatType().noDefault();
break;
case INT:
fieldAssembler = baseFieldTypeBuilder.intType().noDefault();
break;
case LONG:
fieldAssembler = baseFieldTypeBuilder.longType().noDefault();
break;
case STRING:
fieldAssembler = baseFieldTypeBuilder.stringType().noDefault();
break;
default:
break;
}
}
avroSchema = fieldAssembler.endRecord();
LOGGER.info("Avro Schema {}", avroSchema.toString(true));
return avroSchema;
}
Example 4
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void integers() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("integers");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Integer columns.
.optionalInt("optional_int")
.requiredInt("required_int")
.requiredLong("required_long")
.optionalLong("optional_long")
.optionalString("optional_string_long")
.requiredString("required_string_long");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_int` INT64,"
+ "`required_int` INT64 NOT NULL,"
+ "`optional_long` INT64,"
+ "`required_long` INT64 NOT NULL,"
+ "`optional_string_long` INT64,"
+ "`required_string_long` INT64 NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_int", 1)
.set("optional_long", 2L)
.set("required_long", 3L)
.set("required_int", 4)
.set("optional_string_long", "1000")
.set("required_string_long", "5000")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_int", 10)
.set("optional_long", 20L)
.set("required_long", 30L)
.set("required_int", 40)
.set("optional_string_long", "10000")
.set("required_string_long", "50000")
.build()));
}
Example 5
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void floats() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("floats");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Integer columns.
.optionalInt("optional_int")
.requiredInt("required_int")
.requiredLong("required_long")
.optionalLong("optional_long")
// Floating columns
.optionalFloat("optional_float")
.requiredFloat("required_float")
.requiredDouble("required_double")
.optionalDouble("optional_double")
.optionalString("optional_string_double")
.requiredString("required_string_double");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_int` FLOAT64,"
+ "`required_int` FLOAT64 NOT NULL,"
+ "`optional_long` FLOAT64,"
+ "`required_long` FLOAT64 NOT NULL,"
+ "`optional_float` FLOAT64,"
+ "`required_float` FLOAT64 NOT NULL,"
+ "`optional_double` FLOAT64,"
+ "`required_double` FLOAT64 NOT NULL,"
+ "`optional_string_double` FLOAT64,"
+ "`required_string_double` FLOAT64 NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_int", 1)
.set("required_int", 4)
.set("optional_long", 2L)
.set("required_long", 3L)
.set("optional_float", 2.3f)
.set("required_float", 3.4f)
.set("optional_double", 2.5)
.set("required_double", 3.6)
.set("optional_string_double", "100.30")
.set("required_string_double", "0.1e-3")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_int", 10)
.set("required_int", 40)
.set("optional_long", 20L)
.set("required_long", 30L)
.set("optional_float", 2.03f)
.set("required_float", 3.14f)
.set("optional_double", 2.05)
.set("required_double", 3.16)
.set("optional_string_double", "100.301")
.set("required_string_double", "1.1e-3")
.build()));
}
Example 6
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void strings() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("strings");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Integer columns.
.optionalInt("optional_int")
.requiredInt("required_int")
.requiredLong("required_long")
.optionalLong("optional_long")
// Floating columns
.optionalFloat("optional_float")
.requiredFloat("required_float")
.requiredDouble("required_double")
.optionalDouble("optional_double")
// String columns
.optionalString("optional_string")
.requiredString("required_string");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_int` STRING(10),"
+ "`required_int` STRING(MAX) NOT NULL,"
+ "`optional_long` STRING(MAX),"
+ "`required_long` STRING(MAX) NOT NULL,"
+ "`optional_float` STRING(MAX),"
+ "`required_float` STRING(MAX) NOT NULL,"
+ "`optional_double` STRING(MAX),"
+ "`required_double` STRING(MAX) NOT NULL,"
+ "`optional_string` STRING(MAX),"
+ "`required_string` STRING(30) NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_int", 1)
.set("required_int", 4)
.set("optional_long", 2L)
.set("required_long", 3L)
.set("optional_float", 2.3f)
.set("required_float", 3.4f)
.set("optional_double", 2.5)
.set("required_double", 3.6)
.set("optional_string", "ONE STRING")
.set("required_string", "TWO STRING")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_int", 10)
.set("required_int", 40)
.set("optional_long", 20L)
.set("required_long", 30L)
.set("optional_float", 2.03f)
.set("required_float", 3.14f)
.set("optional_double", 2.05)
.set("required_double", 3.16)
.set("optional_string", null)
.set("required_string", "THE STRING")
.build()));
}
Example 7
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void timestamps() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("timestamps");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Long columns.
.requiredLong("required_long")
.optionalLong("optional_long")
// String columns
.optionalString("optional_string")
.requiredString("required_string");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` ("
+ "`id` INT64 NOT NULL,"
+ "`optional_long` TIMESTAMP,"
+ "`required_long` TIMESTAMP NOT NULL,"
+ "`optional_string` TIMESTAMP,"
+ "`required_string` TIMESTAMP NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(
schema,
spannerSchema,
Arrays.asList(
new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_long", 5000000L)
.set("required_long", 6000000L)
.set("optional_string", "2018-06-06T21:00:35.312000000Z")
.set("required_string", "2018-06-06T21:00:35.312000000Z")
.build(),
new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_long", 500000330L)
.set("required_long", 6000020000L)
.set("optional_string", "2017-06-06T21:00:35.312000000Z")
.set("required_string", "2017-06-06T21:00:35.312000000Z")
.build(),
new GenericRecordBuilder(schema)
.set("id", 3L)
.set("optional_long", null)
.set("required_long", 6000020000L)
.set("optional_string", null)
.set("required_string", "0001-01-01T00:00:00Z")
.build()));
}
