Java Code Examples for org.apache.avro.Schema#getFixedSize()
The following examples show how to use
org.apache.avro.Schema#getFixedSize() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: MercifulJsonConverter.java From hudi with Apache License 2.0 | 6 votes |
|
private static JsonToAvroFieldProcessor generateFixedTypeHandler() {
return new JsonToAvroFieldProcessor() {
@Override
public Pair<Boolean, Object> convert(Object value, String name, Schema schema) {
// The ObjectMapper use List to represent FixedType
// eg: "decimal_val": [0, 0, 14, -63, -52] will convert to ArrayList<Integer>
List<Integer> converval = (List<Integer>) value;
byte[] src = new byte[converval.size()];
for (int i = 0; i < converval.size(); i++) {
src[i] = converval.get(i).byteValue();
}
byte[] dst = new byte[schema.getFixedSize()];
System.arraycopy(src, 0, dst, 0, Math.min(schema.getFixedSize(), src.length));
return Pair.of(true, new GenericData.Fixed(schema, dst));
}
};
}
Example 2
| Source File: AvroSchemaValidator.java From registry with Apache License 2.0 | 5 votes |
|
private SchemaCompatibilityResult checkFixedSize(final Schema reader,
final Schema writer,
final Stack<String> location) {
location.push("size");
int actual = reader.getFixedSize();
int expected = writer.getFixedSize();
if (actual != expected) {
String message = String.format("expected: %d, found: %d", expected, actual);
return SchemaCompatibilityResult.incompatible(
SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, reader,
writer, message, location);
}
location.pop();
return SchemaCompatibilityResult.compatible();
}
Example 3
| Source File: Generator.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 4 votes |
|
private GenericFixed generateFixed(Schema schema) {
byte[] bytes = new byte[schema.getFixedSize()];
random.nextBytes(bytes);
return new GenericData.Fixed(schema, bytes);
}
Example 4
| Source File: ValueReaders.java From iceberg with Apache License 2.0 | 4 votes |
|
private GenericFixedReader(Schema schema) {
this.schema = schema;
this.length = schema.getFixedSize();
}
Example 5
| Source File: AvroRandomDataGenerator.java From avro-util with BSD 2-Clause "Simplified" License | 4 votes |
|
private GenericFixed generateFixed(Schema schema) {
byte[] bytes = new byte[schema.getFixedSize()];
random.nextBytes(bytes);
return AvroCompatibilityHelper.newFixedField(schema, bytes);
}
Example 6
| Source File: AvroAdapter.java From avro-util with BSD 2-Clause "Simplified" License | 4 votes |
|
default GenericData.Fixed newFixedField(Schema ofType) {
byte[] emptyDataArray = new byte[ofType.getFixedSize()]; //null is probably unsafe
return newFixedField(ofType, emptyDataArray);
}
Example 7
| Source File: ValueReaders.java From iceberg with Apache License 2.0 | 4 votes |
|
private GenericFixedReader(Schema schema) {
this.schema = schema;
this.length = schema.getFixedSize();
}
Example 8
| Source File: Generator.java From avro-random-generator with Do What The F*ck You Want To Public License | 4 votes |
|
private GenericFixed generateFixed(Schema schema) {
byte[] bytes = new byte[schema.getFixedSize()];
random.nextBytes(bytes);
return new GenericData.Fixed(schema, bytes);
}
Example 9
| Source File: GeneratorFunctions.java From components with Apache License 2.0 | 4 votes |
|
public FixedBytesGenerator(Schema fixedSchema) {
size = fixedSchema.getFixedSize();
jsonSchema = fixedSchema.toString();
this.schema = fixedSchema;
}
Example 10
| Source File: PigAvroDatumWriter.java From Cubert with Apache License 2.0 | 4 votes |
|
/**
* Recursively check whether "datum" is an instance of "schema" and called
* by {@link #resolveUnionSchema(Schema,Object)},
* {@link #unwrappedInstanceOf(Schema,Object)}.
*
*/
protected boolean instanceOf(Schema schema, Object datum)
throws IOException {
try {
switch (schema.getType()) {
case RECORD:
if (datum instanceof Tuple) {
Tuple tuple = (Tuple) datum;
List<Field> fields = schema.getFields();
if (fields.size() != tuple.size()) {
return false;
}
for (int i = 0; i < fields.size(); i++) {
if (!instanceOf(fields.get(i).schema(), tuple.get(i)))
return false;
}
return true;
}
return false;
case UNION:
@SuppressWarnings("unused")
int index = resolveUnionSchema(schema, datum);
return true;
case ENUM:
return datum instanceof String && schema.hasEnumSymbol(((String) datum))
|| unwrappedInstanceOf(schema, datum);
case ARRAY:
return datum instanceof DataBag
|| unwrappedInstanceOf(schema, datum);
case MAP:
return datum instanceof Map
|| unwrappedInstanceOf(schema, datum);
case FIXED:
return datum instanceof DataByteArray && ((DataByteArray) datum).size() == schema.getFixedSize()
|| unwrappedInstanceOf(schema, datum);
case STRING:
return datum instanceof String
|| unwrappedInstanceOf(schema, datum);
case BYTES:
return datum instanceof DataByteArray
|| unwrappedInstanceOf(schema, datum);
case INT:
return datum instanceof Integer
|| unwrappedInstanceOf(schema, datum);
case LONG:
return datum instanceof Long
|| datum instanceof Integer
|| unwrappedInstanceOf(schema, datum);
case FLOAT:
return datum instanceof Float
|| datum instanceof Integer
|| datum instanceof Long
|| unwrappedInstanceOf(schema, datum);
case DOUBLE:
return datum instanceof Double
|| datum instanceof Float
|| datum instanceof Integer
|| datum instanceof Long
|| unwrappedInstanceOf(schema, datum);
case BOOLEAN:
return datum instanceof Boolean
|| datum instanceof Integer
|| unwrappedInstanceOf(schema, datum);
case NULL:
return datum == null;
default:
throw new RuntimeException("Unexpected type: " + schema);
}
} catch (ExecException e) {
e.printStackTrace(System.err);
throw new RuntimeException(e);
}
}
Example 11
| Source File: AvroSchemaCheckDefaultStrategy.java From incubator-gobblin with Apache License 2.0 | 4 votes |
|
/**
* This method will compare the name and types of the two schema
* @param expected The expected schema
* @param toValidate The real schema
* @return true when expected schema and toValidate schema have matching field names and types
*/
public boolean compare(Schema expected, Schema toValidate)
{
if (toValidate.getType() != expected.getType() || !toValidate.getName().equals(expected.getName())) {return false;}
else {
switch (toValidate.getType()) {
case NULL:
case BOOLEAN:
case INT:
case LONG:
case FLOAT:
case DOUBLE:
case BYTES:
case STRING: {
return true;
}
case ARRAY: {
return compare(toValidate.getElementType(), expected.getElementType());
}
case MAP: {
return compare(toValidate.getValueType(), expected.getValueType());
}
case FIXED: {
// fixed size and name must match:
if (toValidate.getFixedSize() != expected.getFixedSize()) {
return false;
}
return true;
}
case ENUM: {
// expected symbols must contain all toValidate symbols:
final Set<String> expectedSymbols = new HashSet<>(expected.getEnumSymbols());
final Set<String> toValidateSymbols = new HashSet<String>(toValidate.getEnumSymbols());
if (expectedSymbols.size() != toValidateSymbols.size()) {
return false;
}
if (!expectedSymbols.containsAll(toValidateSymbols)) {
return false;
}
return true;
}
case RECORD: {
// Check that each field of toValidate schema is in expected schema
if (toValidate.getFields().size() != expected.getFields().size()) {
return false;
}
for (final Schema.Field expectedFiled : expected.getFields()) {
final Schema.Field toValidateField = toValidate.getField(expectedFiled.name());
if (toValidateField == null) {
// expected field does not correspond to any field in the toValidate record schema
return false;
} else {
if (!compare(toValidateField.schema(), expectedFiled.schema())) {
return false;
}
}
}
return true;
}
case UNION: {
// Check existing schema contains all the type in toValidate schema
if (toValidate.getTypes().size() != expected.getTypes().size()) {
return false;
}
HashSet<Schema> types = new HashSet<Schema>(expected.getTypes());
for (Schema toValidateType : toValidate.getTypes()) {
Schema equalSchema = null;
for (Schema type : types) {
if (compare(type, toValidateType)) {
equalSchema = type;
break;
}
}
if (equalSchema == null) {
return false;
}
types.remove(equalSchema);
}
return true;
}
default: {
throw new AvroRuntimeException("Unknown schema type: " + toValidate.getType());
}
}
}
}
Example 12
| Source File: PigAvroDatumWriter.java From spork with Apache License 2.0 | 4 votes |
|
/**
* Recursively check whether "datum" is an instance of "schema" and called
* by {@link #resolveUnionSchema(Schema,Object)},
* {@link #unwrappedInstanceOf(Schema,Object)}.
*
*/
protected boolean instanceOf(Schema schema, Object datum)
throws IOException {
try {
switch (schema.getType()) {
case RECORD:
if (datum instanceof Tuple) {
Tuple tuple = (Tuple) datum;
List<Field> fields = schema.getFields();
if (fields.size() != tuple.size()) {
return false;
}
for (int i = 0; i < fields.size(); i++) {
if (!instanceOf(fields.get(i).schema(), tuple.get(i)))
return false;
}
return true;
}
return false;
case UNION:
@SuppressWarnings("unused")
int index = resolveUnionSchema(schema, datum);
return true;
case ENUM:
return datum instanceof String && schema.hasEnumSymbol(((String) datum))
|| unwrappedInstanceOf(schema, datum);
case ARRAY:
return datum instanceof DataBag
|| unwrappedInstanceOf(schema, datum);
case MAP:
return datum instanceof Map
|| unwrappedInstanceOf(schema, datum);
case FIXED:
return datum instanceof DataByteArray && ((DataByteArray) datum).size() == schema.getFixedSize()
|| unwrappedInstanceOf(schema, datum);
case STRING:
return datum instanceof String
|| unwrappedInstanceOf(schema, datum);
case BYTES:
return datum instanceof DataByteArray
|| unwrappedInstanceOf(schema, datum);
case INT:
return datum instanceof Integer
|| unwrappedInstanceOf(schema, datum);
case LONG:
return datum instanceof Long
|| datum instanceof Integer
|| unwrappedInstanceOf(schema, datum);
case FLOAT:
return datum instanceof Float
|| datum instanceof Integer
|| datum instanceof Long
|| unwrappedInstanceOf(schema, datum);
case DOUBLE:
return datum instanceof Double
|| datum instanceof Float
|| datum instanceof Integer
|| datum instanceof Long
|| unwrappedInstanceOf(schema, datum);
case BOOLEAN:
return datum instanceof Boolean
|| datum instanceof Integer
|| unwrappedInstanceOf(schema, datum);
case NULL:
return datum == null;
default:
throw new RuntimeException("Unexpected type: " + schema);
}
} catch (ExecException e) {
e.printStackTrace(System.err);
throw new RuntimeException(e);
}
}
