Java Code Examples for org.apache.avro.Schema#create()

@Test
public void testWriteUUIDReadStringSchema() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema stringSchema = record("R", field("uuid", stringUuidSchema));
  GenericRecord s1 = instance(stringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(stringSchema, "uuid", u2.get("uuid").toString());

  File test = write(GENERIC, uuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, stringSchema, test));
}
 

public void updateOutputSchemas() {
    Schema inputSchema = main.schema.getValue();

    schemaFlow.schema.setValue(inputSchema);

    final List<Schema.Field> additionalRejectFields = new ArrayList<Schema.Field>();

    Schema.Field field = new Schema.Field("errorCode", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalRejectFields.add(field);

    field = new Schema.Field("errorMessage", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalRejectFields.add(field);

    Schema rejectSchema = CommonUtils.newSchema(inputSchema, "rejectOutput", additionalRejectFields);

    schemaReject.schema.setValue(rejectSchema);
}
 

private Schema createTypeSchema(XSTypeDefinition type, boolean optional, boolean array) {
    typeLevel++;
    Schema schema;

    if (type.getTypeCategory() == XSTypeDefinition.SIMPLE_TYPE)
        schema = Schema.create(getPrimitiveType((XSSimpleTypeDefinition) type));
    else {
        String name = complexTypeName(type);
        debug("Creating schema for type " + name);

        schema = schemas.get(name);
        if (schema == null) schema = createRecordSchema(name, (XSComplexTypeDefinition) type);
    }

    if (array || isGroupTypeWithMultipleOccurs(type))
        schema = Schema.createArray(schema);
    else if (optional) {
        Schema nullSchema = Schema.create(Schema.Type.NULL);
        schema = Schema.createUnion(Arrays.asList(nullSchema, schema));
    }

    typeLevel--;
    return schema;
}
 

@Test
public void shouldReadMapOfJavaStrings() {
    // given
    Schema stringMapSchema = Schema.createMap(Schema.create(Schema.Type.STRING));
    Schema javaStringSchema = Schema.create(Schema.Type.STRING);
    GenericData.setStringType(javaStringSchema, GenericData.StringType.String);
    Schema javaStringMapSchema = Schema.createMap(javaStringSchema);

    Map<String, String> stringMap = new HashMap<>(0);
    stringMap.put("1", "abc");
    stringMap.put("2", "aaa");

    // when
    Map<Utf8, String> resultJavaStringMap = deserializeGenericFast(stringMapSchema, javaStringMapSchema,
            serializeGeneric(stringMap, javaStringMapSchema));

    // then
    Assert.assertEquals(2, resultJavaStringMap.size());
    Assert.assertEquals("abc", resultJavaStringMap.get(new Utf8("1")));
    Assert.assertEquals("aaa", resultJavaStringMap.get(new Utf8("2")));
}
 

public static Schema getSchema()
{
    Field[] fields =
            {
                    new Schema.Field("colname",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("colvalue",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("code", Schema.create(Type.INT), null, null) };

    Schema schema = Schema.createRecord("dictionary", null, null, false);
    schema.setFields(Arrays.asList(fields));

    return schema;
}
 

@Test
public void shouldReadArrayOfJavaStrings() {
    // given
    Schema javaStringSchema = Schema.create(Schema.Type.STRING);
    GenericData.setStringType(javaStringSchema, GenericData.StringType.String);
    Schema javaStringArraySchema = Schema.createArray(javaStringSchema);

    GenericData.Array<String> javaStringArray = new GenericData.Array<>(0, javaStringArraySchema);
    javaStringArray.add("aaa");
    javaStringArray.add("abc");

    GenericData.Array<String> resultJavaStringArray = deserializeGenericFast(javaStringArraySchema, javaStringArraySchema,
            serializeGeneric(javaStringArray));

    // then
    Assert.assertEquals(2, resultJavaStringArray.size());
    Assert.assertEquals("aaa", resultJavaStringArray.get(0));
    Assert.assertEquals("abc", resultJavaStringArray.get(1));
}
 

@Override
public Schema generate(SourceOfRandomness random, GenerationStatus status) {
  Schema.Type type;

  if (nesting(status) >= MAX_NESTING) {
    type = random.choose(PRIMITIVE_TYPES);
  } else {
    type = random.choose(ALL_TYPES);
  }

  if (PRIMITIVE_TYPES.contains(type)) {
    return Schema.create(type);
  } else {
    nestingInc(status);

    if (type == Schema.Type.FIXED) {
      int size = random.choose(Arrays.asList(1, 5, 12));
      return Schema.createFixed("fixed_" + branch(status), "", "", size);
    } else if (type == Schema.Type.UNION) {
      // only nullable fields, everything else isn't supported in row conversion code
      return UnionSchemaGenerator.INSTANCE.generate(random, status);
    } else if (type == Schema.Type.ENUM) {
      return EnumSchemaGenerator.INSTANCE.generate(random, status);
    } else if (type == Schema.Type.RECORD) {
      return RecordSchemaGenerator.INSTANCE.generate(random, status);
    } else if (type == Schema.Type.MAP) {
      return Schema.createMap(generate(random, status));
    } else if (type == Schema.Type.ARRAY) {
      return Schema.createArray(generate(random, status));
    } else {
      throw new AssertionError("Unexpected AVRO type: " + type);
    }
  }
}
 

private static Schema nullable(final Schema schema) {
    if (schema.getType() == Type.UNION) {
        final List<Schema> unionTypes = new ArrayList<>(schema.getTypes());
        final Schema nullSchema = Schema.create(Type.NULL);
        if (unionTypes.contains(nullSchema)) {
            return schema;
        }

        unionTypes.add(nullSchema);
        return Schema.createUnion(unionTypes);
    }

    return Schema.createUnion(Schema.create(Type.NULL), schema);
}
 

@Test
public void testRecordWithFixed () {
    List<Schema.Field> fields = new ArrayList<>();
    Map<String, Object> data = new HashMap<>();
    for (int i = 0; i < PRIMITIVE_VALUES.length; ++i) {
        Schema.Field field = new Schema.Field(NAMES[i], Schema.create(SCHEMA_TYPES[i]), null, null);
        fields.add(field);
        data.put(NAMES[i], PRIMITIVE_VALUES[i]);
    }

    //add fixed to test case
    fields.add(new Schema.Field("fixed", Schema.createFixed("fixedSchema", null, null, 10), null, null));
    data.put("fixed", "bytes".getBytes());

    //add array to test case
    fields.add(new Schema.Field("array", Schema.createArray(Schema.create(Schema.Type.INT)), null, null));
    List<Integer> integerList = new ArrayList<>();
    integerList.add(1);
    integerList.add(2);
    data.put("array", integerList);

    Schema schema = Schema.createRecord(fields);
    GenericRecord expected = new GenericData.Record(schema);
    for (int i = 0; i < PRIMITIVE_VALUES.length; ++i) {
        expected.put(NAMES[i], PRIMITIVE_VALUES[i]);
    }
    expected.put("fixed", new GenericData.Fixed(Schema.createFixed("fixedSchema", null, null, 10), "bytes".getBytes()));
    expected.put("array", new GenericData.Array<Integer>(Schema.createArray(Schema.create(Schema.Type.INT)), integerList));
    StreamlineEvent streamlineEvent = StreamlineEventImpl.builder().fieldsAndValues(data).dataSourceId("dataSourceId").build();
    Assert.assertEquals(expected, StreamlineEventSerializer.getAvroRecord(streamlineEvent, schema));
}
 

@Test(groups = "deserializationTest")
public void testIsSupportedForFastDeserializer() {
  Set<Schema.Type> supportedSchemaTypes = new HashSet<>();
  supportedSchemaTypes.add(Schema.Type.RECORD);
  supportedSchemaTypes.add(Schema.Type.MAP);
  supportedSchemaTypes.add(Schema.Type.ARRAY);

  Map<Schema.Type, Schema> schemaTypes = new HashMap<>();
  /**
   * Those types could be created by {@link Schema#create(org.apache.avro.Schema.Type)} function.
   */
  schemaTypes.put(Schema.Type.RECORD, Schema.parse("{\"type\": \"record\", \"name\": \"test\", \"fields\":[]}"));
  schemaTypes.put(Schema.Type.MAP, Schema.parse("{\"type\": \"map\", \"values\": \"string\"}"));
  schemaTypes.put(Schema.Type.ARRAY, Schema.parse("{\"type\": \"array\", \"items\": \"string\"}"));
  schemaTypes.put(Schema.Type.ENUM, Schema.parse("{\"type\": \"enum\", \"name\": \"test_enum\", \"symbols\":[]}"));
  schemaTypes.put(Schema.Type.UNION, Schema.parse("[\"null\", \"string\"]"));
  schemaTypes.put(Schema.Type.FIXED, Schema.parse("{\"type\": \"fixed\", \"size\": 16, \"name\": \"test_fixed\"}"));

  for (Schema.Type type : Schema.Type.values()) {
    Schema schema = schemaTypes.containsKey(type) ? schemaTypes.get(type) : Schema.create(type);
    if (supportedSchemaTypes.contains(type)) {
      Assert.assertTrue(FastSerdeCache.isSupportedForFastDeserializer(type));
      FastDeserializerGeneratorBase.getClassName(schema, schema, "");
    } else {
      Assert.assertFalse(FastSerdeCache.isSupportedForFastDeserializer(type));
    }
  }
}
 

/**
 * Converts Talend type to Avro type schema
 * 
 * @param talendType data integration native type
 * @param logicalType avro logical type
 * @return field schema
 * @throws {@link UnsupportedOperationException} in case of unsupported Talend type or logical type
 */
public static Schema convertToAvro(TalendType talendType, String logicalType) {
    Schema fieldSchema = LogicalTypeUtils.getSchemaByLogicalType(logicalType);
    if (fieldSchema != null) {
        return fieldSchema;
    }

    switch (talendType) {
    case STRING:
        return Schema.create(Schema.Type.STRING);
    case BOOLEAN:
        return Schema.create(Schema.Type.BOOLEAN);
    case INTEGER:
        return Schema.create(Schema.Type.INT);
    case LONG:
        return Schema.create(Schema.Type.LONG);
    case DOUBLE:
        return Schema.create(Schema.Type.DOUBLE);
    case FLOAT:
        return Schema.create(Schema.Type.FLOAT);
    case BYTE:
        return AvroUtils._byte();
    case SHORT:
        return AvroUtils._short();
    case CHARACTER:
        return AvroUtils._character();
    case BIG_DECIMAL:
        return AvroUtils._decimal();
    case DATE:
        return AvroUtils._logicalTimestamp();
    default:
        throw new UnsupportedOperationException("Unrecognized type " + talendType);
    }
}
 

@Test
public void shouldWritePrimitives() {
    // given
    Schema javaLangStringSchema = Schema.create(Schema.Type.STRING);
    GenericData.setStringType(javaLangStringSchema, GenericData.StringType.String);
    Schema recordSchema = createRecord("testRecord",
            createField("testInt", Schema.create(Schema.Type.INT)),
            createPrimitiveUnionFieldSchema("testIntUnion", Schema.Type.INT),
            createField("testString", Schema.create(Schema.Type.STRING)),
            createPrimitiveUnionFieldSchema("testStringUnion", Schema.Type.STRING),
            createField("testJavaString", javaLangStringSchema),
            createUnionField("testJavaStringUnion", javaLangStringSchema),
            createField("testLong", Schema.create(Schema.Type.LONG)),
            createPrimitiveUnionFieldSchema("testLongUnion", Schema.Type.LONG),
            createField("testDouble", Schema.create(Schema.Type.DOUBLE)),
            createPrimitiveUnionFieldSchema("testDoubleUnion", Schema.Type.DOUBLE),
            createField("testFloat", Schema.create(Schema.Type.FLOAT)),
            createPrimitiveUnionFieldSchema("testFloatUnion", Schema.Type.FLOAT),
            createField("testBoolean", Schema.create(Schema.Type.BOOLEAN)),
            createPrimitiveUnionFieldSchema("testBooleanUnion", Schema.Type.BOOLEAN),
            createField("testBytes", Schema.create(Schema.Type.BYTES)),
            createPrimitiveUnionFieldSchema("testBytesUnion", Schema.Type.BYTES));

    GenericRecordBuilder builder = new GenericRecordBuilder(recordSchema);
    builder.set("testInt", 1);
    builder.set("testIntUnion", 1);
    builder.set("testString", "aaa");
    builder.set("testStringUnion", "aaa");
    builder.set("testJavaString", "aaa");
    builder.set("testJavaStringUnion", "aaa");
    builder.set("testLong", 1L);
    builder.set("testLongUnion", 1L);
    builder.set("testDouble", 1.0);
    builder.set("testDoubleUnion", 1.0);
    builder.set("testFloat", 1.0f);
    builder.set("testFloatUnion", 1.0f);
    builder.set("testBoolean", true);
    builder.set("testBooleanUnion", true);
    builder.set("testBytes", ByteBuffer.wrap(new byte[]{0x01, 0x02}));
    builder.set("testBytesUnion", ByteBuffer.wrap(new byte[]{0x01, 0x02}));

    // when
    GenericRecord record = deserializeGeneric(recordSchema, serializeGenericFast(builder.build()));

    // then
    Assert.assertEquals(1, record.get("testInt"));
    Assert.assertEquals(1, record.get("testIntUnion"));
    Assert.assertEquals("aaa", record.get("testString").toString());
    Assert.assertEquals("aaa", record.get("testStringUnion").toString());
    Assert.assertEquals("aaa", record.get("testJavaString"));
    Assert.assertEquals("aaa", record.get("testJavaStringUnion"));
    Assert.assertEquals(1L, record.get("testLong"));
    Assert.assertEquals(1L, record.get("testLongUnion"));
    Assert.assertEquals(1.0, record.get("testDouble"));
    Assert.assertEquals(1.0, record.get("testDoubleUnion"));
    Assert.assertEquals(1.0f, record.get("testFloat"));
    Assert.assertEquals(1.0f, record.get("testFloatUnion"));
    Assert.assertEquals(true, record.get("testBoolean"));
    Assert.assertEquals(true, record.get("testBooleanUnion"));
    Assert.assertEquals(ByteBuffer.wrap(new byte[]{0x01, 0x02}), record.get("testBytes"));
    Assert.assertEquals(ByteBuffer.wrap(new byte[]{0x01, 0x02}), record.get("testBytesUnion"));

}
 

private void updateOutputSchemas() {
    final Schema inputSchema = module.main.schema.getValue();

    Schema.Field field = null;

    final List<Schema.Field> additionalMainFields = new ArrayList<Schema.Field>();

    if (OutputAction.UPSERT.equals(outputAction.getValue()) && outputUpsertKey.getValue()
            && inputSchema.getField(upsertKeyColumn.getValue()) == null) {
        field = new Schema.Field("UpsertColumnValue", Schema.create(Schema.Type.STRING), null, (Object) null);
        field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
        field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
        field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
        additionalMainFields.add(field);
    }

    field = new Schema.Field("salesforce_id", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalMainFields.add(field);

    field = new Schema.Field("salesforce_created", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalMainFields.add(field);

    Schema mainOutputSchema = newSchema(inputSchema, "output", additionalMainFields);
    schemaFlow.schema.setValue(mainOutputSchema);

    final List<Schema.Field> additionalRejectFields = new ArrayList<Schema.Field>();

    if (OutputAction.UPSERT.equals(outputAction.getValue()) && outputUpsertKey.getValue()
            && inputSchema.getField(upsertKeyColumn.getValue()) == null) {
        field = new Schema.Field("UpsertColumnValue", Schema.create(Schema.Type.STRING), null, (Object) null);
        field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
        field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
        field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
        additionalRejectFields.add(field);
    }

    field = new Schema.Field("error", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalRejectFields.add(field);

    Schema rejectSchema = newSchema(inputSchema, "rejectOutput", additionalRejectFields);
    schemaReject.schema.setValue(rejectSchema);
}
 

/***
 * Flatten the Schema to un-nest recursive Records (to make it optimal for ORC)
 * @param schema Schema to flatten
 * @param shouldPopulateLineage is set to true if the field is going to be flattened and moved up the hierarchy -
 *                              so that lineage information can be tagged to it; which happens when there is a
 *                              Record within a Record OR Record within Option within Record and so on,
 *                              however not when there is a Record within Map or Array
 * @param flattenComplexTypes Flatten complex types recursively other than Record and Option
 * @return Flattened Avro Schema
 */
private Schema flatten(Schema schema, boolean shouldPopulateLineage, boolean flattenComplexTypes) {
  Schema flattenedSchema;

  // Process all Schema Types
  // (Primitives are simply cloned)
  switch (schema.getType()) {
    case ARRAY:
      // Array might be an array of recursive Records, flatten them
      if (flattenComplexTypes) {
        flattenedSchema = Schema.createArray(flatten(schema.getElementType(), false));
      } else {
        flattenedSchema = Schema.createArray(schema.getElementType());
      }
      break;
    case BOOLEAN:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case BYTES:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case DOUBLE:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case ENUM:
      flattenedSchema =
          Schema.createEnum(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.getEnumSymbols());
      break;
    case FIXED:
      flattenedSchema =
          Schema.createFixed(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.getFixedSize());
      break;
    case FLOAT:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case INT:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case LONG:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case MAP:
      if (flattenComplexTypes) {
        flattenedSchema = Schema.createMap(flatten(schema.getValueType(), false));
      } else {
        flattenedSchema = Schema.createMap(schema.getValueType());
      }
      break;
    case NULL:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case RECORD:
      flattenedSchema = flattenRecord(schema, shouldPopulateLineage, flattenComplexTypes);
      break;
    case STRING:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case UNION:
      flattenedSchema = flattenUnion(schema, shouldPopulateLineage, flattenComplexTypes);
      break;
    default:
      String exceptionMessage = String.format("Schema flattening failed for \"%s\" ", schema);
      LOG.error(exceptionMessage);

      throw new AvroRuntimeException(exceptionMessage);
  }

  // Copy schema metadata
  copyProperties(schema, flattenedSchema);

  return flattenedSchema;
}
 

@Override
protected Schema createStringType() {
  return Schema.create(STRING);
}
 

@Override
public Schema floatP(IList<String> path, String name) {
    return Schema.create(Schema.Type.FLOAT);
}
 

@Override
public Object encodePrim(double value, Object out) {
    return Schema.create(Schema.Type.DOUBLE);
}
 

public static Schema primitive(Schema.Type type) {
  return Schema.create(type);
}
 

@Override
public Schema text(TextNode ignored) {
  return Schema.create(Schema.Type.STRING);
}
 

public AvroMap(Map<Object, Object> map, Schema mapSchema) {
  _map = map;
  _keySchema = Schema.create(STRING);
  _valueSchema = mapSchema.getValueType();
}