Java Code Examples for org.apache.avro.Schema#addProp()

@Test
public void testWriteNullableUUID() throws IOException {
  Schema nullableUuidSchema = record("R",
      optionalField("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(nullableUuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(nullableUuidSchema, "uuid", null);

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema nullableStringSchema = record("R", optionalField("uuid", stringUuidSchema));
  GenericRecord s1 = instance(nullableStringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(nullableStringSchema, "uuid", null);

  File test = write(GENERIC, nullableUuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, nullableStringSchema, test));
}
 

public void setupProperties() {
    super.setupProperties();

    Schema schema = SchemaBuilder.builder().record(GoogleDriveDeleteDefinition.COMPONENT_NAME).fields() //
            .name(GoogleDriveDeleteDefinition.RETURN_FILE_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
    schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    schemaMain.schema.setValue(schema);

    deleteMode.setPossibleValues(AccessMethod.values());
    deleteMode.setValue(AccessMethod.Name);
    file.setValue("");
    useTrash.setValue(true);
}
 

@Override
protected SchemaReader<GenericRecord> loadReader(BytesSchemaVersion schemaVersion) {
     SchemaInfo schemaInfo = getSchemaInfoByVersion(schemaVersion.get());
     if (schemaInfo != null) {
         log.info("Load schema reader for version({}), schema is : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             schemaInfo);
         Schema writerSchema = parseAvroSchema(schemaInfo.getSchemaDefinition());
         Schema readerSchema = useProvidedSchemaAsReaderSchema ? schema : writerSchema;
         readerSchema.addProp(OFFSET_PROP, schemaInfo.getProperties().getOrDefault(OFFSET_PROP, "0"));

         return new GenericAvroReader(
                 writerSchema,
                 readerSchema,
                 schemaVersion.get());
     } else {
         log.warn("No schema found for version({}), use latest schema : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             this.schemaInfo);
         return reader;
     }
}
 

@Test
public void testCopy() throws CopyNotSupportedException {
  Schema schema = new Schema.Parser().parse(AVRO_SCHEMA);
  CopyableSchema copyableSchema = new CopyableSchema(schema);
  Schema copy = copyableSchema.copy();
  Assert.assertEquals(schema, copy);
  copy.addProp("foo", "bar");
  Assert.assertNotEquals(schema, copy);
}
 

private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}
 

/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}
 

private Schema mapWithIds(Schema map, Integer keyId, Integer valueId) {
  if (!AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.KEY_ID_PROP) ||
      !AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.VALUE_ID_PROP)) {
    Schema result = Schema.createMap(map.getValueType());
    result.addProp(AvroSchemaUtil.KEY_ID_PROP, keyId);
    result.addProp(AvroSchemaUtil.VALUE_ID_PROP, valueId);
    return result;
  }
  return map;
}
 

/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}
 

/**
 * Sets initial value of schema property
 */
void setupSchema() {
    Schema stringSchema = AvroUtils._string();

    // create Schema for JSON
    Schema.Field jsonField = new Schema.Field("json", stringSchema, null, (Object) null, Order.ASCENDING);
    Schema initialSchema = Schema.createRecord("jira", null, null, false, Collections.singletonList(jsonField));
    initialSchema.addProp(TALEND_IS_LOCKED, "true");

    schema.schema.setValue(initialSchema);
}
 

public static Schema triggerCampaignSchemaFlow() {
    Schema s = SchemaBuilder.builder().record("triggerCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}
 

public static Schema scheduleCampaignSchema() {
    Schema s = SchemaBuilder.builder().record("scheduleCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_STATUS).type().nullable().stringType().noDefault()//
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}
 

private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}
 

static Schema addValueId(int id, Schema schema) {
  schema.addProp(AvroSchemaUtil.VALUE_ID_PROP, id);
  return schema;
}
 

public static Schema addSchemaCreationTime(Schema inputSchema, Schema outputSchema) {
  if (inputSchema.getProp(SCHEMA_CREATION_TIME_KEY) != null && outputSchema.getProp(SCHEMA_CREATION_TIME_KEY) == null) {
    outputSchema.addProp(SCHEMA_CREATION_TIME_KEY, inputSchema.getProp(SCHEMA_CREATION_TIME_KEY));
  }
  return outputSchema;
}
 

@Test
public void testConvertToAvroForNestedObjects() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("FieldA").type(AvroUtils._int()).noDefault() //
            .name("Member_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_FieldA").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_FieldA").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("Campaign");
    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");

    // Intentionally add another value to check that it is skipped by converter
    sObject.addField("FieldA", 300);
    sObject.addField("FieldA", 400);

    SObject sObject1 = new SObject("CampaignMember");
    sObject1.addField("Id", "10001");
    sObject1.addField("Name", "Member A");
    sObject1.addField("FieldA", 10);
    sObject.addField("Member", sObject1);

    SObject sObject11 = new SObject("Contact");
    sObject11.addField("Id", "20001");
    sObject11.addField("Name", "Contact A");
    sObject11.addField("FieldA", "foo");
    sObject1.addField("Contact", sObject11);

    SObject sObject2 = new SObject("CampaignMember");
    sObject2.addField("Id", "10002");
    sObject2.addField("Name", "Member B");
    sObject2.addField("FieldA", 20);
    sObject.addField("Member", sObject2);

    SObject sObject21 = new SObject("Contact");
    sObject21.addField("Id", "20002");
    sObject21.addField("Name", "Contact B");
    sObject21.addField("FieldA", "bar");
    sObject2.addField("Contact", sObject21);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(300), indexedRecord.get(2));
    assertEquals("Member A|Member B", indexedRecord.get(3));
    assertEquals("10|20", indexedRecord.get(4));
    assertEquals("Contact A|Contact B", indexedRecord.get(5));
    assertEquals("foo|bar", indexedRecord.get(6));
}
 

private static Schema traverse(Map.Entry<String, HiveTypeInfo> node)
    throws StageException {
  switch(node.getValue().getHiveType()){
    case STRING:
      return Schema.create(Schema.Type.STRING);

    case BOOLEAN:
      return Schema.create(Schema.Type.BOOLEAN);

    case INT:
      return Schema.create(Schema.Type.INT);

    case BIGINT:
      return Schema.create(Schema.Type.LONG);

    case FLOAT:
      return Schema.create(Schema.Type.FLOAT);

    case DOUBLE:
      return Schema.create(Schema.Type.DOUBLE);

    case BINARY:
      return Schema.create(Schema.Type.BYTES);

    case TIMESTAMP:
      Schema timestampSchema = Schema.create(Schema.Type.LONG);
      timestampSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
      return timestampSchema;

    case DATE:
      Schema dateSchema = Schema.create(Schema.Type.INT);
      dateSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
      return dateSchema;

    case DECIMAL:
      Utils.checkArgument(node.getValue() instanceof DecimalTypeInfo, "Invalid type used in HiveTypeInfo");
      DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo)node.getValue();
      Schema schema = Schema.create(Schema.Type.BYTES);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, getJsonNode(decimalTypeInfo.getPrecision()));
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, getJsonNode(decimalTypeInfo.getScale()));
      return schema;

    default:
      // Accessing Unsupported Type (Map, Union, Array, Enum, NULL, Record)
      throw new StageException(Errors.HIVE_24, node.getValue().getHiveType());
  }
}
 

private Schema addAdjustToUtc(Schema schema, boolean adjustToUTC) {
  schema.addProp(AvroSchemaUtil.ADJUST_TO_UTC_PROP, adjustToUTC);
  return schema;
}
 

@Test
public void testConvertToAvroForAggregateResult() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("Field_A").type(AvroUtils._int()).noDefault() //
            .name("Field_B").type(AvroUtils._boolean()).noDefault() //
            .name("Field_C").type(AvroUtils._date()).noDefault() //
            .name("Field_D").type(AvroUtils._double()).noDefault() //
            .name("Field_E_Value").type(AvroUtils._string()).noDefault() //
            .name("Field_F").type(AvroUtils._string()).noDefault() //
            .name("Field_G").type(AvroUtils._string()).noDefault() //
            .name("Field_H").type(AvroUtils._string()).noDefault() //
            .name("Field_I").type(AvroUtils._string()).noDefault() //
            .name("Field_J").type(AvroUtils._string()).noDefault() //
            .name("Field_K").type(AvroUtils._string()).noDefault() //
            .name("Field_L").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.getField("Field_C").addProp(SchemaConstants.TALEND_COLUMN_PATTERN,
            "yyyy-MM-dd'T'HH:mm:ss'.000Z'");
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("AggregateResult");

    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");
    sObject.addField("Field_A", "42");
    sObject.addField("Field_B", "true");
    sObject.addField("Field_C", dateFormat.parse("2017-06-15T18:26:34.000Z"));
    sObject.addField("Field_D", 10200.45);

    SObject sObject1 = new SObject();
    sObject1.addField("Value", 245);
    sObject.addField("Field_E", sObject1);
    SObject sObject2 = new SObject();
    sObject2.addField("Value", 542);
    sObject.addField("Field_E", sObject2);

    sObject.addField("Field_F", new BigDecimal("20000000000000000000000000.123456789"));
    sObject.addField("Field_G", 899.5f);
    sObject.addField("Field_H", Boolean.TRUE);
    sObject.addField("Field_I", new byte[]{0x0a, 0x0b, 0x0c, 0x0d});
    sObject.addField("Field_J", 102030405060708090L);
    sObject.addField("Field_K", SalesforceRuntime.convertDateToCalendar(
            dateFormat.parse("2017-06-16T10:45:02.000Z"),false));
    sObject.addField("Field_L", null);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(42), indexedRecord.get(2));
    assertEquals(Boolean.TRUE, indexedRecord.get(3));
    assertEquals(dateFormat.parse("2017-06-15T18:26:34.000Z").getTime(), indexedRecord.get(4));
    assertEquals(Double.valueOf(10200.45), indexedRecord.get(5));
    assertEquals("245|542", indexedRecord.get(6));
    assertEquals("20000000000000000000000000.123456789", indexedRecord.get(7));
    assertEquals("899.5", indexedRecord.get(8));
    assertEquals("true", indexedRecord.get(9));
    assertEquals("CgsMDQ==", indexedRecord.get(10));
    assertEquals("102030405060708090", indexedRecord.get(11));
    assertEquals("2017-06-16T07:45:02.000Z", indexedRecord.get(12));
    assertNull(indexedRecord.get(13));
}
 

default Schema setProp(final Schema schema, final String name, final String value) {
    schema.addProp("talend.component." + name, value);
    return schema;
}
 

@Test
public void validLogicalType() throws Exception {
  Schema schema = SchemaBuilder.builder().intType();
  schema.addProp(LogicalType.LOGICAL_TYPE_PROP, "date");
  underTest.onVisit(schema, breadcrumb);
}