Java Code Examples for org.apache.avro.Schema#Field

@Test
public void nestedRecursiveRecords() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:complexType name='type'>" +
            "    <xs:sequence>" +
            "      <xs:element name='node' type='type' minOccurs='0'/>" +
            "    </xs:sequence>" +
            "  </xs:complexType>" +
            "  <xs:element name='root' type='type'/>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);

    Schema.Field field = schema.getField("node");
    Schema subSchema = field.schema();
    assertSame(schema, subSchema.getTypes().get(1));

    String xml = "<root><node></node></root>";
    GenericData.Record record = Converter.createDatum(schema, xml);

    GenericData.Record child = (GenericData.Record) record.get("node");
    assertEquals(record.getSchema(), child.getSchema());

    assertNull(child.get("node"));
}
 

@Test
public void testNotAllowNullSchema() throws JSONException {
    JSONSchema<Foo> jsonSchema = JSONSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).withAlwaysAllowNull(false).build());
    Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON);
    Schema.Parser parser = new Schema.Parser();
    String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema());
    assertJSONEqual(schemaJson, SCHEMA_JSON_NOT_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 

@Test
public void testSetupSchema_serialization() {
    PythonRowProperties properties = new PythonRowProperties("test");
    properties.init();
    AvroRegistry registry = new AvroRegistry();
    Schema stringSchema = registry.getConverter(String.class).getSchema();
    Schema.Field inputValue1Field = new Schema.Field("inputValue1", stringSchema, null, null, Order.ASCENDING);
    Schema.Field inputValue2Field = new Schema.Field("inputValue2", stringSchema, null, null, Order.ASCENDING);
    Schema inputSchema = Schema.createRecord("inputSchema", null, null, false,
            Arrays.asList(inputValue1Field, inputValue2Field));
    properties.main.schema.setValue(inputSchema);

    properties = Helper.fromSerializedPersistent(properties.toSerialized(), PythonRowProperties.class).object;

    assertThat(inputSchema, equalTo(properties.main.schema.getValue()));
}
 

@Test
public void testInferSchemaWithReferenceField() {
    Field referenceField = new Field();
    referenceField.setName("reference");
    referenceField.setType(FieldType.string);
    referenceField.setReferenceTo(new String[]{"SomeRecord"});
    referenceField.setRelationshipName("relationship");

    DescribeSObjectResult dsor = new DescribeSObjectResult();
    dsor.setName("MySObjectRecord");
    dsor.setFields(new Field[] { referenceField });

    Schema schema = SalesforceAvroRegistryString.get().inferSchema(dsor);

    Schema.Field field = schema.getField("reference");

    assertThat(field.schema().getType(), is(Schema.Type.STRING));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_MODULE_NAME), is("SomeRecord"));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_FIELD_NAME), is("relationship"));
}
 

protected void populateRowData(IndexedRecord input,
        List<Schema.Field> recordFields, List<Schema.Field> remoteFields) {
    for (int i = 0, j = 0; i < row.length && j < remoteFields.size(); j++) {
        Field f = recordFields.get(j);
        Field remoteTableField = remoteFields.get(j);
        if (f == null) {
            if (Boolean.valueOf(remoteTableField.getProp(SnowflakeAvroRegistry.TALEND_FIELD_AUTOINCREMENTED))) {
                continue;
            }
            Object defaultValue = remoteTableField.defaultVal();
            row[i] = StringUtils.EMPTY.equals(defaultValue) ? null : defaultValue;
        } else {
            Object inputValue = input.get(f.pos());
            row[i] = getFieldValue(inputValue, remoteTableField);
        }
        i++;
    }

    loader.submitRow(row);
}
 

@Test
public void testAllowNullSchema() throws JSONException {
    AvroSchema<Foo> avroSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO);
    Schema.Parser parser = new Schema.Parser();
    parser.setValidateDefaults(false);
    String schemaJson = new String(avroSchema.getSchemaInfo().getSchema());
    assertJSONEquals(schemaJson, SCHEMA_AVRO_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 

@Test
public void testSearch() throws Exception {
    NetSuiteInputProperties properties = new NetSuiteInputProperties("test");
    properties.init();
    properties.connection.referencedComponent.componentInstanceId.setValue(CONNECTION_COMPONENT_ID);
    properties.connection.referencedComponent.setReference(connectionProperties);
    properties.module.moduleName.setValue("Account");

    NetSuiteRuntimeImpl runtime = new NetSuiteRuntimeImpl();
    NetSuiteDatasetRuntime dataSetRuntime = runtime.getDatasetRuntime(CONTAINER, properties);
    Schema schema = dataSetRuntime.getSchema(properties.module.moduleName.getValue());
    properties.module.main.schema.setValue(schema);

    properties.module.searchQuery.field.setValue(Arrays.asList("Type"));
    properties.module.searchQuery.operator.setValue(Arrays.asList("List.anyOf"));
    properties.module.searchQuery.value1.setValue(Arrays.<Object>asList("Bank"));
    properties.module.searchQuery.value2.setValue(Arrays.<Object>asList((String) null));

    NetSuiteSource source = new NetSuiteSourceImpl();
    source.initialize(CONTAINER, properties);

    NetSuiteSearchInputReader reader = (NetSuiteSearchInputReader) source.createReader(CONTAINER);
    boolean started = reader.start();
    assertTrue(started);

    IndexedRecord record = reader.getCurrent();
    assertNotNull(record);

    List<Schema.Field> fields = record.getSchema().getFields();
    for (int i = 0; i < fields.size(); i++) {
        Schema.Field typeField = getFieldByName(fields, "AcctType");
        Object value = record.get(typeField.pos());
        assertNotNull(value);
        assertEquals(AccountType.BANK.value(), value);
    }
}
 

public static Schema.Field getField(Schema schema, String fieldName) {
    if (schema == null) {
        return null;
    }

    for (Schema.Field outField : schema.getFields()) {
        if (outField.name().equals(fieldName)) {
            return outField;
        }
    }

    return null;
}
 

@Test
public void testSyncLeadSOAPFail() throws Exception {
    props = getSOAPProperties();
    List<Field> fields = new ArrayList<>();
    Field field = new Schema.Field("Id", Schema.create(Type.INT), null, (Object) null);
    fields.add(field);
    field = new Schema.Field("Email", Schema.create(Schema.Type.STRING), null, (Object) null);
    fields.add(field);
    Schema s = MarketoUtils.newSchema(MarketoConstants.getEmptySchema(), "leadAttribute", fields);
    props.schemaInput.schema.setValue(s);
    props.updateOutputSchemas();
    props.beforeMappingInput();
    IndexedRecord record = new GenericData.Record(s);
    record.put(0, 12345);
    record.put(1, "[email protected]");
    props.outputOperation.setValue(OutputOperation.syncLead);
    try {
        writer = getWriter(props);
        writer.open("testDieOnError");
        writer.write(record);
    } catch (IOException e) {
        assertNotNull(e);
        assertTrue(e.getMessage().contains("20103"));
    }
    props.dieOnError.setValue(false);
    writer = getWriter(props);
    writer.open("test");
    writer.write(record);
    MarketoResult result = (MarketoResult) writer.close();
    assertEquals(1, result.getApiCalls());
    assertEquals(0, result.getSuccessCount());
    assertEquals(1, result.getRejectCount());
    assertEquals("failed", writer.getRejectedWrites().get(0).get(2).toString());
    assertEquals(Collections.emptyList(), writer.getSuccessfulWrites());
}
 

private boolean isLegacySchema( Schema schema ) {
  if ( schema.getFields().size() > 0 ) {
    Schema.Field field = schema.getFields().get( 0 );
    return field != null && field.name() != null && field.name()
      .contains( PentahoAvroInputFormat.FieldName.FIELDNAME_DELIMITER );
  } else {
    return false;
  }
}
 

private static HiveAttribute convertAvroSchemaFieldToHiveAttribute(Schema.Field field) {
  String avroFieldType = field.schema().getType().toString();
  if (avroFieldType.equalsIgnoreCase("UNION")) {
    avroFieldType = extractAvroTypeFromUnion(field);
  }
  if (HiveAttribute.fromAvroType(avroFieldType) == null) {
    throw new RuntimeException("Hive does not support attribute type '" + avroFieldType + "'");
  }
  return new HiveAttribute(field.name(), HiveAttribute.fromAvroType(avroFieldType));
}
 

public static <T> T visit(Schema schema, AvroSchemaVisitor<T> visitor) {
  switch (schema.getType()) {
    case RECORD:
      // check to make sure this hasn't been visited before
      String name = schema.getFullName();
      Preconditions.checkState(!visitor.recordLevels.contains(name),
          "Cannot process recursive Avro record %s", name);

      visitor.recordLevels.push(name);

      List<Schema.Field> fields = schema.getFields();
      List<String> names = Lists.newArrayListWithExpectedSize(fields.size());
      List<T> results = Lists.newArrayListWithExpectedSize(fields.size());
      for (Schema.Field field : schema.getFields()) {
        names.add(field.name());
        results.add(visit(field.schema(), visitor));
      }

      visitor.recordLevels.pop();

      return visitor.record(schema, names, results);

    case UNION:
      List<Schema> types = schema.getTypes();
      List<T> options = Lists.newArrayListWithExpectedSize(types.size());
      for (Schema type : types) {
        options.add(visit(type, visitor));
      }
      return visitor.union(schema, options);

    case ARRAY:
      return visitor.array(schema, visit(schema.getElementType(), visitor));

    case MAP:
      return visitor.map(schema, visit(schema.getValueType(), visitor));

    default:
      return visitor.primitive(schema);
  }
}
 

/**
 * Create a KV-SChema from a schema and a list of keys
 * 
 * @param inputSchema a Schema
 * @param keyPaths the list of key
 * @return a hierarchical KV-Schema
 */
public static Schema extractKeyValues(Schema inputSchema, List<String> keyPaths) {
    List<Schema.Field> fieldList = new ArrayList<>();

    Schema keySchema = extractKeys(inputSchema, keyPaths);
    fieldList.add(new Field(RECORD_KEY_PREFIX, keySchema, "", ""));
    Schema valueSchema = extractValues(inputSchema, keyPaths);
    fieldList.add(new Field(RECORD_VALUE_PREFIX, valueSchema, "", ""));
    return Schema.createRecord(RECORD_KEYVALUE_PREFIX, null, null, false, fieldList);
}
 

private void fillReflect(Object record, String[] data) {
  for (int i = 0; i < indexes.length; i += 1) {
    Schema.Field field = fields[i];
    int index = indexes[i];
    Object value = makeValue(index < data.length ? data[index] : null, field);
    ReflectData.get().setField(record, field.name(), i, value);
  }
}
 

@Override
public ValidationResult validate(RuntimeContainer arg0) {
    for (FilterDescriptor descriptor : filterDescriptors) {
        Schema.Field f = inputSchema.getField(descriptor.getColumnName());
        ValidationResult filteringValidationResult = new FilterPrerequisitesValidator().validate(descriptor.getFunctionType(),
                descriptor.getOperatorType(), AvroUtils.unwrapIfNullable(f.schema()).getType(),
                descriptor.getPredefinedValue());
        if (filteringValidationResult.getStatus() != Result.OK) {
            return filteringValidationResult;
        }
    }
    return ValidationResult.OK;
}
 

BooleanConverter(Schema.Field field) {
    super(field);
}
 

private AvroSpec.DataType findActualDataType( Schema.Field field ) {
  AvroSpec.DataType avroDataType = null;
  LogicalType logicalType = null;
  Schema.Type primitiveAvroType = null;

  if ( field.schema().getType().equals( Schema.Type.UNION ) ) {
    for ( Schema typeSchema : field.schema().getTypes() ) {
      if ( !typeSchema.getType().equals( Schema.Type.NULL ) ) {
        logicalType = typeSchema.getLogicalType();
        primitiveAvroType = typeSchema.getType();
        break;
      }
    }
  } else {
    logicalType = field.schema().getLogicalType();
    primitiveAvroType = field.schema().getType();
  }

  if ( logicalType != null ) {
    for ( AvroSpec.DataType tmpType : AvroSpec.DataType.values() ) {
      if ( !tmpType.isPrimitiveType() && tmpType.getType().equals( logicalType.getName() ) ) {
        avroDataType = tmpType;
        break;
      }
    }
  } else {
    switch ( primitiveAvroType ) {
      case INT:
        avroDataType = AvroSpec.DataType.INTEGER;
        break;
      case LONG:
        avroDataType = AvroSpec.DataType.LONG;
        break;
      case BYTES:
        avroDataType = AvroSpec.DataType.BYTES;
        break;
      case FLOAT:
        avroDataType = AvroSpec.DataType.FLOAT;
        break;
      case DOUBLE:
        avroDataType = AvroSpec.DataType.DOUBLE;
        break;
      case STRING:
        avroDataType = AvroSpec.DataType.STRING;
        break;
      case BOOLEAN:
        avroDataType = AvroSpec.DataType.BOOLEAN;
        break;
    }
  }

  return avroDataType;
}
 

/**
 * Convert a schema with field name to a pig schema
 */
 private static ResourceFieldSchema inconvert(Schema in, String fieldName, Set<Schema> visitedRecords)
         throws IOException {

    AvroStorageLog.details("InConvert avro schema with field name " + fieldName);

    Schema.Type avroType = in.getType();
    ResourceFieldSchema fieldSchema = new ResourceFieldSchema();
    fieldSchema.setName(fieldName);

    if (avroType.equals(Schema.Type.RECORD)) {

        AvroStorageLog.details("convert to a pig tuple");

        if (visitedRecords.contains(in)) {
            fieldSchema.setType(DataType.BYTEARRAY);
        } else {
            visitedRecords.add(in);
            fieldSchema.setType(DataType.TUPLE);
            ResourceSchema tupleSchema = new ResourceSchema();
            List<Schema.Field> fields = in.getFields();
            ResourceFieldSchema[] childFields = new ResourceFieldSchema[fields.size()];
            int index = 0;
            for (Schema.Field field : fields) {
                childFields[index++] = inconvert(field.schema(), field.name(), visitedRecords);
            }

            tupleSchema.setFields(childFields);
            fieldSchema.setSchema(tupleSchema);
            visitedRecords.remove(in);
        }

    } else if (avroType.equals(Schema.Type.ARRAY)) {

        AvroStorageLog.details("convert array to a pig bag");
        fieldSchema.setType(DataType.BAG);
        Schema elemSchema = in.getElementType();
        ResourceFieldSchema subFieldSchema = inconvert(elemSchema, ARRAY_FIELD, visitedRecords);
        add2BagSchema(fieldSchema, subFieldSchema);

    } else if (avroType.equals(Schema.Type.MAP)) {

        AvroStorageLog.details("convert map to a pig map");
        fieldSchema.setType(DataType.MAP);

    } else if (avroType.equals(Schema.Type.UNION)) {

        if (AvroStorageUtils.isAcceptableUnion(in)) {
            Schema acceptSchema = AvroStorageUtils.getAcceptedType(in);
            ResourceFieldSchema realFieldSchema = inconvert(acceptSchema, null, visitedRecords);
            fieldSchema.setType(realFieldSchema.getType());
            fieldSchema.setSchema(realFieldSchema.getSchema());
        } else
            throw new IOException("Do not support generic union:" + in);

    } else if (avroType.equals(Schema.Type.FIXED)) {
         fieldSchema.setType(DataType.BYTEARRAY);
    } else if (avroType.equals(Schema.Type.BOOLEAN)) {
        fieldSchema.setType(DataType.BOOLEAN);
    } else if (avroType.equals(Schema.Type.BYTES)) {
        fieldSchema.setType(DataType.BYTEARRAY);
    } else if (avroType.equals(Schema.Type.DOUBLE)) {
        fieldSchema.setType(DataType.DOUBLE);
    } else if (avroType.equals(Schema.Type.ENUM)) {
        fieldSchema.setType(DataType.CHARARRAY);
    } else if (avroType.equals(Schema.Type.FLOAT)) {
        fieldSchema.setType(DataType.FLOAT);
    } else if (avroType.equals(Schema.Type.INT)) {
        fieldSchema.setType(DataType.INTEGER);
    } else if (avroType.equals(Schema.Type.LONG)) {
        fieldSchema.setType(DataType.LONG);
    } else if (avroType.equals(Schema.Type.STRING)) {
        fieldSchema.setType(DataType.CHARARRAY);
    } else if (avroType.equals(Schema.Type.NULL)) {
        // value of NULL is always NULL
        fieldSchema.setType(DataType.INTEGER);
    } else {
        throw new IOException("Unsupported avro type:" + avroType);
    }
    return fieldSchema;
}
 

@Test
public void simple() {
  DdlToAvroSchemaConverter converter = new DdlToAvroSchemaConverter("spannertest", "booleans");
  Ddl ddl = Ddl.builder()
      .createTable("Users")
      .column("id").int64().notNull().endColumn()
      .column("first_name").string().size(10).endColumn()
      .column("last_name").type(Type.string()).max().endColumn()
      .primaryKey().asc("id").desc("last_name").end()
      .indexes(ImmutableList.of("CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)"))
      .foreignKeys(
          ImmutableList.of(
              "ALTER TABLE `Users` ADD CONSTRAINT `fk` FOREIGN KEY (`first_name`)"
                  + " REFERENCES `AllowedNames` (`first_name`)"))
      .endTable()
      .build();

  Collection<Schema> result = converter.convert(ddl);
  assertThat(result, hasSize(1));
  Schema avroSchema = result.iterator().next();

  assertThat(avroSchema.getNamespace(), equalTo("spannertest"));
  assertThat(avroSchema.getProp("googleFormatVersion"), equalTo("booleans"));
  assertThat(avroSchema.getProp("googleStorage"), equalTo("CloudSpanner"));

  assertThat(avroSchema.getName(), equalTo("Users"));

  List<Schema.Field> fields = avroSchema.getFields();

  assertThat(fields, hasSize(3));

  assertThat(fields.get(0).name(), equalTo("id"));
  // Not null
  assertThat(fields.get(0).schema().getType(), equalTo(Schema.Type.LONG));
  assertThat(fields.get(0).getProp("sqlType"), equalTo("INT64"));

  assertThat(fields.get(1).name(), equalTo("first_name"));
  assertThat(fields.get(1).schema(), equalTo(nullableUnion(Schema.Type.STRING)));
  assertThat(fields.get(1).getProp("sqlType"), equalTo("STRING(10)"));

  assertThat(fields.get(2).name(), equalTo("last_name"));
  assertThat(fields.get(2).schema(), equalTo(nullableUnion(Schema.Type.STRING)));
  assertThat(fields.get(2).getProp("sqlType"), equalTo("STRING(MAX)"));

  // spanner pk
  assertThat(avroSchema.getProp("spannerPrimaryKey_0"), equalTo("`id` ASC"));
  assertThat(avroSchema.getProp("spannerPrimaryKey_1"), equalTo("`last_name` DESC"));
  assertThat(avroSchema.getProp("spannerParent"), nullValue());
  assertThat(avroSchema.getProp("spannerOnDeleteAction"), nullValue());

  assertThat(
      avroSchema.getProp("spannerIndex_0"),
      equalTo("CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)"));
  assertThat(
      avroSchema.getProp("spannerForeignKey_0"),
      equalTo(
          "ALTER TABLE `Users` ADD CONSTRAINT `fk` FOREIGN KEY (`first_name`)"
              + " REFERENCES `AllowedNames` (`first_name`)"));

  System.out.println(avroSchema.toString(true));
}
 

Object getSpecificDefaultValue(Schema.Field field);