Java Code Examples for org.apache.avro.Schema#getField()

URI createData(
    URI tableUri,
    Schema schema,
    String hour,
    int id,
    String fieldName,
    Object data) throws IOException {
  GenericData.Record record = new GenericData.Record(schema);
  record.put("id", id);

  if (fieldName != null) {
    Schema.Field field = schema.getField(fieldName);
    Schema fieldSchema = field.schema();
    if (data instanceof Map) {
      GenericData.Record schemaRecord = new GenericData.Record(fieldSchema);
      ((Map<String, String>) data).forEach(schemaRecord::put);
      record.put(fieldName, schemaRecord);
    } else if (data != null) {
      record.put(fieldName, data);
    }
  }

  URI partition = URI.create(tableUri + "/hour=" + hour);
  String path = partition.getPath();
  File parentFolder = new File(path);
  parentFolder.mkdirs();
  File partitionFile = new File(parentFolder, "parquet0000");
  Path filePath = new Path(partitionFile.toURI());
  ParquetWriter<GenericData.Record> writer = AvroParquetWriter.<GenericData.Record>builder(filePath)
      .withSchema(schema)
      .withConf(new Configuration())
      .build();

  try {
    writer.write(record);
  } finally {
    writer.close();
  }
  return partition;
}
 

private void initKACRecordDatumMaps(String fieldName, Schema fieldSchema,
    Schema writtenFieldSchema) {
  Map<String, DatumReader<Object>> recordFieldReaderMap = new HashMap<String, DatumReader<Object>>();
  Map<String, DatumWriter<Object>> recordFieldWriterMap = new HashMap<String, DatumWriter<Object>>();
  kacRecordDatumReaders.put(fieldName, recordFieldReaderMap);
  kacRecordDatumWriters.put(fieldName, recordFieldWriterMap);
  for (Field recordField : fieldSchema.getFields()) {
    Field writtenRecordField = writtenFieldSchema
        .getField(recordField.name());
    if (writtenRecordField == null) {
      continue;
    }
    recordFieldReaderMap.put(recordField.name(),
        buildDatumReader(recordField.schema(), writtenRecordField.schema()));
    recordFieldWriterMap.put(recordField.name(),
        buildDatumWriter(recordField.schema()));
  }
}
 

@Test
public void testAllowNullSchema() throws JSONException {
    AvroSchema<Foo> avroSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO);
    Schema.Parser parser = new Schema.Parser();
    parser.setValidateDefaults(false);
    String schemaJson = new String(avroSchema.getSchemaInfo().getSchema());
    assertJSONEquals(schemaJson, SCHEMA_AVRO_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 

@Test
public void testAllowNullSchema() throws JSONException {
    JSONSchema<Foo> jsonSchema = JSONSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON);
    Schema.Parser parser = new Schema.Parser();
    parser.setValidateDefaults(false);
    String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema());
    assertJSONEqual(schemaJson, SCHEMA_JSON_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 

private static Stream<TableFieldSchema> mapTableFieldSchema(
    TableFieldSchema fieldSchema, Schema avroSchema) {
  Field avroFieldSchema = avroSchema.getField(fieldSchema.getName());
  if (avroFieldSchema == null) {
    return Stream.empty();
  } else if (avroFieldSchema.schema().getType() != Type.RECORD) {
    return Stream.of(fieldSchema);
  }

  List<TableFieldSchema> subSchemas =
      fieldSchema.getFields().stream()
          .flatMap(subSchema -> mapTableFieldSchema(subSchema, avroFieldSchema.schema()))
          .collect(Collectors.toList());

  TableFieldSchema output =
      new TableFieldSchema()
          .setCategories(fieldSchema.getCategories())
          .setDescription(fieldSchema.getDescription())
          .setFields(subSchemas)
          .setMode(fieldSchema.getMode())
          .setName(fieldSchema.getName())
          .setType(fieldSchema.getType());

  return Stream.of(output);
}
 

@Test
public void nestedRecursiveRecords() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:complexType name='type'>" +
            "    <xs:sequence>" +
            "      <xs:element name='node' type='type' minOccurs='0'/>" +
            "    </xs:sequence>" +
            "  </xs:complexType>" +
            "  <xs:element name='root' type='type'/>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);

    Schema.Field field = schema.getField("node");
    Schema subSchema = field.schema();
    assertSame(schema, subSchema.getTypes().get(1));

    String xml = "<root><node></node></root>";
    GenericData.Record record = Converter.createDatum(schema, xml);

    GenericData.Record child = (GenericData.Record) record.get("node");
    assertEquals(record.getSchema(), child.getSchema());

    assertNull(child.get("node"));
}
 

/**
 * given a (parent) schema, and a field name, find the schema for that field.
 * if the field is a union, returns the (only) non-null branch of the union
 * @param parent parent schema containing field
 * @param fieldName name of the field in question
 * @return schema of the field (or non-null union branch thereof)
 */
public static Schema findNonNullUnionBranch(Schema parent, String fieldName) {
  if (parent == null || fieldName == null || fieldName.isEmpty()) {
    throw new IllegalArgumentException("arguments must not be null/empty");
  }
  Schema.Field field = parent.getField(fieldName);
  if (field == null) {
    return null;
  }
  Schema fieldSchema = field.schema();
  Schema.Type fieldSchemaType = fieldSchema.getType();
  if (!Schema.Type.UNION.equals(fieldSchemaType)) {
    return fieldSchema; //field is not a union
  }
  List<Schema> branches = fieldSchema.getTypes();
  List<Schema> nonNullBranches = branches.stream().
    filter(schema -> !Schema.Type.NULL.equals(schema.getType())).collect(Collectors.toList());
  if (nonNullBranches.size() != 1) {
    throw new IllegalArgumentException(String.format("field %s has %d non-null union branches, where exactly 1 is expected in %s",
      fieldName, nonNullBranches.size(), parent));
  }
  return nonNullBranches.get(0);
}
 

@Test
public void testInferSchemaWithReferenceField() {
    Field referenceField = new Field();
    referenceField.setName("reference");
    referenceField.setType(FieldType.string);
    referenceField.setReferenceTo(new String[]{"SomeRecord"});
    referenceField.setRelationshipName("relationship");

    DescribeSObjectResult dsor = new DescribeSObjectResult();
    dsor.setName("MySObjectRecord");
    dsor.setFields(new Field[] { referenceField });

    Schema schema = SalesforceAvroRegistryString.get().inferSchema(dsor);

    Schema.Field field = schema.getField("reference");

    assertThat(field.schema().getType(), is(Schema.Type.STRING));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_MODULE_NAME), is("SomeRecord"));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_FIELD_NAME), is("relationship"));
}
 

@Test
public void testInferSchemaWithReferenceField() {
    Field referenceField = new Field();
    referenceField.setName("reference");
    referenceField.setType(FieldType.string);
    referenceField.setReferenceTo(new String[]{"SomeRecord"});
    referenceField.setRelationshipName("relationship");

    DescribeSObjectResult dsor = new DescribeSObjectResult();
    dsor.setName("MySObjectRecord");
    dsor.setFields(new Field[] { referenceField });

    Schema schema = sRegistry.inferSchema(dsor);

    Schema.Field field = schema.getField("reference");

    assertThat(field.schema().getType(), is(Schema.Type.STRING));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_MODULE_NAME), is("SomeRecord"));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_FIELD_NAME), is("relationship"));
}
 

@Test
  public void testComplexDefaultValue() throws Exception {
    Schema schema = by14.HasComplexDefaults.SCHEMA$;

    Schema.Field field = schema.getField("fieldWithDefaultEnum");
    Object specificDefault = AvroCompatibilityHelper.getSpecificDefaultValue(field);
    Assert.assertNotNull(specificDefault);
    Assert.assertTrue(specificDefault instanceof by14.DefaultEnum);
    Object genericDefault = AvroCompatibilityHelper.getGenericDefaultValue(field);
    Assert.assertNotNull(genericDefault);
    Assert.assertTrue(genericDefault instanceof GenericData.EnumSymbol);

    //avro-1.4 fixed classes cant even be instantiated under avro 1.5+
    //TODO - update this test to use compat generated code
//    field = schema.getField("fieldWithDefaultFixed");
//    specificDefault = AvroCompatibilityHelper.getSpecificDefaultValue(field);
//    Assert.assertNotNull(specificDefault);
//    Assert.assertTrue(specificDefault instanceof by14.DefaultFixed);
//    genericDefault = AvroCompatibilityHelper.getGenericDefaultValue(field);
//    Assert.assertNotNull(genericDefault);
//    Assert.assertTrue(genericDefault instanceof GenericData.Fixed);

    field = schema.getField("fieldWithDefaultRecord");
    specificDefault = AvroCompatibilityHelper.getSpecificDefaultValue(field);
    Assert.assertNotNull(specificDefault);
    Assert.assertTrue(specificDefault instanceof by14.DefaultRecord);
    genericDefault = AvroCompatibilityHelper.getGenericDefaultValue(field);
    Assert.assertNotNull(genericDefault);
    Assert.assertTrue(genericDefault instanceof GenericData.Record);
  }
 

/**
 * Generate a new Index Record which is the filtered result of the input record.
 *
 * The user can freely remove column, add empty column or change the place of column in the same hierarchical level.
 *
 * @return the new record
 */
public static IndexedRecord extractIndexedRecord(IndexedRecord inputRecord, Schema outputSchema) {
    GenericRecordBuilder outputRecord = new GenericRecordBuilder(outputSchema);
    Schema inputSchema = getUnwrappedSchema(inputRecord);
    for (Field field : outputSchema.getFields()) {
        if (inputSchema.getField(field.name()) != null) {
            // The column was existing on the input record, we forward it to the output record.
            Object inputValue = inputRecord.get(inputSchema.getField(field.name()).pos());

            // The current column can be a Record (an hierarchical sub-object) or directly a value.
            // If we are on a record, we need to recursively do the process
            // if we are on a object, we save it to the output.
            if (inputValue instanceof Record) {
                // The sub-schema at this level is a union of "empty" and a record,
                // so we need to get the true sub-schema
                Schema inputChildSchema = getUnwrappedSchema(inputSchema.getField(field.name()));
                Schema outputChildSchema = getUnwrappedSchema(outputSchema.getField(field.name()));
                if (inputChildSchema.getType().equals(Type.RECORD)
                        && outputChildSchema.getType().equals(Type.RECORD)) {
                    Object childRecord = extractIndexedRecord((IndexedRecord) inputValue, outputChildSchema);
                    outputRecord.set(field.name(), childRecord);
                }
            } else {
                outputRecord.set(field.name(), inputValue);
            }
        } else {
            // element not found => set to the value and its hierarchy to null
            outputRecord.set(field.name(), KeyValueUtils.generateEmptyRecord(outputSchema, field.name()));
        }
    }
    return outputRecord.build();
}
 

@Test
public void testIncludeAllFields() throws Exception {
    NetSuiteClientService<?> connection = webServiceTestFixture.getClientService();

    connection.login();

    TypeDesc basicTypeDesc = connection.getBasicMetaData().getTypeInfo("Opportunity");
    Schema schema = getDynamicSchema();

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(connection, schema, basicTypeDesc.getTypeName());

    SearchResultSet<Record> rs = connection.newSearch()
            .target(basicTypeDesc.getTypeName())
            .search();

    TypeDesc typeDesc = connection.getMetaDataSource().getTypeInfo(basicTypeDesc.getTypeName());

    int count = 0;
    while (count++ < connection.getSearchPageSize() && rs.next()) {
        Record record = rs.get();
        IndexedRecord indexedRecord = transducer.read(record);
        logger.debug("Indexed record: {}", indexedRecord);

        Schema recordSchema = indexedRecord.getSchema();
        assertEquals(typeDesc.getFields().size(), recordSchema.getFields().size());

        for (FieldDesc fieldDesc : typeDesc.getFields()) {
            String fieldName = fieldDesc.getName();
            Schema.Field field = recordSchema.getField(fieldName);
            assertNotNull(field);

            Object value = indexedRecord.get(field.pos());
        }
    }
    if (count == 0) {
        throw new IllegalStateException("No records");
    }
}
 

@Test
public void testSignedIntShouldBeInt() throws SQLException, IllegalArgumentException, IllegalAccessException {
    final ResultSetMetaData metadata = mock(ResultSetMetaData.class);
    when(metadata.getColumnCount()).thenReturn(1);
    when(metadata.getColumnType(1)).thenReturn(Types.INTEGER);
    when(metadata.isSigned(1)).thenReturn(true);
    when(metadata.getColumnName(1)).thenReturn("Col1");
    when(metadata.getTableName(1)).thenReturn("Table1");

    final ResultSet rs = mock(ResultSet.class);
    when(rs.getMetaData()).thenReturn(metadata);

    Schema schema = JdbcCommon.createSchema(rs);
    Assert.assertNotNull(schema);

    Schema.Field field = schema.getField("Col1");
    Schema fieldSchema = field.schema();
    Assert.assertEquals(2, fieldSchema.getTypes().size());

    boolean foundIntSchema = false;
    boolean foundNullSchema = false;

    for (Schema type : fieldSchema.getTypes()) {
        if (type.getType().equals(Schema.Type.INT)) {
            foundIntSchema = true;
        } else if (type.getType().equals(Schema.Type.NULL)) {
            foundNullSchema = true;
        }
    }

    assertTrue(foundIntSchema);
    assertTrue(foundNullSchema);
}
 

@Test
public void array() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:element name='root'>" +
            "    <xs:complexType>" +
            "      <xs:sequence>" +
            "        <xs:element name='value' type='xs:string' maxOccurs='unbounded'/>" +
            "      </xs:sequence>" +
            "    </xs:complexType>" +
            "  </xs:element>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);
    Schema.Field valueField = schema.getField("value");
    assertEquals(Schema.Type.ARRAY, valueField.schema().getType());
    assertEquals(Schema.Type.STRING, valueField.schema().getElementType().getType());

    String xml = "<root>" +
                 "  <value>1</value>" +
                 "  <value>2</value>" +
                 "  <value>3</value>" +
                 "</root>";

    GenericData.Record record = Converter.createDatum(schema, xml);
    assertEquals(Arrays.asList("1", "2", "3"), record.get("value"));
}
 

@Override
public void validate(Schema schema) {
  super.validate(schema);
  if (schema.getField(refFieldName) == null) {
    throw new IllegalArgumentException("Invalid field name for reference field: " + refFieldName);
  }
}
 

private void setFieldFromNode(Schema schema, GenericData.Record record, Node node) {
    if (node.getNodeType() != Node.ELEMENT_NODE)
        return;

    Element child = (Element) node;
    boolean setRecordFromNode = false;
    final String fieldName = child.getLocalName();
    Schema.Field field = getFieldBySource(schema, new Source(fieldName, false));
    if(field == null) {
      field = getNestedFieldBySource(schema, new Source(fieldName, false));
      setRecordFromNode = true;
    }

    if (field != null) {
        boolean array = field.schema().getType() == Schema.Type.ARRAY;
      Object datum = createNodeDatum(!array ? field.schema() : field.schema().getElementType(), child, setRecordFromNode);

        if (!array)
            record.put(field.name(), datum);
        else {
            @SuppressWarnings("unchecked") List<Object> values = (List<Object>) record.get(field.name());
            values.add(datum);
        }
    } else {
        Schema.Field anyField = schema.getField(Source.WILDCARD);
        if (anyField == null)
            throw new ConverterException("Could not find field " + fieldName + " in Avro Schema " + schema.getName() +  " , neither as specific field nor 'any' element");

        @SuppressWarnings("unchecked") Map<String, String> map = (HashMap<String, String>) record.get(Source.WILDCARD);
        map.put(fieldName, getContentAsText(child));
    }
}
 

private static GenericRecord convertToGenericRecord(SamzaSqlRelRecord relRecord, Schema schema) {
  GenericRecord record = new GenericData.Record(schema);
  List<String> fieldNames = relRecord.getFieldNames();
  List<Object> values = relRecord.getFieldValues();
  for (int index = 0; index < fieldNames.size(); index++) {
    if (!fieldNames.get(index).equalsIgnoreCase(SamzaSqlRelMessage.KEY_NAME)) {
      String fieldName = fieldNames.get(index);
      /**
       * It is possible that the destination Avro schema doesn't have all the fields that are projected from the
       * SQL. This is especially possible in SQL statements like
       *        insert into kafka.outputTopic select id, company from profile
       * where company is an avro record in itself whose schema can evolve. When this happens we will end up with
       * fields in the SamzaSQLRelRecord for company field which doesn't have equivalent fields in the outputTopic's schema
       * for company. To support this scenario where the input schemas and output schemas can evolve in their own cadence,
       * We ignore the fields which doesn't have corresponding schema in the output topic.
       */
      if (schema.getField(fieldName) == null) {
        LOG.debug("Schema with Name {} and Namespace {} doesn't contain the fieldName {}, Skipping it.",
            schema.getName(), schema.getNamespace(), fieldName);
        continue;
      }
      Object relObj = values.get(index);
      Schema fieldSchema = schema.getField(fieldName).schema();
      record.put(fieldName, convertToAvroObject(relObj, getNonNullUnionSchema(fieldSchema)));
    }
  }

  return record;
}
 

@Override
public Object buildKeyAsColumnField(String fieldName,
    Map<CharSequence, Object> keyAsColumnValues) {
  Schema schema = avroSchema.getAvroSchema();
  Field field = schema.getField(fieldName);
  if (field == null) {
    throw new ValidationException("No field named " + fieldName
        + " in schema " + schema);
  }

  Schema.Type fieldType = field.schema().getType();
  if (fieldType == Schema.Type.MAP) {
    Map<CharSequence, Object> retMap = new HashMap<CharSequence, Object>();
    for (Entry<CharSequence, Object> entry : keyAsColumnValues.entrySet()) {
      retMap.put(entry.getKey(), entry.getValue());
    }
    return retMap;
  } else if (fieldType == Schema.Type.RECORD) {
    AvroRecordBuilder<E> builder = kacRecordBuilderFactories.get(fieldName)
        .getBuilder();
    for (Entry<CharSequence, Object> keyAsColumnEntry : keyAsColumnValues
        .entrySet()) {
      builder.put(keyAsColumnEntry.getKey().toString(),
          keyAsColumnEntry.getValue());
    }
    return builder.build();
  } else {
    throw new ValidationException(
        "Only MAP or RECORD type valid for keyAsColumn fields. Found "
            + fieldType);
  }
}
 

/**
 * Returns true if the types of two avro schemas are equal. This ignores
 * things like custom field properties that the equals() implementation of
 * Schema checks.
 * 
 * @param schema1
 *          The first schema to compare
 * @param schema2
 *          The second schema to compare
 * @return True if the types are equal, otherwise false.
 */
public static boolean avroSchemaTypesEqual(Schema schema1, Schema schema2) {
  if (schema1.getType() != schema2.getType()) {
    // if the types aren't equal, no need to go further. Return false
    return false;
  }

  if (schema1.getType() == Schema.Type.ENUM
      || schema1.getType() == Schema.Type.FIXED) {
    // Enum and Fixed types schemas should be equal using the Schema.equals
    // method.
    return schema1.equals(schema2);
  }
  if (schema1.getType() == Schema.Type.ARRAY) {
    // Avro element schemas should be equal, which is tested by recursively
    // calling this method.
    return avroSchemaTypesEqual(schema1.getElementType(),
        schema2.getElementType());
  } else if (schema1.getType() == Schema.Type.MAP) {
    // Map type values schemas should be equal, which is tested by recursively
    // calling this method.
    return avroSchemaTypesEqual(schema1.getValueType(),
        schema2.getValueType());
  } else if (schema1.getType() == Schema.Type.UNION) {
    // Compare Union fields in the same position by comparing their schemas
    // recursively calling this method.
    if (schema1.getTypes().size() != schema2.getTypes().size()) {
      return false;
    }
    for (int i = 0; i < schema1.getTypes().size(); i++) {
      if (!avroSchemaTypesEqual(schema1.getTypes().get(i), schema2.getTypes()
          .get(i))) {
        return false;
      }
    }
    return true;
  } else if (schema1.getType() == Schema.Type.RECORD) {
    // Compare record fields that match in name by comparing their schemas
    // recursively calling this method.
    if (schema1.getFields().size() != schema2.getFields().size()) {
      return false;
    }
    for (Field field1 : schema1.getFields()) {
      Field field2 = schema2.getField(field1.name());
      if (field2 == null) {
        return false;
      }
      if (!avroSchemaTypesEqual(field1.schema(), field2.schema())) {
        return false;
      }
    }
    return true;
  } else {
    // All other types are primitive, so them matching in type is enough.
    return true;
  }
}
 

/** get field schema given index number */
public static Field getUDField(Schema s, int index) {
    return s.getField(getDummyFieldName(index));
}