org.apache.avro.JsonProperties Java Examples

static Schema createProjectionMap(String recordName,
                        int keyId, String keyName, Schema keySchema,
                        int valueId, String valueName, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, (Object) null);
  if (!"key".equals(keyName)) {
    keyField.addAlias(keyName);
  }
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE : null);
  valueField.addProp(FIELD_ID_PROP, valueId);
  if (!"value".equals(valueName)) {
    valueField.addAlias(valueName);
  }

  Schema keyValueRecord = Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField));
  if (!keyValueName.equals(recordName)) {
    keyValueRecord.addAlias(recordName);
  }

  return LogicalMap.get().addToSchema(Schema.createArray(keyValueRecord));
}
 

@Test
public void testArrayOfOptionalRecordsOldBehavior() throws Exception {
  Schema innerRecord = Schema.createRecord("InnerRecord", null, null, false);
  Schema optionalString = optional(Schema.create(Schema.Type.STRING));
  innerRecord.setFields(Lists.newArrayList(
      new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
      new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)
  ));
  Schema schema = Schema.createRecord("HasArray", null, null, false);
  schema.setFields(Lists.newArrayList(
      new Schema.Field("myarray", Schema.createArray(optional(innerRecord)),
          null, null)
  ));
  System.err.println("Avro schema: " + schema.toString(true));

  // Cannot use round-trip assertion because InnerRecord optional is removed
  testAvroToParquetConversion(schema, "message HasArray {\n" +
      "  required group myarray (LIST) {\n" +
      "    repeated group array {\n" +
      "      optional binary s1 (UTF8);\n" +
      "      optional binary s2 (UTF8);\n" +
      "    }\n" +
      "  }\n" +
      "}\n");
}
 

static Schema createProjectionMap(String recordName,
                        int keyId, String keyName, Schema keySchema,
                        int valueId, String valueName, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, null);
  if (!"key".equals(keyName)) {
    keyField.addAlias(keyName);
  }
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE: null);
  valueField.addProp(FIELD_ID_PROP, valueId);
  if (!"value".equals(valueName)) {
    valueField.addAlias(valueName);
  }

  Schema keyValueRecord = Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField));
  if (!keyValueName.equals(recordName)) {
    keyValueRecord.addAlias(recordName);
  }

  return LogicalMap.get().addToSchema(Schema.createArray(keyValueRecord));
}
 

@Test
public void testArrayOfOptionalRecords() throws Exception {
  Schema innerRecord = Schema.createRecord("element", null, null, false);
  Schema optionalString = optional(Schema.create(Schema.Type.STRING));
  innerRecord.setFields(Lists.newArrayList(
      new Schema.Field("s1", optionalString, null, JsonProperties.NULL_VALUE),
      new Schema.Field("s2", optionalString, null, JsonProperties.NULL_VALUE)
  ));
  Schema schema = Schema.createRecord("HasArray", null, null, false);
  schema.setFields(Lists.newArrayList(
      new Schema.Field("myarray", Schema.createArray(optional(innerRecord)),
          null, null)
  ));
  System.err.println("Avro schema: " + schema.toString(true));

  testRoundTripConversion(NEW_BEHAVIOR, schema, "message HasArray {\n" +
      "  required group myarray (LIST) {\n" +
      "    repeated group list {\n" +
      "      optional group element {\n" +
      "        optional binary s1 (UTF8);\n" +
      "        optional binary s2 (UTF8);\n" +
      "      }\n" +
      "    }\n" +
      "  }\n" +
      "}\n");
}
 

@Test
public void testUnionOfTwoTypes() throws Exception {
  Schema schema = Schema.createRecord("record2", null, null, false);
  Schema multipleTypes = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type
          .NULL),
      Schema.create(INT),
      Schema.create(Schema.Type.FLOAT)));
  schema.setFields(Arrays.asList(
      new Schema.Field("myunion", multipleTypes, null, JsonProperties.NULL_VALUE)));

  // Avro union is modelled using optional data members of the different
  // types. This does not translate back into an Avro union
  testAvroToParquetConversion(
      schema,
      "message record2 {\n" +
          "  optional group myunion {\n" +
          "    optional int32 member0;\n" +
          "    optional float member1;\n" +
          "  }\n" +
          "}\n");
}
 

private static GenericRecord rewrite(GenericRecord record, LinkedHashSet<Field> fieldsToWrite, Schema newSchema) {
  GenericRecord newRecord = new GenericData.Record(newSchema);
  for (Schema.Field f : fieldsToWrite) {
    if (record.get(f.name()) == null) {
      if (f.defaultVal() instanceof JsonProperties.Null) {
        newRecord.put(f.name(), null);
      } else {
        newRecord.put(f.name(), f.defaultVal());
      }
    } else {
      newRecord.put(f.name(), record.get(f.name()));
    }
  }
  if (!GenericData.get().validate(newSchema, newRecord)) {
    throw new SchemaCompatabilityException(
        "Unable to validate the rewritten record " + record + " against schema " + newSchema);
  }
  return newRecord;
}
 

/**
 * Read custom record meta data from a given <code>JsonProperties</code>.
 *
 * @see NetSuiteSchemaConstants
 *
 * @param basicMetaData basic meta data
 * @param properties properties object which to read meta data from
 * @return custom record type info or <code>null</code> if meta data was not found
 */
public static CustomRecordTypeInfo readCustomRecord(BasicMetaData basicMetaData, JsonProperties properties) {
    String scriptId = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_SCRIPT_ID);
    if (StringUtils.isEmpty(scriptId)) {
        return null;
    }
    String internalId = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_INTERNAL_ID);
    String customizationType = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_CUSTOMIZATION_TYPE);
    String recordType = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_TYPE);

    NsRef ref = new NsRef();
    ref.setRefType(RefType.CUSTOMIZATION_REF);
    ref.setScriptId(scriptId);
    ref.setInternalId(internalId);
    ref.setType(customizationType);

    RecordTypeDesc recordTypeDesc = basicMetaData.getRecordType(recordType);
    CustomRecordTypeInfo recordTypeInfo = new CustomRecordTypeInfo(scriptId, recordTypeDesc, ref);

    return recordTypeInfo;
}
 

public static Pair<LinkedHashMap<String, ColumnDef>, List<String>> toColumnDefs(org.apache.avro.Schema schema) {
    int size = schema.getFields().size();
    final LinkedHashMap<String, ColumnDef> columnDefs = new LinkedHashMap<>();
    String[] keyNames = new String[size];
    for (org.apache.avro.Schema.Field field : schema.getFields()) {
        org.apache.avro.Schema fieldSchema = field.schema();
        Integer keyIndex = (Integer) field.getObjectProp(SQL_KEY_INDEX_PROP);
        if (keyIndex != null) {
            keyNames[keyIndex] = field.name();
        }
        ColumnDef columnDef = toColumnDef(fieldSchema);
        if (columnDef == null) {
            throw new IllegalArgumentException("Unsupported type " + fieldSchema.getType());
        }
        if (field.hasDefaultValue()) {
            Object defaultVal = field.defaultVal();
            // Already handled null strategy
            if (defaultVal != JsonProperties.NULL_VALUE) {
                columnDef = new ColumnDef(columnDef.getColumnType(),
                    new ColumnStrategy.DefaultStrategy(defaultVal),
                    columnDef.getPrecision(), columnDef.getScale());
            }
        }
        columnDefs.put(field.name(), columnDef);
    }
    List<String> keyFields = new ArrayList<>(size);
    for (String keyName : keyNames) {
        if (keyName == null) {
            break;
        }
        keyFields.add(keyName);
    }
    return Pair.of(columnDefs, keyFields);
}
 

/**
 * Adds fields to <tt>recordFields</tt> list.
 * @param recordFields - record fields are added to this list.
 * @param field - the field
 * @param fieldName - field name
 * @param fieldSchema - field schema
 * @param dataType -  data type
 * @param nullable - is nullable?
 */
private static void addFieldToList(final List<RecordField> recordFields, final Field field, final String fieldName,
        final Schema fieldSchema, final DataType dataType, final boolean nullable) {
    if (field.defaultVal() == JsonProperties.NULL_VALUE) {
        recordFields.add(new RecordField(fieldName, dataType, field.aliases(), nullable));
    } else {
        Object defaultValue = field.defaultVal();
        if (defaultValue != null && fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) {
            defaultValue = defaultValue instanceof List ? ((List<?>) defaultValue).toArray() : new Object[0];
        }
        recordFields.add(new RecordField(fieldName, dataType, defaultValue, field.aliases(), nullable));
    }
}
 

@Test
public void testUnknownTwoLevelListOfLists() throws Exception {
  // This tests the case where we don't detect a 2-level list by the repeated
  // group's name, but it must be 2-level because the repeated group doesn't
  // contain an optional or repeated element as required for 3-level lists
  Schema listOfLists = optional(Schema.createArray(Schema.createArray(
      Schema.create(INT))));
  Schema schema = Schema.createRecord("UnknownTwoLevelListInList", null, null, false);
  schema.setFields(Lists.newArrayList(
      new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
  ));
  System.err.println("Avro schema: " + schema.toString(true));

  // Cannot use round-trip assertion because repeated group names differ
  testParquetToAvroConversion(schema,
      "message UnknownTwoLevelListInList {\n" +
          "  optional group listOfLists (LIST) {\n" +
          "    repeated group mylist (LIST) {\n" +
          "      repeated int32 innerlist;\n" +
          "    }\n" +
          "  }\n" +
          "}");
  // Cannot use round-trip assertion because 3-level representation is used
  testParquetToAvroConversion(NEW_BEHAVIOR, schema,
      "message UnknownTwoLevelListInList {\n" +
          "  optional group listOfLists (LIST) {\n" +
          "    repeated group mylist (LIST) {\n" +
          "      repeated int32 innerlist;\n" +
          "    }\n" +
          "  }\n" +
          "}");
}
 

@Test
public void testOldThriftListOfLists() throws Exception {
  Schema listOfLists = optional(Schema.createArray(Schema.createArray(
      Schema.create(INT))));
  Schema schema = Schema.createRecord("ThriftCompatListInList", null, null, false);
  schema.setFields(Lists.newArrayList(
      new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
  ));
  System.err.println("Avro schema: " + schema.toString(true));

  // Cannot use round-trip assertion because repeated group names differ
  testParquetToAvroConversion(schema,
      "message ThriftCompatListInList {\n" +
          "  optional group listOfLists (LIST) {\n" +
          "    repeated group listOfLists_tuple (LIST) {\n" +
          "      repeated int32 listOfLists_tuple_tuple;\n" +
          "    }\n" +
          "  }\n" +
          "}");
  // Cannot use round-trip assertion because 3-level representation is used
  testParquetToAvroConversion(NEW_BEHAVIOR, schema,
      "message ThriftCompatListInList {\n" +
      "  optional group listOfLists (LIST) {\n" +
      "    repeated group listOfLists_tuple (LIST) {\n" +
      "      repeated int32 listOfLists_tuple_tuple;\n" +
      "    }\n" +
      "  }\n" +
      "}");
}
 

@Test
public void testOldAvroListOfLists() throws Exception {
  Schema listOfLists = optional(Schema.createArray(Schema.createArray(
      Schema.create(INT))));
  Schema schema = Schema.createRecord("AvroCompatListInList", null, null, false);
  schema.setFields(Lists.newArrayList(
      new Schema.Field("listOfLists", listOfLists, null, JsonProperties.NULL_VALUE)
  ));
  System.err.println("Avro schema: " + schema.toString(true));

  testRoundTripConversion(schema,
      "message AvroCompatListInList {\n" +
          "  optional group listOfLists (LIST) {\n" +
          "    repeated group array (LIST) {\n" +
          "      repeated int32 array;\n" +
          "    }\n" +
          "  }\n" +
          "}");
  // Cannot use round-trip assertion because 3-level representation is used
  testParquetToAvroConversion(NEW_BEHAVIOR, schema,
      "message AvroCompatListInList {\n" +
          "  optional group listOfLists (LIST) {\n" +
          "    repeated group array (LIST) {\n" +
          "      repeated int32 array;\n" +
          "    }\n" +
          "  }\n" +
          "}");
}
 

@Test
public void testOptionalFields() throws Exception {
  Schema schema = Schema.createRecord("record1", null, null, false);
  Schema optionalInt = optional(Schema.create(INT));
  schema.setFields(Collections.singletonList(
    new Schema.Field("myint", optionalInt, null, JsonProperties.NULL_VALUE)
  ));
  testRoundTripConversion(
      schema,
      "message record1 {\n" +
          "  optional int32 myint;\n" +
          "}\n");
}
 

/**
 * Read custom field meta data from a given <code>JsonProperties</code>.
 *
 * @see NetSuiteSchemaConstants
 *
 * @param properties properties object which to read meta data from
 * @return custom field info or <code>null</code> if meta data was not found
 */
public static CustomFieldDesc readCustomField(JsonProperties properties) {
    String scriptId = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_SCRIPT_ID);
    if (StringUtils.isEmpty(scriptId)) {
        return null;
    }
    String internalId = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_INTERNAL_ID);
    String customizationType = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_CUSTOMIZATION_TYPE);
    String type = properties.getProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_TYPE);

    NsRef ref = new NsRef();
    ref.setRefType(RefType.CUSTOMIZATION_REF);
    ref.setScriptId(scriptId);
    ref.setInternalId(internalId);
    ref.setType(customizationType);

    CustomFieldRefType customFieldRefType = CustomFieldRefType.valueOf(type);

    CustomFieldDesc fieldDesc = new CustomFieldDesc();
    fieldDesc.setCustomFieldType(customFieldRefType);
    fieldDesc.setCustomizationRef(ref);
    fieldDesc.setName(scriptId);
    fieldDesc.setValueType(getCustomFieldValueClass(customFieldRefType));
    fieldDesc.setNullable(true);

    return fieldDesc;
}
 

/**
 * Write custom field meta data to a given <code>JsonProperties</code>.
 *
 * @see NetSuiteSchemaConstants
 *
 * @param properties properties object which to write meta data to
 * @param fieldDesc information about custom field to be used
 */
public static void writeCustomField(JsonProperties properties, CustomFieldDesc fieldDesc) {
    NsRef ref = fieldDesc.getCustomizationRef();
    CustomFieldRefType customFieldRefType = fieldDesc.getCustomFieldType();

    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD, "true");
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_SCRIPT_ID, ref.getScriptId());
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_INTERNAL_ID, ref.getInternalId());
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_CUSTOMIZATION_TYPE, ref.getType());
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_FIELD_TYPE, customFieldRefType.name());
}
 

/**
 * Write custom record meta data to a given <code>JsonProperties</code>.
 *
 * @see NetSuiteSchemaConstants
 *
 * @param basicMetaData basic meta data
 * @param properties properties object which to write meta data to
 * @param recordTypeInfo information about record type to be used
 */
public static void writeCustomRecord(BasicMetaData basicMetaData, JsonProperties properties, CustomRecordTypeInfo recordTypeInfo) {
    NsRef ref = recordTypeInfo.getCustomizationRef();
    RecordTypeDesc recordTypeDesc = recordTypeInfo.getRecordType();

    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD, "true");
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_SCRIPT_ID, ref.getScriptId());
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_INTERNAL_ID, ref.getInternalId());
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_CUSTOMIZATION_TYPE, ref.getType());
    properties.addProp(NetSuiteSchemaConstants.NS_CUSTOM_RECORD_TYPE, recordTypeDesc.getType());
}
 

@Test
public void testJsonNodeNullWithDefaultValues() {
  List<Schema.Field> fields = new ArrayList<>();
  Schema initialSchema = Schema.createRecord("test_record", "test record", "org.test.namespace", false);
  Schema.Field field1 = new Schema.Field("key", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field field2 = new Schema.Field("key1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field field3 = new Schema.Field("key2", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  fields.add(field1);
  fields.add(field2);
  fields.add(field3);
  initialSchema.setFields(fields);
  GenericRecord rec = new GenericData.Record(initialSchema);
  rec.put("key", "val");
  rec.put("key1", "val1");
  rec.put("key2", "val2");

  List<Schema.Field> evolvedFields = new ArrayList<>();
  Schema evolvedSchema = Schema.createRecord("evolved_record", "evolved record", "org.evolved.namespace", false);
  Schema.Field evolvedField1 = new Schema.Field("key", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField2 = new Schema.Field("key1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField3 = new Schema.Field("key2", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField4 = new Schema.Field("evolved_field", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
  Schema.Field evolvedField5 = new Schema.Field("evolved_field1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  evolvedFields.add(evolvedField1);
  evolvedFields.add(evolvedField2);
  evolvedFields.add(evolvedField3);
  evolvedFields.add(evolvedField4);
  evolvedFields.add(evolvedField5);
  evolvedSchema.setFields(evolvedFields);

  GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, evolvedSchema);
  //evolvedField4.defaultVal() returns a JsonProperties.Null instance.
  assertNull(rec1.get("evolved_field"));
  //evolvedField5.defaultVal() returns null.
  assertNull(rec1.get("evolved_field1"));
}
 

/**
 * Adds the Hoodie metadata fields to the given schema.
 */
public static Schema addMetadataFields(Schema schema) {
  List<Schema.Field> parentFields = new ArrayList<>();

  Schema.Field commitTimeField =
      new Schema.Field(HoodieRecord.COMMIT_TIME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field commitSeqnoField =
      new Schema.Field(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field recordKeyField =
      new Schema.Field(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field partitionPathField =
      new Schema.Field(HoodieRecord.PARTITION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field fileNameField =
      new Schema.Field(HoodieRecord.FILENAME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);

  parentFields.add(commitTimeField);
  parentFields.add(commitSeqnoField);
  parentFields.add(recordKeyField);
  parentFields.add(partitionPathField);
  parentFields.add(fileNameField);
  for (Schema.Field field : schema.getFields()) {
    if (!isMetadataField(field.name())) {
      Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal());
      for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) {
        newField.addProp(prop.getKey(), prop.getValue());
      }
      parentFields.add(newField);
    }
  }

  Schema mergedSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), false);
  mergedSchema.setFields(parentFields);
  return mergedSchema;
}
 

@Override
public HapiConverter<Schema> visitContained(String elementPath,
    String elementTypeUrl,
    Map<String, StructureField<HapiConverter<Schema>>> contained) {

  String recordName = DefinitionVisitorsUtil.recordNameFor(elementPath);
  String recordNamespace = DefinitionVisitorsUtil.namespaceFor(basePackage, elementTypeUrl);

  List<Field> fields = contained.values()
      .stream()
      .map(containedEntry -> {

        String doc = "Field for FHIR property " + containedEntry.propertyName();

        return new Field(containedEntry.fieldName(),
            nullable(containedEntry.result().getDataType()),
            doc,
            JsonProperties.NULL_VALUE);

      }).collect(Collectors.toList());

  Schema containerType = Schema.createArray(
      Schema.createRecord(recordName,
          "Structure for FHIR type contained",
          recordNamespace,
          false,
          fields));

  return new HapiContainedToAvroConverter(contained, containerType);
}
 

static Schema createMap(int keyId, Schema keySchema,
                        int valueId, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, null);
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE: null);
  valueField.addProp(FIELD_ID_PROP, valueId);

  return LogicalMap.get().addToSchema(Schema.createArray(Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField))));
}
 

static Schema createMap(int keyId, Schema keySchema,
                        int valueId, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, (Object) null);
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE : null);
  valueField.addProp(FIELD_ID_PROP, valueId);

  return LogicalMap.get().addToSchema(Schema.createArray(Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField))));
}
 

private static Schema.Field copyField(Schema.Field field, Schema newSchema, Integer fieldId) {
  Schema newSchemaReordered;
  // if the newSchema is an optional schema, make sure the NULL option is always the first
  if (isOptionSchemaWithNonNullFirstOption(newSchema)) {
    newSchemaReordered = AvroSchemaUtil.toOption(AvroSchemaUtil.fromOption(newSchema));
  } else {
    newSchemaReordered = newSchema;
  }
  // do not copy over default values as the file is expected to have values for fields already in the file schema
  Schema.Field copy = new Schema.Field(field.name(),
      newSchemaReordered, field.doc(),
      AvroSchemaUtil.isOptionSchema(newSchemaReordered) ? JsonProperties.NULL_VALUE : null, field.order());

  for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  if (AvroSchemaUtil.hasFieldId(field)) {
    int existingFieldId = AvroSchemaUtil.getFieldId(field);
    Preconditions.checkArgument(existingFieldId == fieldId,
        "Existing field does match with that fetched from name mapping");
  } else {
    // field may not have a fieldId if the fieldId was fetched from nameMapping
    copy.addProp(AvroSchemaUtil.FIELD_ID_PROP, fieldId);
  }

  return copy;
}
 

private void addAvroRecordField(
    List<org.apache.avro.Schema.Field> fields,
    String fieldName, Schema fieldSchema,
    FromConnectContext fromConnectContext) {

    Object defaultVal = null;
    if (fieldSchema.defaultValue() != null) {
        defaultVal = fieldSchema.defaultValue();

        // If this is a logical, convert to the primitive form for the Avro default value
        defaultVal = toAvroLogical(fieldSchema, defaultVal);

        // Avro doesn't handle a few types that Connect uses, so convert those explicitly here
        if (defaultVal instanceof Byte) {
            // byte are mapped to integers in Avro
            defaultVal = ((Byte) defaultVal).intValue();
        } else if (defaultVal instanceof Short) {
            // Shorts are mapped to integers in Avro
            defaultVal = ((Short) defaultVal).intValue();
        } else if (defaultVal instanceof ByteBuffer) {
            // Avro doesn't handle ByteBuffer directly, but does handle 'byte[]'
            // Copy the contents of the byte buffer without side effects on the buffer
            ByteBuffer buffer = (ByteBuffer) defaultVal;
            byte[] bytes = new byte[buffer.remaining()];
            buffer.duplicate().get(bytes);
            defaultVal = bytes;
        }
    } else if (fieldSchema.isOptional()) {
        defaultVal = JsonProperties.NULL_VALUE;
    }
    org.apache.avro.Schema.Field field = new org.apache.avro.Schema.Field(
        fieldName,
        fromConnectSchema(fieldSchema, fromConnectContext, false),
        fieldSchema.doc(),
        defaultVal);
    fields.add(field);
}
 

Field build() {
    requireNonNull(type, "Schema type is not provided");
    // verify the default value and object
    SchemaUtils.validateFieldSchema(
        fieldName,
        type,
        defaultVal
    );

    final Schema baseSchema;
    switch (type) {
        case INT32:
            baseSchema = SchemaBuilder.builder().intType();
            break;
        case INT64:
            baseSchema = SchemaBuilder.builder().longType();
            break;
        case STRING:
            baseSchema = SchemaBuilder.builder().stringType();
            break;
        case FLOAT:
            baseSchema = SchemaBuilder.builder().floatType();
            break;
        case DOUBLE:
            baseSchema = SchemaBuilder.builder().doubleType();
            break;
        case BOOLEAN:
            baseSchema = SchemaBuilder.builder().booleanType();
            break;
        case BYTES:
            baseSchema = SchemaBuilder.builder().bytesType();
            break;
        // DATE, TIME, TIMESTAMP support from generic record
        case DATE:
            baseSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIME:
            baseSchema = LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIMESTAMP:
            baseSchema = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
            break;
        case AVRO:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.AVRO,
                    "The field is expected to be using AVRO schema but "
                            + genericSchema.getSchemaInfo().getType() + " schema is found");
            GenericAvroSchema genericAvroSchema = (GenericAvroSchema) genericSchema;
            baseSchema = genericAvroSchema.getAvroSchema();
            break;
        default:
            throw new RuntimeException("Schema `" + type + "` is not supported to be used as a field for now");
    }

    for (Map.Entry<String, String> entry : properties.entrySet()) {
        baseSchema.addProp(entry.getKey(), entry.getValue());
    }

    if (null != aliases) {
        for (String alias : aliases) {
            baseSchema.addAlias(alias);
        }
    }

    final Schema finalSchema;
    if (optional) {
        if (defaultVal != null) {
            finalSchema = SchemaBuilder.builder().unionOf()
                .type(baseSchema)
                .and()
                .nullType()
                .endUnion();
        } else {
            finalSchema = SchemaBuilder.builder().unionOf()
                .nullType()
                .and()
                .type(baseSchema)
                .endUnion();
        }
    } else {
        finalSchema = baseSchema;
    }

    final Object finalDefaultValue;
    if (defaultVal != null) {
        finalDefaultValue = SchemaUtils.toAvroObject(defaultVal);
    } else {
        if (optional) {
            finalDefaultValue = JsonProperties.NULL_VALUE;
        } else {
            finalDefaultValue = null;
        }
    }

    return new Field(
        fieldName,
        finalSchema,
        doc,
        finalDefaultValue
    );
}
 

public static Schema.Field optionalField(String name, Schema schema) {
  return new Schema.Field(name, optional(schema), null, JsonProperties.NULL_VALUE);
}
 

@Override
public HapiConverter<Schema> visitComposite(String elementName,
    String elementPath,
    String baseType,
    String elementTypeUrl,
    List<StructureField<HapiConverter<Schema>>> children) {

  String recordName = DefinitionVisitorsUtil.recordNameFor(elementPath);
  String recordNamespace = DefinitionVisitorsUtil.namespaceFor(basePackage, elementTypeUrl);
  String fullName = recordNamespace + "." + recordName;

  HapiConverter<Schema> converter = visitedConverters.get(fullName);

  if (converter == null) {

    List<Field> fields = children.stream()
        .map((StructureField<HapiConverter<Schema>> field) -> {

          String doc = field.extensionUrl() != null
              ? "Extension field for " + field.extensionUrl()
              : "Field for FHIR property " + field.propertyName();

          return new Field(field.fieldName(),
              nullable(field.result().getDataType()),
              doc,
              JsonProperties.NULL_VALUE);

        }).collect(Collectors.toList());

    Schema schema = Schema.createRecord(recordName,
        "Structure for FHIR type " + baseType,
        recordNamespace,
        false,
        fields);

    converter = new CompositeToAvroConverter(baseType, children, schema, fhirSupport);

    visitedConverters.put(fullName, converter);
  }

  return converter;
}
 

static Schema.Field optionalField(int id, String name, Schema schema) {
  return addId(id, new Schema.Field(name, toOption(schema), null, JsonProperties.NULL_VALUE));

}
 

static Schema.Field optionalField(int id, String name, Schema schema) {
  return addId(id, new Schema.Field(name, toOption(schema), null, JsonProperties.NULL_VALUE));

}