Java Code Examples for org.apache.avro.JsonProperties#NULL_VALUE

static Schema createProjectionMap(String recordName,
                        int keyId, String keyName, Schema keySchema,
                        int valueId, String valueName, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, (Object) null);
  if (!"key".equals(keyName)) {
    keyField.addAlias(keyName);
  }
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE : null);
  valueField.addProp(FIELD_ID_PROP, valueId);
  if (!"value".equals(valueName)) {
    valueField.addAlias(valueName);
  }

  Schema keyValueRecord = Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField));
  if (!keyValueName.equals(recordName)) {
    keyValueRecord.addAlias(recordName);
  }

  return LogicalMap.get().addToSchema(Schema.createArray(keyValueRecord));
}
 

static Schema createProjectionMap(String recordName,
                        int keyId, String keyName, Schema keySchema,
                        int valueId, String valueName, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, null);
  if (!"key".equals(keyName)) {
    keyField.addAlias(keyName);
  }
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE: null);
  valueField.addProp(FIELD_ID_PROP, valueId);
  if (!"value".equals(valueName)) {
    valueField.addAlias(valueName);
  }

  Schema keyValueRecord = Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField));
  if (!keyValueName.equals(recordName)) {
    keyValueRecord.addAlias(recordName);
  }

  return LogicalMap.get().addToSchema(Schema.createArray(keyValueRecord));
}
 

private void addAvroRecordField(
    List<org.apache.avro.Schema.Field> fields,
    String fieldName, Schema fieldSchema,
    FromConnectContext fromConnectContext) {

    Object defaultVal = null;
    if (fieldSchema.defaultValue() != null) {
        defaultVal = fieldSchema.defaultValue();

        // If this is a logical, convert to the primitive form for the Avro default value
        defaultVal = toAvroLogical(fieldSchema, defaultVal);

        // Avro doesn't handle a few types that Connect uses, so convert those explicitly here
        if (defaultVal instanceof Byte) {
            // byte are mapped to integers in Avro
            defaultVal = ((Byte) defaultVal).intValue();
        } else if (defaultVal instanceof Short) {
            // Shorts are mapped to integers in Avro
            defaultVal = ((Short) defaultVal).intValue();
        } else if (defaultVal instanceof ByteBuffer) {
            // Avro doesn't handle ByteBuffer directly, but does handle 'byte[]'
            // Copy the contents of the byte buffer without side effects on the buffer
            ByteBuffer buffer = (ByteBuffer) defaultVal;
            byte[] bytes = new byte[buffer.remaining()];
            buffer.duplicate().get(bytes);
            defaultVal = bytes;
        }
    } else if (fieldSchema.isOptional()) {
        defaultVal = JsonProperties.NULL_VALUE;
    }
    org.apache.avro.Schema.Field field = new org.apache.avro.Schema.Field(
        fieldName,
        fromConnectSchema(fieldSchema, fromConnectContext, false),
        fieldSchema.doc(),
        defaultVal);
    fields.add(field);
}
 

public static Pair<LinkedHashMap<String, ColumnDef>, List<String>> toColumnDefs(org.apache.avro.Schema schema) {
    int size = schema.getFields().size();
    final LinkedHashMap<String, ColumnDef> columnDefs = new LinkedHashMap<>();
    String[] keyNames = new String[size];
    for (org.apache.avro.Schema.Field field : schema.getFields()) {
        org.apache.avro.Schema fieldSchema = field.schema();
        Integer keyIndex = (Integer) field.getObjectProp(SQL_KEY_INDEX_PROP);
        if (keyIndex != null) {
            keyNames[keyIndex] = field.name();
        }
        ColumnDef columnDef = toColumnDef(fieldSchema);
        if (columnDef == null) {
            throw new IllegalArgumentException("Unsupported type " + fieldSchema.getType());
        }
        if (field.hasDefaultValue()) {
            Object defaultVal = field.defaultVal();
            // Already handled null strategy
            if (defaultVal != JsonProperties.NULL_VALUE) {
                columnDef = new ColumnDef(columnDef.getColumnType(),
                    new ColumnStrategy.DefaultStrategy(defaultVal),
                    columnDef.getPrecision(), columnDef.getScale());
            }
        }
        columnDefs.put(field.name(), columnDef);
    }
    List<String> keyFields = new ArrayList<>(size);
    for (String keyName : keyNames) {
        if (keyName == null) {
            break;
        }
        keyFields.add(keyName);
    }
    return Pair.of(columnDefs, keyFields);
}
 

private static Schema.Field copyField(Schema.Field field, Schema newSchema, Integer fieldId) {
  Schema newSchemaReordered;
  // if the newSchema is an optional schema, make sure the NULL option is always the first
  if (isOptionSchemaWithNonNullFirstOption(newSchema)) {
    newSchemaReordered = AvroSchemaUtil.toOption(AvroSchemaUtil.fromOption(newSchema));
  } else {
    newSchemaReordered = newSchema;
  }
  // do not copy over default values as the file is expected to have values for fields already in the file schema
  Schema.Field copy = new Schema.Field(field.name(),
      newSchemaReordered, field.doc(),
      AvroSchemaUtil.isOptionSchema(newSchemaReordered) ? JsonProperties.NULL_VALUE : null, field.order());

  for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  if (AvroSchemaUtil.hasFieldId(field)) {
    int existingFieldId = AvroSchemaUtil.getFieldId(field);
    Preconditions.checkArgument(existingFieldId == fieldId,
        "Existing field does match with that fetched from name mapping");
  } else {
    // field may not have a fieldId if the fieldId was fetched from nameMapping
    copy.addProp(AvroSchemaUtil.FIELD_ID_PROP, fieldId);
  }

  return copy;
}
 

static Schema createMap(int keyId, Schema keySchema,
                        int valueId, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, (Object) null);
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE : null);
  valueField.addProp(FIELD_ID_PROP, valueId);

  return LogicalMap.get().addToSchema(Schema.createArray(Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField))));
}
 

static Schema createMap(int keyId, Schema keySchema,
                        int valueId, Schema valueSchema) {
  String keyValueName = "k" + keyId + "_v" + valueId;

  Schema.Field keyField = new Schema.Field("key", keySchema, null, null);
  keyField.addProp(FIELD_ID_PROP, keyId);

  Schema.Field valueField = new Schema.Field("value", valueSchema, null,
      isOptionSchema(valueSchema) ? JsonProperties.NULL_VALUE: null);
  valueField.addProp(FIELD_ID_PROP, valueId);

  return LogicalMap.get().addToSchema(Schema.createArray(Schema.createRecord(
      keyValueName, null, null, false, ImmutableList.of(keyField, valueField))));
}
 

/**
 * Adds the Hoodie metadata fields to the given schema.
 */
public static Schema addMetadataFields(Schema schema) {
  List<Schema.Field> parentFields = new ArrayList<>();

  Schema.Field commitTimeField =
      new Schema.Field(HoodieRecord.COMMIT_TIME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field commitSeqnoField =
      new Schema.Field(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field recordKeyField =
      new Schema.Field(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field partitionPathField =
      new Schema.Field(HoodieRecord.PARTITION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field fileNameField =
      new Schema.Field(HoodieRecord.FILENAME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);

  parentFields.add(commitTimeField);
  parentFields.add(commitSeqnoField);
  parentFields.add(recordKeyField);
  parentFields.add(partitionPathField);
  parentFields.add(fileNameField);
  for (Schema.Field field : schema.getFields()) {
    if (!isMetadataField(field.name())) {
      Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal());
      for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) {
        newField.addProp(prop.getKey(), prop.getValue());
      }
      parentFields.add(newField);
    }
  }

  Schema mergedSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), false);
  mergedSchema.setFields(parentFields);
  return mergedSchema;
}
 

@Test
public void testJsonNodeNullWithDefaultValues() {
  List<Schema.Field> fields = new ArrayList<>();
  Schema initialSchema = Schema.createRecord("test_record", "test record", "org.test.namespace", false);
  Schema.Field field1 = new Schema.Field("key", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field field2 = new Schema.Field("key1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field field3 = new Schema.Field("key2", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  fields.add(field1);
  fields.add(field2);
  fields.add(field3);
  initialSchema.setFields(fields);
  GenericRecord rec = new GenericData.Record(initialSchema);
  rec.put("key", "val");
  rec.put("key1", "val1");
  rec.put("key2", "val2");

  List<Schema.Field> evolvedFields = new ArrayList<>();
  Schema evolvedSchema = Schema.createRecord("evolved_record", "evolved record", "org.evolved.namespace", false);
  Schema.Field evolvedField1 = new Schema.Field("key", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField2 = new Schema.Field("key1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField3 = new Schema.Field("key2", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField4 = new Schema.Field("evolved_field", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
  Schema.Field evolvedField5 = new Schema.Field("evolved_field1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  evolvedFields.add(evolvedField1);
  evolvedFields.add(evolvedField2);
  evolvedFields.add(evolvedField3);
  evolvedFields.add(evolvedField4);
  evolvedFields.add(evolvedField5);
  evolvedSchema.setFields(evolvedFields);

  GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, evolvedSchema);
  //evolvedField4.defaultVal() returns a JsonProperties.Null instance.
  assertNull(rec1.get("evolved_field"));
  //evolvedField5.defaultVal() returns null.
  assertNull(rec1.get("evolved_field1"));
}
 

/**
 * Adds fields to <tt>recordFields</tt> list.
 * @param recordFields - record fields are added to this list.
 * @param field - the field
 * @param fieldName - field name
 * @param fieldSchema - field schema
 * @param dataType -  data type
 * @param nullable - is nullable?
 */
private static void addFieldToList(final List<RecordField> recordFields, final Field field, final String fieldName,
        final Schema fieldSchema, final DataType dataType, final boolean nullable) {
    if (field.defaultVal() == JsonProperties.NULL_VALUE) {
        recordFields.add(new RecordField(fieldName, dataType, field.aliases(), nullable));
    } else {
        Object defaultValue = field.defaultVal();
        if (defaultValue != null && fieldSchema.getType() == Schema.Type.ARRAY && !DataTypeUtils.isArrayTypeCompatible(defaultValue, ((ArrayDataType) dataType).getElementType())) {
            defaultValue = defaultValue instanceof List ? ((List<?>) defaultValue).toArray() : new Object[0];
        }
        recordFields.add(new RecordField(fieldName, dataType, defaultValue, field.aliases(), nullable));
    }
}
 

Field build() {
    requireNonNull(type, "Schema type is not provided");
    // verify the default value and object
    SchemaUtils.validateFieldSchema(
        fieldName,
        type,
        defaultVal
    );

    final Schema baseSchema;
    switch (type) {
        case INT32:
            baseSchema = SchemaBuilder.builder().intType();
            break;
        case INT64:
            baseSchema = SchemaBuilder.builder().longType();
            break;
        case STRING:
            baseSchema = SchemaBuilder.builder().stringType();
            break;
        case FLOAT:
            baseSchema = SchemaBuilder.builder().floatType();
            break;
        case DOUBLE:
            baseSchema = SchemaBuilder.builder().doubleType();
            break;
        case BOOLEAN:
            baseSchema = SchemaBuilder.builder().booleanType();
            break;
        case BYTES:
            baseSchema = SchemaBuilder.builder().bytesType();
            break;
        // DATE, TIME, TIMESTAMP support from generic record
        case DATE:
            baseSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIME:
            baseSchema = LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT));
            break;
        case TIMESTAMP:
            baseSchema = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
            break;
        case AVRO:
            checkArgument(genericSchema.getSchemaInfo().getType() == SchemaType.AVRO,
                    "The field is expected to be using AVRO schema but "
                            + genericSchema.getSchemaInfo().getType() + " schema is found");
            GenericAvroSchema genericAvroSchema = (GenericAvroSchema) genericSchema;
            baseSchema = genericAvroSchema.getAvroSchema();
            break;
        default:
            throw new RuntimeException("Schema `" + type + "` is not supported to be used as a field for now");
    }

    for (Map.Entry<String, String> entry : properties.entrySet()) {
        baseSchema.addProp(entry.getKey(), entry.getValue());
    }

    if (null != aliases) {
        for (String alias : aliases) {
            baseSchema.addAlias(alias);
        }
    }

    final Schema finalSchema;
    if (optional) {
        if (defaultVal != null) {
            finalSchema = SchemaBuilder.builder().unionOf()
                .type(baseSchema)
                .and()
                .nullType()
                .endUnion();
        } else {
            finalSchema = SchemaBuilder.builder().unionOf()
                .nullType()
                .and()
                .type(baseSchema)
                .endUnion();
        }
    } else {
        finalSchema = baseSchema;
    }

    final Object finalDefaultValue;
    if (defaultVal != null) {
        finalDefaultValue = SchemaUtils.toAvroObject(defaultVal);
    } else {
        if (optional) {
            finalDefaultValue = JsonProperties.NULL_VALUE;
        } else {
            finalDefaultValue = null;
        }
    }

    return new Field(
        fieldName,
        finalSchema,
        doc,
        finalDefaultValue
    );
}
 

public static Schema.Field optionalField(String name, Schema schema) {
  return new Schema.Field(name, optional(schema), null, JsonProperties.NULL_VALUE);
}