org.apache.avro.Schema.Type Java Examples

private void validateSchemaIsSupported(Schema schema) {
  for (Field field : schema.getFields()) {
    Type type = field.schema().getType();

    if (type.equals(Type.UNION)) {
      List<Schema> types = field.schema().getTypes();

      if (types.size() != 2) {
        throw new RuntimeException("Union type in Avro serializer schema must only contain two types");
      }

      if (types.get(0).getType().equals(Type.NULL)) {
        type = types.get(1).getType();
      }
      else {
        type = types.get(0).getType();
      }
    }

    if (!supportedTypes.contains(type)) {
      throw new RuntimeException("Avro serializer for Kafka output does not support Avro schema type: " + type);
    }
  }
}
 

/**
 * Use a Schema to generate a hierarchical GenericRecord that contains only null values.
 *
 * @param schema the parent schema of the field to set as null
 * @param fieldName the name of the field to set as null
 * @return if fieldName is a Record of the schema, the method will return a GenericRecord with any leaf set as null,
 * otherwise return null
 */
public static IndexedRecord generateEmptyRecord(Schema schema, String fieldName) {
    if (schema.getType().equals(Type.RECORD)) {
        Schema unwrappedSchema = getUnwrappedSchema(schema.getField(fieldName));
        if (unwrappedSchema.getType().equals(Type.RECORD)) {
            GenericRecordBuilder outputRecord = new GenericRecordBuilder(unwrappedSchema);
            for (Field field : unwrappedSchema.getFields()) {
                IndexedRecord value = generateEmptyRecord(unwrappedSchema, field.name());
                outputRecord.set(field.name(), value);
            }
            return outputRecord.build();
        } else {
            return null;
        }
    } else {
        return null;
    }
}
 

@Test
public void testGoodSchema() {
  AvroEntitySchema avroEntitySchema = parser.parseEntitySchema(entitySchema);
  ColumnMapping descriptor = avroEntitySchema
      .getColumnMappingDescriptor();
  assertEquals(9, descriptor.getFieldMappings().size());
  assertEquals(FieldMapping.MappingType.COLUMN, descriptor.getFieldMapping("field1")
      .getMappingType());
  assertEquals(FieldMapping.MappingType.KEY_AS_COLUMN, descriptor
      .getFieldMapping("field4").getMappingType());
  assertEquals(FieldMapping.MappingType.OCC_VERSION, descriptor.getFieldMapping("version")
      .getMappingType());

  AvroKeySchema avroKeySchema = parser.parseKeySchema(entitySchema);
  assertEquals(Type.STRING, avroKeySchema.getAvroSchema()
      .getField("keyPart1_copy").schema().getType());
  assertEquals(Type.STRING, avroKeySchema.getAvroSchema()
      .getField("keyPart2_copy").schema().getType());
  assertEquals(2, Accessor.getDefault().getFieldPartitioners(avroKeySchema
      .getPartitionStrategy())
      .size());
}
 

/**
 * Convert a passed String value to the given type for the key as per Schema
 */
public static Object convertValueStringToAvroKeyType(Schema schema, String key, String value) throws ParseException
{
  Type type = null;

  if (schema.getField(key) != null) {
    type = schema.getField(key).schema().getType();
  } else {
    return value;
  }

  Object convertedValue = null;

  if (type == Type.UNION) {
    convertedValue = convertAndResolveUnionToPrimitive(schema, key, value);
  } else {
    convertedValue = convertValueToAvroPrimitive(type, key, value);
  }

  return convertedValue;

}
 

@Override
public CharSequence deserializeKeyAsColumnKeyFromBytes(String fieldName,
    byte[] columnKeyBytes) {
  Field field = avroSchema.getAvroSchema().getField(fieldName);
  if (field == null) {
    throw new ValidationException("Invalid field name " + fieldName
        + " for schema " + avroSchema.toString());
  }

  Schema.Type schemaType = field.schema().getType();
  if (schemaType == Schema.Type.MAP) {
    String stringProp = field.schema().getProp("avro.java.string");
    if (stringProp != null && stringProp.equals("String")) {
      return new String(columnKeyBytes);
    } else {
      return new Utf8(columnKeyBytes);
    }
  } else if (schemaType == Schema.Type.RECORD) {
    return new String(columnKeyBytes);
  } else {
    throw new ValidationException("Unsupported type for keyAsColumn: "
        + schemaType);
  }
}
 

@Test(expected = InvalidKeyPathException.class)
public void arrayFail() throws InvalidKeyPathException {
  Path path = KeyPathParser.parse("$.a.b.c");
  Schema schema = SchemaBuilder
      .record("X")
      .fields()
      .name("a")
      .type(SchemaBuilder.array().items(
          SchemaBuilder.record("Z").fields().name("c").type(Schema.create(Type.STRING)).noDefault().endRecord()))
      .noDefault()
      .endRecord();
  try {
    new KeyPathValidator(path, schema).validate();
  } catch (Exception e) {
    assertThat(e.getMessage(), is("Element '$.a' not a traversable type (found 'ARRAY'), in path: $.a.b.c"));
    throw e;
  }
}
 

@Test(expected = InvalidKeyPathException.class)
public void mapFail() throws InvalidKeyPathException {
  Path path = KeyPathParser.parse("$.a.b[\"c\"]");
  Schema schema = SchemaBuilder
      .record("X")
      .fields()
      .name("a")
      .type(SchemaBuilder.map().values(
          SchemaBuilder.record("Z").fields().name("c").type(Schema.create(Type.STRING)).noDefault().endRecord()))
      .noDefault()
      .endRecord();
  try {
    new KeyPathValidator(path, schema).validate();
  } catch (Exception e) {
    assertThat(e.getMessage(), is("Element '$.a' not a traversable type (found 'MAP'), in path: $.a.b[\"c\"]"));
    throw e;
  }
}
 

public static GenericRecord read(InputStream record, char delimiter, Schema schema, char quoteChar) {
    Record avroRecord = new GenericData.Record(schema);
    String[] parsedRecord = parseFields(convertInputStreamToString(record), delimiter, quoteChar);
    List<Field> fields = schema.getFields();
    if (parsedRecord.length != fields.size()) {
        throw new IllegalStateException("Incompatible schema. Parsed fields count does not match the count of fields from schema. "
                + "Schema: " + schema.toString(true) + "\n Record: " + record);
    }

    for (int i = 0; i < fields.size(); i++) {
        Field field = fields.get(i);
        Type type = field.schema().getType();
        updateRecord(field, type, parsedRecord[i], avroRecord);
    }
    return avroRecord;
}
 

@Test(expected = InvalidKeyPathException.class)
public void nestedFail() throws InvalidKeyPathException {
  Path path = KeyPathParser.parse("$[\"a\"].b[\"c\"]");
  Schema schema = SchemaBuilder
      .record("X")
      .fields()
      .name("a")
      .type(SchemaBuilder.record("Y").fields().name("b").type(Schema.create(Type.STRING)).noDefault().endRecord())
      .noDefault()
      .endRecord();
  try {
    new KeyPathValidator(path, schema).validate();
  } catch (Exception e) {
    assertThat(e.getMessage(),
        is("Element '$[\"a\"].b' not a traversable type (found 'STRING'), in path: $[\"a\"].b[\"c\"]"));
    throw e;
  }
}
 

protected static boolean isValidValue(Field f, Object value) {
  if (value != null) {
    return true;
  } else {
    Schema schema = f.schema();
    Type type = schema.getType();
    if (type == Type.NULL) {
      return true;
    } else {
      if (type == Type.UNION) {
        Iterator i$ = schema.getTypes().iterator();

        while(i$.hasNext()) {
          Schema s = (Schema)i$.next();
          if (s.getType() == Type.NULL) {
            return true;
          }
        }
      }

      return false;
    }
  }
}
 

/**
 * For the schema that is a UNION type with NULL and Record type, it provides Records type.
 * @param inputSchema
 * @return
 */
private static Schema getActualRecord(Schema inputSchema) {
  if (Type.RECORD.equals(inputSchema.getType())) {
    return inputSchema;
  }

  Preconditions.checkArgument(Type.UNION.equals(inputSchema.getType()), "Nested schema is only support with either record or union type of null with record");
  Preconditions.checkArgument(inputSchema.getTypes().size() <= 2,
      "For union type in nested record, it should only have NULL and Record type");

  for (Schema inner : inputSchema.getTypes()) {
    if (Type.NULL.equals(inner.getType())) {
      continue;
    }
    Preconditions.checkArgument(Type.RECORD.equals(inner.getType()), "For union type in nested record, it should only have NULL and Record type");
    return inner;

  }
  throw new IllegalArgumentException(inputSchema + " is not supported.");
}
 

private static Schema determineType(Schema schema) throws SchemaConversionException {
  if (!AVRO_SUPPORTED_TYPES.contains(schema.getType())) {
    throw new SchemaConversionException(schema.getType() + " is not supported");
  }

  if (!Type.UNION.equals(schema.getType())) {
    return schema;
  }

  //For UNION, only supported avro type with NULL is allowed.
  List<Schema> schemas = schema.getTypes();
  if (schemas.size() > 2) {
    throw new SchemaConversionException("More than two types are not supported " + schemas);
  }

  for (Schema s : schemas) {
    if (Type.NULL.equals(s.getType())) {
      continue;
    }
    return s;
  }

  throw new SchemaConversionException("Cannot determine type of " + schema);
}
 

private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Map<Schema, Optional<Schema>> processed) {
  Preconditions.checkArgument(record.getType() == Schema.Type.RECORD);

  Optional<Schema> result = processed.get(record);
  if (null != result) {
    return result;
  }

  List<Field> fields = Lists.newArrayList();
  for (Field field : record.getFields()) {
    Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed);
    if (newFieldSchema.isPresent()) {
      fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue()));
    }
  }

  Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false);
  newSchema.setFields(fields);
  result = Optional.of(newSchema);
  processed.put(record, result);

  return result;
}
 

/**
 * Helper method that does the actual work for {@link #getFieldSchema(Schema, String)}
 * @param schema passed from {@link #getFieldSchema(Schema, String)}
 * @param pathList passed from {@link #getFieldSchema(Schema, String)}
 * @param field keeps track of the index used to access the list pathList
 * @return the schema of the field
 */
private static Optional<Schema> getFieldSchemaHelper(Schema schema, List<String> pathList, int field) {
  if (schema.getType() == Type.RECORD && schema.getField(pathList.get(field)) == null) {
    return Optional.absent();
  }
  switch (schema.getType()) {
    case UNION:
      if (AvroSerdeUtils.isNullableType(schema)) {
        return AvroUtils.getFieldSchemaHelper(AvroSerdeUtils.getOtherTypeFromNullableType(schema), pathList, field);
      }
      throw new AvroRuntimeException("Union of complex types cannot be handled : " + schema);
    case MAP:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getValueType());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getValueType(), pathList, ++field);
    case RECORD:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getField(pathList.get(field)).schema());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getField(pathList.get(field)).schema(), pathList, ++field);
    default:
      throw new AvroRuntimeException("Invalid type in schema : " + schema);
  }
}
 

private static Type convertToAvroType(DataType colType)
{
    final Type subType;
    if (colType == DataType.TUPLE)
    {
        /* Pig converts RECORD to TUPLE. Converting it back. */
        subType = Type.RECORD;
    }
    else if (colType == DataType.BAG)
    {
        subType = Type.ARRAY;
    }
    else if (colType == DataType.MAP)
    {
      subType = Type.MAP;
    }
    else
    {
        subType = Type.valueOf(colType.toString().toUpperCase());
    }
    return subType;
}
 

public static Schema getSchema()
{
    Field[] fields =
            {
                    new Schema.Field("colname",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("colvalue",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("code", Schema.create(Type.INT), null, null) };

    Schema schema = Schema.createRecord("dictionary", null, null, false);
    schema.setFields(Arrays.asList(fields));

    return schema;
}
 

public static DataType getColumnType(Field field) {
  org.apache.avro.Schema fieldSchema = field.schema();
  fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);

  final Type type = fieldSchema.getType();
  if (type == Type.ARRAY) {
    org.apache.avro.Schema elementSchema = extractSchemaFromUnionIfNeeded(fieldSchema.getElementType());
    if (elementSchema.getType() == Type.RECORD) {
      if (elementSchema.getFields().size() == 1) {
        elementSchema = elementSchema.getFields().get(0).schema();
      } else {
        throw new RuntimeException("More than one schema in Multi-value column!");
      }
      elementSchema = extractSchemaFromUnionIfNeeded(elementSchema);
    }
    return AvroSchemaUtil.valueOf(elementSchema.getType());
  } else {
    return AvroSchemaUtil.valueOf(type);
  }
}
 

/**
 * Packs a Pig DataBag into an Avro array.
 * @param db the Pig databad to pack into the avro array
 * @param s The avro schema for which to determine the type
 * @return the avro array corresponding to the input bag
 * @throws IOException
 */
public static GenericData.Array<Object> packIntoAvro(
    final DataBag db, final Schema s) throws IOException {

  try {
    GenericData.Array<Object> array
      = new GenericData.Array<Object>(new Long(db.size()).intValue(), s);
    for (Tuple t : db) {
      if (s.getElementType() != null
          && s.getElementType().getType() == Type.RECORD) {
        array.add(packIntoAvro(t, s.getElementType()));
      } else if (t.size() == 1) {
        array.add(t.get(0));
      } else {
        throw new IOException(
            "AvroStorageDataConversionUtilities.packIntoAvro: Can't pack "
                + t + " into schema " + s);
      }
    }
    return array;
  } catch (Exception e) {
    throw new IOException(
        "exception in AvroStorageDataConversionUtilities.packIntoAvro", e);
  }
}
 

public void testNonIdentCharactersInColumnName() throws IOException {
  String [] names = { "test_a-v+r/o" };
  String [] types = { "INT" };
  String [] vals = { "2015" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
  DataFileReader<GenericRecord> reader = read(outputFile);
  Schema schema = reader.getSchema();
  assertEquals(Schema.Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());

  checkField(fields.get(0), "TEST_A_V_R_O", Type.INT);

  GenericRecord record1 = reader.next();
  assertEquals("TEST_A_V_R_O", 2015, record1.get("TEST_A_V_R_O"));
}
 

public void testNonstandardCharactersInColumnName() throws IOException {
  String [] names = { "avro\uC3A11" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
  DataFileReader<GenericRecord> reader = read(outputFile);
  Schema schema = reader.getSchema();
  assertEquals(Schema.Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());

  checkField(fields.get(0), "AVRO1", Type.INT);

  GenericRecord record1 = reader.next();
  assertEquals("AVRO1", 1987, record1.get("AVRO1"));
}
 

public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
  DataFileReader<GenericRecord> reader = read(outputFile);
  Schema schema = reader.getSchema();
  assertEquals(Schema.Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());

  checkField(fields.get(0), "__NAME", Type.INT);

  GenericRecord record1 = reader.next();
  assertEquals("__NAME", 1987, record1.get("__NAME"));
}
 

@Override
public void append(final Object o) {
  List<Field> fields = avroObject.getSchema().getFields();
  avroObject.put(fields.size(), o);
  Schema fieldSchema = null;
  if (o instanceof String) {
    fieldSchema = Schema.create(Type.STRING);
  } else if (o instanceof Integer) {
    fieldSchema = Schema.create(Type.INT);
  } else if (o instanceof Long) {
    fieldSchema = Schema.create(Type.LONG);
  } else if (o instanceof Double) {
    fieldSchema = Schema.create(Type.DOUBLE);
  } else if (o instanceof Float) {
    fieldSchema = Schema.create(Type.FLOAT);
  } else if (o == null) {
    fieldSchema = Schema.create(Type.NULL);
  } else if (o instanceof Boolean) {
    fieldSchema = Schema.create(Type.BOOLEAN);
  } else if (o instanceof Map) {
    fieldSchema = Schema.create(Type.MAP);
  }
  Field newField = new Field("FIELD_" + fields.size(), fieldSchema, "", null);
  fields.add(newField);
  avroObject.getSchema().setFields(fields);
}
 

public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "__NAME", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("__NAME", 1987, record1.get("__NAME"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

private Type toAvroType(String columnName, int sqlType) {
  Properties mapping = options.getMapColumnJava();

  if (mapping.containsKey(columnName)) {
    String type = mapping.getProperty(columnName);
    if (LOG.isDebugEnabled()) {
      LOG.info("Overriding type of column " + columnName + " to " + type);
    }

    if (type.equalsIgnoreCase("INTEGER")) { return Type.INT; }
    if (type.equalsIgnoreCase("LONG")) { return Type.LONG; }
    if (type.equalsIgnoreCase("BOOLEAN")) { return Type.BOOLEAN; }
    if (type.equalsIgnoreCase("FLOAT")) { return Type.FLOAT; }
    if (type.equalsIgnoreCase("DOUBLE")) { return Type.DOUBLE; }
    if (type.equalsIgnoreCase("STRING")) { return Type.STRING; }
    if (type.equalsIgnoreCase("BYTES")) { return Type.BYTES; }

    // Mapping was not found
    throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
  }

  return connManager.toAvroType(tableName, columnName, sqlType);
}
 

@Test
public void testNullableArrayFieldToBeamArrayField() {
  org.apache.avro.Schema.Field avroField =
      new org.apache.avro.Schema.Field(
          "arrayField",
          ReflectData.makeNullable(
              org.apache.avro.Schema.createArray((org.apache.avro.Schema.create(Type.INT)))),
          "",
          null);

  Field expectedBeamField = Field.nullable("arrayField", FieldType.array(FieldType.INT32));

  Field beamField = AvroUtils.toBeamField(avroField);
  assertEquals(expectedBeamField, beamField);
}
 

/**
 * Flattens Avro's (possibly recursive) structure and provides field name and type.
 * It assumes that the leaf level field name has unique name.
 * @param schema
 * @return
 * @throws SchemaConversionException if there's duplicate name in leaf level of Avro Schema
 */
private static Map<String, Type> flatten(Schema schema) throws SchemaConversionException {
  Map<String, Type> flattened = new LinkedHashMap<>();
  Schema recordSchema = determineType(schema);

  Preconditions.checkArgument(Type.RECORD.equals(recordSchema.getType()), "%s is expected. Schema: %s",
      Type.RECORD, recordSchema);

  for (Field f : recordSchema.getFields()) {
    produceFlattenedHelper(f, flattened);
  }
  return flattened;
}
 

/**
 * Generate a new Index Record which is the filtered result of the input record.
 *
 * The user can freely remove column, add empty column or change the place of column in the same hierarchical level.
 *
 * @return the new record
 */
public static IndexedRecord extractIndexedRecord(IndexedRecord inputRecord, Schema outputSchema) {
    GenericRecordBuilder outputRecord = new GenericRecordBuilder(outputSchema);
    Schema inputSchema = getUnwrappedSchema(inputRecord);
    for (Field field : outputSchema.getFields()) {
        if (inputSchema.getField(field.name()) != null) {
            // The column was existing on the input record, we forward it to the output record.
            Object inputValue = inputRecord.get(inputSchema.getField(field.name()).pos());

            // The current column can be a Record (an hierarchical sub-object) or directly a value.
            // If we are on a record, we need to recursively do the process
            // if we are on a object, we save it to the output.
            if (inputValue instanceof Record) {
                // The sub-schema at this level is a union of "empty" and a record,
                // so we need to get the true sub-schema
                Schema inputChildSchema = getUnwrappedSchema(inputSchema.getField(field.name()));
                Schema outputChildSchema = getUnwrappedSchema(outputSchema.getField(field.name()));
                if (inputChildSchema.getType().equals(Type.RECORD)
                        && outputChildSchema.getType().equals(Type.RECORD)) {
                    Object childRecord = extractIndexedRecord((IndexedRecord) inputValue, outputChildSchema);
                    outputRecord.set(field.name(), childRecord);
                }
            } else {
                outputRecord.set(field.name(), inputValue);
            }
        } else {
            // element not found => set to the value and its hierarchy to null
            outputRecord.set(field.name(), KeyValueUtils.generateEmptyRecord(outputSchema, field.name()));
        }
    }
    return outputRecord.build();
}
 

@Override
protected java.lang.reflect.Type convertDefault(TypeDescriptor<?> type) {
  if (type.isSubtypeOf(TypeDescriptor.of(GenericFixed.class))) {
    return byte[].class;
  } else {
    return super.convertDefault(type);
  }
}
 

@Test
public void testIsDateTypeField() throws Exception {
    assertTrue(MarketoClientUtils.isDateTypeField(fValidDate));
    assertFalse(MarketoClientUtils.isDateTypeField(null));
    assertFalse(MarketoClientUtils.isDateTypeField(fInvalidDate));
    assertFalse(MarketoClientUtils.isDateTypeField(generateFieldType(Type.LONG)));
}
 

private final Value convertAvroPrimitiveToValue(Type type, Object avroRecord) {
	switch (type) {
	case STRING:
		sString.setValue((CharSequence) avroRecord);
		return sString;
	case INT:
		sInt.setValue((Integer) avroRecord);
		return sInt;
	case BOOLEAN:
		sBool.setValue((Boolean) avroRecord);
		return sBool;
	case DOUBLE:
		sDouble.setValue((Double) avroRecord);
		return sDouble;
	case FLOAT:
		sFloat.setValue((Float) avroRecord);
		return sFloat;
	case LONG:
		sLong.setValue((Long) avroRecord);
		return sLong;
	case NULL:
		return NullValue.getInstance();
	default:
		throw new RuntimeException(
				"Type "
						+ type
						+ " for AvroInputFormat is not implemented. Open an issue on GitHub.");
	}
}