Java Code Examples for org.apache.avro.Schema.Field#schema()

protected static boolean isValidValue(Field f, Object value) {
  if (value != null) {
    return true;
  } else {
    Schema schema = f.schema();
    Type type = schema.getType();
    if (type == Type.NULL) {
      return true;
    } else {
      if (type == Type.UNION) {
        Iterator i$ = schema.getTypes().iterator();

        while(i$.hasNext()) {
          Schema s = (Schema)i$.next();
          if (s.getType() == Type.NULL) {
            return true;
          }
        }
      }

      return false;
    }
  }
}
 

/**
 * Builds the Meta Data of from the Avro Schema
 * 
 * @return
 */
protected InstanceInformation getHeader() {

  String relation = schema.getName();
  attributes = new ArrayList<Attribute>();

  /** By Definition, the returned list is in the order of their positions. **/
  List<Schema.Field> fields = schema.getFields();

  for (Field field : fields) {
    Schema attributeSchema = field.schema();

    /** Currently SAMOA supports only NOMINAL & Numeric Types. **/
    if (attributeSchema.getType() == Schema.Type.ENUM)
    {
      List<String> attributeLabels = attributeSchema.getEnumSymbols();
      attributes.add(new Attribute(field.name(), attributeLabels));
    }
    else if (isNumeric(field))
      attributes.add(new Attribute(field.name()));
  }
  return new InstanceInformation(relation, attributes);
}
 

/**
 * Identifies if the dataset is is Sparse or Dense
 * 
 * @return boolean
 */
protected boolean isSparseData()
{
  List<Schema.Field> fields = schema.getFields();
  for (Field field : fields) {
    Schema attributeSchema = field.schema();

    /** If even one attribute has a null union (nullable attribute) consider it as sparse data **/
    if (attributeSchema.getType() == Schema.Type.UNION)
    {
      List<Schema> unionTypes = attributeSchema.getTypes();
      for (Schema unionSchema : unionTypes) {
        if (unionSchema.getType() == Schema.Type.NULL)
          return true;
      }
    }

  }
  return false;
}
 

/**
 * Initialize converters per each schema field
 * 
 * @param schema
 *            design schema
 */
private void initConverters(Schema schema) {
	converters = new StringConverter[size];
	List<Field> fields = schema.getFields();
	for (int i = 0; i < size; i++) {
		Field field = fields.get(i);
		Schema fieldSchema = field.schema();
		fieldSchema = AvroUtils.unwrapIfNullable(fieldSchema);
		if (LogicalTypeUtils.isLogicalTimestampMillis(fieldSchema)) {
			String datePattern = field.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
			converters[i] = new StringTimestampConverter(datePattern);
		} else {
			Type type = fieldSchema.getType();
			converters[i] = converterRegistry.get(type);
		}
	}
}
 

public void updatePartitionKeyAndRowKey() {
    Schema inputSchema = schema.schema.getValue();
    List<String> possibleValues = new ArrayList<String>();
    for (Field field : inputSchema.getFields()) {
        Schema fSchema = field.schema();
        if (fSchema.getType() == Type.UNION) {
            for (Schema s : field.schema().getTypes()) {
                if (s.getType() != Type.NULL) {
                    fSchema = s;
                    break;
                }
            }
        }
        if (fSchema.getType().equals(Type.STRING)) {
            possibleValues.add(field.name());
        }
    }
    partitionKey.setPossibleValues(possibleValues);
    rowKey.setPossibleValues(possibleValues);
}
 

/**
 * Helper method that does the actual work for {@link #getField(Schema, String)}
 * @param schema passed from {@link #getFieldSchema(Schema, String)}
 * @param pathList passed from {@link #getFieldSchema(Schema, String)}
 * @param field keeps track of the index used to access the list pathList
 * @return the field
 */
private static Optional<Field> getFieldHelper(Schema schema, List<String> pathList, int field) {
  Field curField = schema.getField(pathList.get(field));
  if (field + 1 == pathList.size()) {
    return Optional.fromNullable(curField);
  }

  Schema fieldSchema = curField.schema();
  switch (fieldSchema.getType()) {
    case UNION:
      throw new AvroRuntimeException("Union of complex types cannot be handled : " + schema);
    case MAP:
      return AvroUtils.getFieldHelper(fieldSchema.getValueType(), pathList, ++field);
    case RECORD:
      return AvroUtils.getFieldHelper(fieldSchema, pathList, ++field);
    case ARRAY:
      return AvroUtils.getFieldHelper(fieldSchema.getElementType(), pathList, ++field);
    default:
      throw new AvroRuntimeException("Invalid type " + fieldSchema.getType() + " in schema : " + schema);
  }
}
 

/**
 * Converts an Avro Schema to a RecordSchema
 *
 * @param avroSchema the Avro Schema to convert
 * @param schemaText the textual representation of the schema
 * @param schemaId the identifier of the schema
 * @return the Corresponding Record Schema
 */
public static RecordSchema createSchema(final Schema avroSchema, final String schemaText, final SchemaIdentifier schemaId) {
    if (avroSchema == null) {
        throw new IllegalArgumentException("Avro Schema cannot be null");
    }

    final String schemaFullName = avroSchema.getNamespace() + "." + avroSchema.getName();
    final SimpleRecordSchema recordSchema = schemaText == null ? new SimpleRecordSchema(schemaId) : new SimpleRecordSchema(schemaText, AVRO_SCHEMA_FORMAT, schemaId);
    recordSchema.setSchemaName(avroSchema.getName());
    recordSchema.setSchemaNamespace(avroSchema.getNamespace());
    final DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema);
    final Map<String, DataType> knownRecords = new HashMap<>();
    knownRecords.put(schemaFullName, recordSchemaType);

    final List<RecordField> recordFields = new ArrayList<>(avroSchema.getFields().size());
    for (final Field field : avroSchema.getFields()) {
        final String fieldName = field.name();
        final Schema fieldSchema = field.schema();
        final DataType dataType = AvroTypeUtil.determineDataType(fieldSchema, knownRecords);
        final boolean nullable = isNullable(fieldSchema);
        addFieldToList(recordFields, field, fieldName, fieldSchema, dataType, nullable);
    }

    recordSchema.setFields(recordFields);
    return recordSchema;
}
 

@Override
public void onVisit(Field field, Collection<String> breadcrumb) {
  if (PII.equalsIgnoreCase(field.getProp(SENSITIVITY))) {
    Schema schema = field.schema();
    if (!isStringOrBytes(schema) && !isNullableStringOrBytes(schema)) {
      throw new InvalidPiiAnnotationException(breadcrumb);
    }
    onPiiField(field, breadcrumb);
  }
}
 

/**
 * Validate a Pig tuple is compatible with Avro record. If the Avro schema 
 * is not complete (with uncovered fields), then convert those fields using 
 * methods in set 1. 
 * 
 * Notice that users can get rid of Pig tuple wrappers, e.g. an Avro schema
 * "int" is compatible with a Pig schema "T:(int)"
 * 
 */
protected static Schema validateAndConvertRecord(Schema avroSchema, ResourceFieldSchema[] pigFields) throws IOException {

    /* Get rid of Pig tuple wrappers. */
    if (!avroSchema.getType().equals(Schema.Type.RECORD)) {
        if (pigFields.length != 1)
            throw new IOException("Expect only one field in Pig tuple schema. Avro schema is " + avroSchema.getType());

        return validateAndConvert(avroSchema, pigFields[0]);
    }

    /* validate and convert a pig tuple with avro record */
    boolean isPartialSchema = AvroStorageUtils.isUDPartialRecordSchema(avroSchema);
    AvroStorageLog.details("isPartialSchema=" + isPartialSchema);

    String typeName = isPartialSchema ? getRecordName() : avroSchema.getName();
    Schema outSchema = Schema.createRecord(typeName, avroSchema.getDoc(), avroSchema.getNamespace(), false);

    List<Schema.Field> inFields = avroSchema.getFields();
    if (!isPartialSchema && inFields.size() != pigFields.length) {
        throw new IOException("Expect " + inFields.size() + " fields in pig schema." + " But there are " + pigFields.length);
    }

    List<Schema.Field> outFields = new ArrayList<Schema.Field>();

    for (int i = 0; i < pigFields.length; i++) {
        /* get user defined avro field schema */
        Field inputField = isPartialSchema ? AvroStorageUtils.getUDField(avroSchema, i) : inFields.get(i);

        /* get schema */
        Schema fieldSchema = null;
        if (inputField == null) { 
            /* convert pig schema (nullable) */
            fieldSchema = convert(pigFields[i], true);
        } else if (inputField.schema() == null) { 
            /* convert pig schema (not-null) */
            fieldSchema = convert(pigFields[i], false);
        } else { 
            /* validate pigFields[i] with given avro schema */
            fieldSchema = validateAndConvert(inputField.schema(),
                                            pigFields[i]);
        }

        /* get field name of output */
        String outname = (isPartialSchema) ? pigFields[i].getName() : inputField.name();
        if (outname == null)
            outname = FIELD_NAME + "_" + i; // field name cannot be null

        /* get doc of output */
        String doc = (isPartialSchema) ? pigFields[i].getDescription() : inputField.doc();

        JsonNode defaultvalue = (inputField != null) ? inputField.defaultValue() : null;

        outFields.add(new Field(outname, fieldSchema, doc, defaultvalue));

    }

    outSchema.setFields(outFields);
    return outSchema;

}
 

/**
 * Convert to the output schema of a field
 */
protected Field convertFieldSchema(Schema inputSchema, Field field, WorkUnitState workUnit)
    throws SchemaConversionException {
  if (field.name().equals(payloadField)) {
    // Create a payload field with latest schema
    return createLatestPayloadField(field);
  }
  // Make a copy of the field to the output schema
  return new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order());
}
 

public static Schema getSchema(SeekableInput input) throws IOException
{
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
    DataFileReader<GenericRecord> dataFileReader =
            new DataFileReader<GenericRecord>(input, datumReader);
    Schema schema = dataFileReader.getSchema();

    if (PadDefaultNullsToSchema)
    {
        // a list of "cloned" fields, with optional default value set to null
        ArrayList<Field> paddedFields = new ArrayList<Field>();

        for (Field field: schema.getFields())
        {
            // should this field be padded?
            boolean needsNullPadding = (field.schema() != null) // the field has nested schema
                && (field.schema().getType().equals(Type.UNION)) // the nested schema is UNION
                && (field.schema().getTypes().get(0).getType().equals(Type.NULL)); // the first element of union is NULL type

            JsonNode defValue = needsNullPadding ? NullNode.getInstance() : field.defaultValue();

            Field f = new Field(field.name(), field.schema(), field.doc(), defValue);
            paddedFields.add(f);
        }

        schema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError());
        schema.setFields(paddedFields);
    }

    return schema;
}
 

public static Field generateNewField(Field origin) {
    Schema.Field field = new Schema.Field(origin.name(), origin.schema(), origin.doc(), origin.defaultVal(), origin.order());
    field.getObjectProps().putAll(origin.getObjectProps());
    for (Map.Entry<String, Object> entry : origin.getObjectProps().entrySet()) {
        field.addProp(entry.getKey(), entry.getValue());
    }
    return field;
}
 

public static List<Field> getSchemaFieldsList(Schema schema) {
    List<Field> result = new ArrayList<>();
    for (Field f : schema.getFields()) {
        Field nf = new Field(f.name(), f.schema(), f.doc(), f.defaultVal());
        nf.getObjectProps().putAll(f.getObjectProps());
        for (Map.Entry<String, Object> entry : f.getObjectProps().entrySet()) {
            nf.addProp(entry.getKey(), entry.getValue());
        }
        result.add(nf);
    }
    return result;
}
 

@Override
Optional<ValidationError> validateTypes(final Field target) {
    final Schema targetSchema = target.schema();
    final Optional<ValidationError> validationError =
            validateTrivialUnion(targetSchema,
                                 s -> s.getType() == Type.ARRAY,
                                 "must map to an Avro array, not %s", target);
    return validationError.isPresent()
            ? validationError
            : validateTrivialUnion(unpackNullableUnion(targetSchema).orElseThrow(IllegalArgumentException::new).getElementType(),
                                   s -> COMPATIBLE_PRIMITIVES.get(listType.getRawType()) == s.getType(),
                                   "array type must be compatible with %s", listType);
}
 

/**
 * Validate a Pig tuple is compatible with Avro record. If the Avro schema 
 * is not complete (with uncovered fields), then convert those fields using 
 * methods in set 1. 
 * 
 * Notice that users can get rid of Pig tuple wrappers, e.g. an Avro schema
 * "int" is compatible with a Pig schema "T:(int)"
 * 
 */
protected static Schema validateAndConvertRecord(Schema avroSchema, ResourceFieldSchema[] pigFields) throws IOException {

    /* Get rid of Pig tuple wrappers. */
    if (!avroSchema.getType().equals(Schema.Type.RECORD)) {
        if (pigFields.length != 1)
            throw new IOException("Expect only one field in Pig tuple schema. Avro schema is " + avroSchema.getType());

        return validateAndConvert(avroSchema, pigFields[0]);
    }

    /* validate and convert a pig tuple with avro record */
    boolean isPartialSchema = AvroStorageUtils.isUDPartialRecordSchema(avroSchema);
    AvroStorageLog.details("isPartialSchema=" + isPartialSchema);

    String typeName = isPartialSchema ? getRecordName() : avroSchema.getName();
    Schema outSchema = Schema.createRecord(typeName, avroSchema.getDoc(), avroSchema.getNamespace(), false);

    List<Schema.Field> inFields = avroSchema.getFields();
    if (!isPartialSchema && inFields.size() != pigFields.length) {
        throw new IOException("Expect " + inFields.size() + " fields in pig schema." + " But there are " + pigFields.length);
    }

    List<Schema.Field> outFields = new ArrayList<Schema.Field>();

    for (int i = 0; i < pigFields.length; i++) {
        /* get user defined avro field schema */
        Field inputField = isPartialSchema ? AvroStorageUtils.getUDField(avroSchema, i) : inFields.get(i);

        /* get schema */
        Schema fieldSchema = null;
        if (inputField == null) { 
            /* convert pig schema (nullable) */
            fieldSchema = convert(pigFields[i], true);
        } else if (inputField.schema() == null) { 
            /* convert pig schema (not-null) */
            fieldSchema = convert(pigFields[i], false);
        } else { 
            /* validate pigFields[i] with given avro schema */
            fieldSchema = validateAndConvert(inputField.schema(),
                                            pigFields[i]);
        }

        /* get field name of output */
        String outname = (isPartialSchema) ? pigFields[i].getName() : inputField.name();
        if (outname == null)
            outname = FIELD_NAME + "_" + i; // field name cannot be null

        /* get doc of output */
        String doc = (isPartialSchema) ? pigFields[i].getDescription() : inputField.doc();

        JsonNode defaultvalue = (inputField != null) ? inputField.defaultValue() : null;

        outFields.add(new Field(outname, fieldSchema, doc, defaultvalue));

    }

    outSchema.setFields(outFields);
    return outSchema;

}
 

/**
 * Initialize given a schema
 */
protected void init(String namespace, Schema schema,
                                boolean ignoreNameMap) {

    /* put to map[type name]=>schema */
    if (isNamedSchema(schema)) {
        String typeName = schema.getName();
        if (typeName2Schema.containsKey(typeName))
            AvroStorageLog.warn("Duplicate schemas defined for type:"
                    + typeName
                    + ". will ignore the second one:"
                    + schema);
        else {
            AvroStorageLog.details("add " + schema.getName() + "=" + schema
                    + " to type2Schema");
            typeName2Schema.put(schema.getName(), schema);
        }
    }

    /* put field schema to map[field name]=>schema*/
    if (schema.getType().equals(Type.RECORD)) {

        List<Field> fields = schema.getFields();
        for (Field field : fields) {

            Schema fieldSchema = field.schema();
            String name = (namespace == null) ? field.name()  : namespace + "." + field.name();

            if (!ignoreNameMap) {
                if (name2Schema.containsKey(name))
                    AvroStorageLog.warn("Duplicate schemas defined for alias:" + name
                                      + ". Will ignore the second one:"+ fieldSchema);
                else {
                    AvroStorageLog.details("add " + name + "=" + fieldSchema + " to name2Schema");
                    name2Schema.put(name, fieldSchema);
                }
            }

            init(name, fieldSchema, ignoreNameMap);
        }
    } else if (schema.getType().equals(Type.UNION)) {

        if (AvroStorageUtils.isAcceptableUnion(schema)) {
            Schema realSchema = AvroStorageUtils.getAcceptedType(schema);
            init(namespace, realSchema, ignoreNameMap);
        } else {
            List<Schema> list = schema.getTypes();
            for (Schema s : list) {
                init(namespace, s, true);
            }
        }
    } else if (schema.getType().equals(Type.ARRAY)) {
        Schema elemSchema = schema.getElementType();
        init(namespace, elemSchema, true);
    } else if (schema.getType().equals(Type.MAP)) {
        Schema valueSchema = schema.getValueType();
        init(namespace, valueSchema, true);
    }
}
 

/**
 * Packs a Pig Tuple into an Avro record.
 * @param t the Pig tuple to pack into the avro object
 * @param s The avro schema for which to determine the type
 * @return the avro record corresponding to the input tuple
 * @throws IOException
 */
public static GenericData.Record packIntoAvro(final Tuple t, final Schema s)
    throws IOException {

  try {
    GenericData.Record record = new GenericData.Record(s);
    for (Field f : s.getFields()) {
      Object o = t.get(f.pos());
      Schema innerSchema = f.schema();
      if (AvroStorageSchemaConversionUtilities.isNullableUnion(innerSchema)) {
        if (o == null) {
          record.put(f.pos(), null);
          continue;
        }
        innerSchema = AvroStorageSchemaConversionUtilities
            .removeSimpleUnion(innerSchema);
      }
      switch(innerSchema.getType()) {
      case RECORD:
        record.put(f.pos(), packIntoAvro((Tuple) o, innerSchema));
        break;
      case ARRAY:
        record.put(f.pos(), packIntoAvro((DataBag) o, innerSchema));
        break;
      case BYTES:
        record.put(f.pos(), ByteBuffer.wrap(((DataByteArray) o).get()));
        break;
      case FIXED:
        record.put(f.pos(), new GenericData.Fixed(
            innerSchema, ((DataByteArray) o).get()));
        break;
      default:
        if (t.getType(f.pos()) == DataType.DATETIME) {
          record.put(f.pos(), ((DateTime) o).getMillis() );
        } else {
          record.put(f.pos(), o);
        }
      }
    }
    return record;
  } catch (Exception e) {
    throw new IOException(
        "exception in AvroStorageDataConversionUtilities.packIntoAvro", e);
  }
}
 

/**
 * Initialize given a schema
 */
protected void init(String namespace, Schema schema,
                                boolean ignoreNameMap) {

    /* put to map[type name]=>schema */
    if (isNamedSchema(schema)) {
        String typeName = schema.getName();
        if (typeName2Schema.containsKey(typeName))
            AvroStorageLog.warn("Duplicate schemas defined for type:"
                    + typeName
                    + ". will ignore the second one:"
                    + schema);
        else {
            AvroStorageLog.details("add " + schema.getName() + "=" + schema
                    + " to type2Schema");
            typeName2Schema.put(schema.getName(), schema);
        }
    }

    /* put field schema to map[field name]=>schema*/
    if (schema.getType().equals(Type.RECORD)) {

        List<Field> fields = schema.getFields();
        for (Field field : fields) {

            Schema fieldSchema = field.schema();
            String name = (namespace == null) ? field.name()  : namespace + "." + field.name();

            if (!ignoreNameMap) {
                if (name2Schema.containsKey(name))
                    AvroStorageLog.warn("Duplicate schemas defined for alias:" + name
                                      + ". Will ignore the second one:"+ fieldSchema);
                else {
                    AvroStorageLog.details("add " + name + "=" + fieldSchema + " to name2Schema");
                    name2Schema.put(name, fieldSchema);
                }
            }

            init(name, fieldSchema, ignoreNameMap);
        }
    } else if (schema.getType().equals(Type.UNION)) {

        if (AvroStorageUtils.isAcceptableUnion(schema)) {
            Schema realSchema = AvroStorageUtils.getAcceptedType(schema);
            init(namespace, realSchema, ignoreNameMap);
        } else {
            List<Schema> list = schema.getTypes();
            for (Schema s : list) {
                init(namespace, s, true);
            }
        }
    } else if (schema.getType().equals(Type.ARRAY)) {
        Schema elemSchema = schema.getElementType();
        init(namespace, elemSchema, true);
    } else if (schema.getType().equals(Type.MAP)) {
        Schema valueSchema = schema.getValueType();
        init(namespace, valueSchema, true);
    }
}
 

/**
 * Constructor for AvroEntitySerDe instances.
 * 
 * @param entityComposer
 *          An entity composer that can construct Avro entities
 * @param avroSchema
 *          The avro schema for entities this SerDe serializes and
 *          deserializes
 * @param writtenAvroSchema
 *          The avro schema a record we are reading was written with
 * @param specific
 *          True if the entity is a Specific avro record. False indicates it's
 *          a generic
 */
public AvroEntitySerDe(EntityComposer<E> entityComposer,
    AvroEntitySchema avroSchema, AvroEntitySchema writtenAvroSchema,
    boolean specific) {
  super(entityComposer);
  this.specific = specific;
  this.avroSchema = avroSchema;
  this.defaultValueMap = AvroUtils.getDefaultValueMap(avroSchema.getAvroSchema());

  // For each field in entity, initialize the appropriate datum readers and
  // writers.
  for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) {
    String fieldName = fieldMapping.getFieldName();
    Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName)
        .schema();
    Field writtenField = writtenAvroSchema.getAvroSchema()
        .getField(fieldName);
    if (writtenField == null) {
      // No field for the written version, so don't worry about datum
      // readers and writers.
      continue;
    }
    Schema writtenFieldSchema = writtenField.schema();

    if (fieldMapping.getMappingType() == MappingType.COLUMN
        || fieldMapping.getMappingType() == MappingType.COUNTER) {
      initColumnDatumMaps(fieldName, fieldSchema, writtenFieldSchema);
    } else if (fieldMapping.getMappingType() == MappingType.KEY_AS_COLUMN) {
      if (fieldSchema.getType() == Schema.Type.RECORD) {
        // Each field of the kac record has a different type, so we need
        // to track each one in a different map.
        initKACRecordDatumMaps(fieldName, fieldSchema, writtenFieldSchema);
      } else if (fieldSchema.getType() == Schema.Type.MAP) {
        // Only one value type for a map, so just put the type in the column
        // datum maps.
        initColumnDatumMaps(fieldName, fieldSchema.getValueType(),
            writtenFieldSchema.getValueType());
      } else {
        throw new ValidationException(
            "Unsupported type for keyAsColumn: " + fieldSchema.getType());
      }
    }
  }
}
 

/**
 * Given an avro Schema.Field instance, make a clone of it.
 * 
 * @param field
 *          The field to clone.
 * @return The cloned field.
 */
public static Field cloneField(Field field) {
  return new Field(field.name(), field.schema(), field.doc(),
      field.defaultValue());
}