Java Code Examples for org.apache.avro.Schema.Type#ARRAY

The following examples show how to use org.apache.avro.Schema.Type#ARRAY . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AvroUtils.java    From Cubert with Apache License 2.0 6 votes vote down vote up 
private static Type convertToAvroType(DataType colType)
{
    final Type subType;
    if (colType == DataType.TUPLE)
    {
        /* Pig converts RECORD to TUPLE. Converting it back. */
        subType = Type.RECORD;
    }
    else if (colType == DataType.BAG)
    {
        subType = Type.ARRAY;
    }
    else if (colType == DataType.MAP)
    {
      subType = Type.MAP;
    }
    else
    {
        subType = Type.valueOf(colType.toString().toUpperCase());
    }
    return subType;
}
Example 2
Source File: SegmentTestUtils.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up 
public static DataType getColumnType(Field field) {
  org.apache.avro.Schema fieldSchema = field.schema();
  fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);

  final Type type = fieldSchema.getType();
  if (type == Type.ARRAY) {
    org.apache.avro.Schema elementSchema = extractSchemaFromUnionIfNeeded(fieldSchema.getElementType());
    if (elementSchema.getType() == Type.RECORD) {
      if (elementSchema.getFields().size() == 1) {
        elementSchema = elementSchema.getFields().get(0).schema();
      } else {
        throw new RuntimeException("More than one schema in Multi-value column!");
      }
      elementSchema = extractSchemaFromUnionIfNeeded(elementSchema);
    }
    return AvroSchemaUtil.valueOf(elementSchema.getType());
  } else {
    return AvroSchemaUtil.valueOf(type);
  }
}
Example 3
Source File: AvroUtils.java    From Cubert with Apache License 2.0 5 votes vote down vote up 
private static Field[] createFields(BlockSchema schema){
  Field[] fields = new Field[schema.getNumColumns()];
  for (int idx = 0; idx < fields.length; idx++)
  {
    final ColumnType col = schema.getColumnType(idx);
    final DataType colType = col.getType();
    final Type subType = convertToAvroType(colType);
    
    final Schema colSchema;
    if (col.getColumnSchema() != null ||
        subType == Type.ARRAY || subType == Type.MAP)
    {
      colSchema =
        convertFromBlockSchema(col.getName(),
                               subType,
                               col.getColumnSchema(), false);
      
    }
    else
    {
      List<Schema> unionSchema = new ArrayList<Schema>();
      unionSchema.add(Schema.create(Type.NULL));
      unionSchema.add(Schema.create(subType));
      
      colSchema = Schema.createUnion(unionSchema);
    }
    fields[idx] = new Field(col.getName(), colSchema, null, null);
  }
  return fields;
}
Example 4
Source File: AvroStorage.java    From spork with Apache License 2.0 5 votes vote down vote up 
/**
 * @see org.apache.pig.LoadFunc#getInputFormat()
 */
@Override
public InputFormat<NullWritable, GenericData.Record> getInputFormat()
    throws IOException {

  return new org.apache.pig.backend.hadoop.executionengine.mapReduceLayer
      .PigFileInputFormat<NullWritable, GenericData.Record>() {

    @Override
    public RecordReader<NullWritable, GenericData.Record>
      createRecordReader(final InputSplit is, final TaskAttemptContext tc)
        throws IOException, InterruptedException {
      Schema s = getInputAvroSchema();
      RecordReader<NullWritable, GenericData.Record> rr = null;
      if (s.getType() == Type.ARRAY) {
        rr = new AvroArrayReader(s);
      } else {
        rr = new AvroRecordReader(s);
      }
      try {
          rr.initialize(is, tc);
      } finally {
          rr.close();
      }
      tc.setStatus(is.toString());
      return rr;
    }
  };

}
Example 5
Source File: SegmentTestUtils.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up 
private static boolean isSingleValueField(Field field) {
  org.apache.avro.Schema fieldSchema = field.schema();
  fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);

  final Type type = fieldSchema.getType();
  if (type == Type.ARRAY) {
    return false;
  }
  return true;
}
Example 6
Source File: CSVUtils.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up 
/**
 *
 */
private static void updateRecord(Field field, Type type, String providedValue, Record avroRecord) {
    if (Type.NULL != type) {
        Object value;
        if (Type.INT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, IntNode.class).getIntValue()
                    : Integer.parseInt(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.BOOLEAN == type) {
            value = null == providedValue
                    ? possiblyGetDefaultValue(field, BooleanNode.class).getBooleanValue()
                    : Boolean.parseBoolean(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.DOUBLE == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Double.parseDouble(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.FLOAT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Float.parseFloat(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.LONG == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, LongNode.class).getLongValue()
                    : Long.parseLong(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.STRING == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, TextNode.class).getTextValue()
                    : providedValue;
            avroRecord.put(field.name(), value);
        } else if (Type.BYTES == type) {
            value = encodeLogicalType(field, providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.UNION == type) {
            field.schema().getTypes()
                    .forEach(schema -> updateRecord(field, schema.getType(), providedValue, avroRecord));
        } else if (Type.ARRAY == type || Type.ENUM == type || Type.FIXED == type || Type.MAP == type
                || Type.NULL == type || Type.RECORD == type) {
            throw new IllegalArgumentException("The field type '" + type + "' is not supported at the moment");
        } else {
            avroRecord.put(field.name(), providedValue);
        }
    }
}