Java Code Examples for org.apache.avro.Schema.Type#UNION

protected static boolean isValidValue(Field f, Object value) {
  if (value != null) {
    return true;
  } else {
    Schema schema = f.schema();
    Type type = schema.getType();
    if (type == Type.NULL) {
      return true;
    } else {
      if (type == Type.UNION) {
        Iterator i$ = schema.getTypes().iterator();

        while(i$.hasNext()) {
          Schema s = (Schema)i$.next();
          if (s.getType() == Type.NULL) {
            return true;
          }
        }
      }

      return false;
    }
  }
}
 

/**
 * Convert a passed String value to the given type for the key as per Schema
 */
public static Object convertValueStringToAvroKeyType(Schema schema, String key, String value) throws ParseException
{
  Type type = null;

  if (schema.getField(key) != null) {
    type = schema.getField(key).schema().getType();
  } else {
    return value;
  }

  Object convertedValue = null;

  if (type == Type.UNION) {
    convertedValue = convertAndResolveUnionToPrimitive(schema, key, value);
  } else {
    convertedValue = convertValueToAvroPrimitive(type, key, value);
  }

  return convertedValue;

}
 

public void updatePartitionKeyAndRowKey() {
    Schema inputSchema = schema.schema.getValue();
    List<String> possibleValues = new ArrayList<String>();
    for (Field field : inputSchema.getFields()) {
        Schema fSchema = field.schema();
        if (fSchema.getType() == Type.UNION) {
            for (Schema s : field.schema().getTypes()) {
                if (s.getType() != Type.NULL) {
                    fSchema = s;
                    break;
                }
            }
        }
        if (fSchema.getType().equals(Type.STRING)) {
            possibleValues.add(field.name());
        }
    }
    partitionKey.setPossibleValues(possibleValues);
    rowKey.setPossibleValues(possibleValues);
}
 

@Override
public void onVisit(Schema schema, Collection<String> breadcrumb) {
  if (Type.UNION == schema.getType()) {
    List<Schema> types = schema.getTypes();
    if (types.size() != 2 || (types.get(0).getType() != NULL && types.get(1).getType() != NULL)) {
      String path = breadcrumb.stream().collect(joining("/", "/", ""));
      throw new IllegalArgumentException(String
          .format("Only union[any, null] or union[null, any] are supported. At path %s got %s", path, schema));
    }
  }
}
 

/**
 * Check to see if there is a default value to use, if not will throw
 * {@link IllegalArgumentException}
 */
private static <T extends JsonNode> JsonNode possiblyGetDefaultValue(Field field, Class<T> expectedDefaultType) {
    JsonNode jsonNode = retrieveDefaultFieldValue(field);
    if (field.schema().getType() != Type.UNION && !expectedDefaultType.isAssignableFrom(jsonNode.getClass())) {
        // since we do not support schema evolution here we need to throw an
        // exception here as the data is in error.
        throw new IllegalArgumentException("The field '" + field.name() + "' has a default value that "
                + "does not match the field type. Field Type is: '" + expectedDefaultType.getName() + "' and the "
                + "default value type is: '" + field.defaultValue().toString());
    }
    return jsonNode;
}
 

public static boolean hasNonNullUnion(org.apache.avro.Schema schema) {
  if (schema.getType() == Type.UNION) {
    final List<org.apache.avro.Schema> types = schema.getTypes();

    if (types.size() == 2) {
      return !types.contains(NULL_SCHEMA);
    } else {
      return true;
    }
  }

  return false;
}
 

/**
 * Checks to see if an avro schema is a combination of
 * null and another object.
 * @param s The object to check
 * @return whether it's a nullable union
 */
public static boolean isNullableUnion(final Schema s) {
  return (
      s.getType() == Type.UNION
      && ((s.getTypes().size() == 1)
          || (s.getTypes().size() == 2
           && (s.getTypes().get(0).getType() == Type.NULL
              || s.getTypes().get(1).getType() == Type.NULL))));
}
 

/**
 * Given an input schema that is a union of an avro schema
 * and null (or just a union with one type), return the avro schema.
 * @param s The input schema object
 * @return The non-null part of the union
 */
public static Schema removeSimpleUnion(final Schema s) {
  if (s.getType() == Type.UNION) {
    List<Schema> types = s.getTypes();
    for (Schema t : types) {
      if (t.getType() != Type.NULL) {
        return t;
      }
    }
  }
  return s;
}
 

public Map<String,Schema> getMapInputSchemas()
{    
  if (_mapInputSchemas == null)
  {
    _mapInputSchemas = new HashMap<String,Schema>();
    
    for (Entry<String,String> schemaPair : _inputSchemas.entrySet())
    {
      Schema schema = new Schema.Parser().parse(schemaPair.getValue());
      
      List<Schema> mapInputSchemas = new ArrayList<Schema>();
      
      if (schema.getType() == Type.UNION)
      {
        mapInputSchemas.addAll(schema.getTypes());
      }
      else
      {
        mapInputSchemas.add(schema);
      }
      
      // feedback from output (optional)
      mapInputSchemas.add(getReduceOutputSchema());
      
      _mapInputSchemas.put(schemaPair.getKey(), Schema.createUnion(mapInputSchemas));
    }
    
    
  }
  return Collections.unmodifiableMap(_mapInputSchemas);
}
 

private static Optional<Schema> unpackNullableUnion(final Schema source) {
    if (source.getType() == Type.UNION) {
        if (source.getTypes().size() != 2) {
            return Optional.empty();
        } else {
            return source.getTypes().stream().filter((t) -> t.getType() != Type.NULL).findFirst();
        }
    } else {
        return Optional.of(source);
    }
}
 

private static org.apache.avro.Schema extractSchemaFromUnionIfNeeded(org.apache.avro.Schema fieldSchema) {
  if ((fieldSchema).getType() == Type.UNION) {
    fieldSchema = ((org.apache.avro.Schema) CollectionUtils.find(fieldSchema.getTypes(), new Predicate() {
      @Override
      public boolean evaluate(Object object) {
        return ((org.apache.avro.Schema) object).getType() != Type.NULL;
      }
    }));
  }
  return fieldSchema;
}
 

private static Schema nullable(final Schema schema) {
    if (schema.getType() == Type.UNION) {
        final List<Schema> unionTypes = new ArrayList<>(schema.getTypes());
        final Schema nullSchema = Schema.create(Type.NULL);
        if (unionTypes.contains(nullSchema)) {
            return schema;
        }

        unionTypes.add(nullSchema);
        return Schema.createUnion(unionTypes);
    }

    return Schema.createUnion(Schema.create(Type.NULL), schema);
}
 

public static boolean isNullable(final Schema schema) {
    final Type schemaType = schema.getType();
    if (schemaType == Type.UNION) {
        for (final Schema unionSchema : schema.getTypes()) {
            if (isNullable(unionSchema)) {
                return true;
            }
        }
    }

    return schemaType == Type.NULL;
}
 

private final Type checkTypeConstraintsAndGetType(final Schema schema) {
	final Type type = schema.getType();
	if (type == Type.RECORD) {
		throw new RuntimeException("The given Avro file contains complex data types which are not supported right now");
	}

	if (type == Type.UNION) {
		List<Schema> types = schema.getTypes();
		if (types.size() > 2) {
			throw new RuntimeException("The given Avro file contains a union that has more than two elements");
		}
		if (types.size() == 1 && types.get(0).getType() != Type.UNION) {
			return types.get(0).getType();
		}
		if (types.get(0).getType() == Type.UNION || types.get(1).getType() == Type.UNION) {
			throw new RuntimeException("The given Avro file contains a nested union");
		}
		if (types.get(0).getType() == Type.NULL) {
			return types.get(1).getType();
		} else {
			if (types.get(1).getType() != Type.NULL) {
				throw new RuntimeException("The given Avro file is contains a union with two non-null types.");
			}
			return types.get(0).getType();
		}
	}
	return type;
}
 

/**
 *
 */
private static void updateRecord(Field field, Type type, String providedValue, Record avroRecord) {
    if (Type.NULL != type) {
        Object value;
        if (Type.INT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, IntNode.class).getIntValue()
                    : Integer.parseInt(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.BOOLEAN == type) {
            value = null == providedValue
                    ? possiblyGetDefaultValue(field, BooleanNode.class).getBooleanValue()
                    : Boolean.parseBoolean(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.DOUBLE == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Double.parseDouble(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.FLOAT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Float.parseFloat(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.LONG == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, LongNode.class).getLongValue()
                    : Long.parseLong(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.STRING == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, TextNode.class).getTextValue()
                    : providedValue;
            avroRecord.put(field.name(), value);
        } else if (Type.BYTES == type) {
            value = encodeLogicalType(field, providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.UNION == type) {
            field.schema().getTypes()
                    .forEach(schema -> updateRecord(field, schema.getType(), providedValue, avroRecord));
        } else if (Type.ARRAY == type || Type.ENUM == type || Type.FIXED == type || Type.MAP == type
                || Type.NULL == type || Type.RECORD == type) {
            throw new IllegalArgumentException("The field type '" + type + "' is not supported at the moment");
        } else {
            avroRecord.put(field.name(), providedValue);
        }
    }
}