Java Code Examples for org.apache.avro.Schema.Type#NULL

protected static boolean isValidValue(Field f, Object value) {
  if (value != null) {
    return true;
  } else {
    Schema schema = f.schema();
    Type type = schema.getType();
    if (type == Type.NULL) {
      return true;
    } else {
      if (type == Type.UNION) {
        Iterator i$ = schema.getTypes().iterator();

        while(i$.hasNext()) {
          Schema s = (Schema)i$.next();
          if (s.getType() == Type.NULL) {
            return true;
          }
        }
      }

      return false;
    }
  }
}
 

private static Object convertAndResolveUnionToPrimitive(Schema schema, String key, String value) throws ParseException
{
  Schema unionSchema = schema.getField(key).schema();
  List<Schema> types = unionSchema.getTypes();
  Object convertedValue = null;
  for (int i = 0; i < types.size(); i++) {
    try {
      if (types.get(i).getType() == Type.NULL) {
        if (value == null || value.equals("null")) {
          convertedValue = null;
          break;
        } else {
          continue;
        }
      }
      convertedValue = convertValueToAvroPrimitive(types.get(i).getType(), key, value);
    } catch (RuntimeException e) {
      LOG.error("Could not handle schema resolution", e);
      continue;
    }
    break;
  }

  return convertedValue;
}
 

public void updatePartitionKeyAndRowKey() {
    Schema inputSchema = schema.schema.getValue();
    List<String> possibleValues = new ArrayList<String>();
    for (Field field : inputSchema.getFields()) {
        Schema fSchema = field.schema();
        if (fSchema.getType() == Type.UNION) {
            for (Schema s : field.schema().getTypes()) {
                if (s.getType() != Type.NULL) {
                    fSchema = s;
                    break;
                }
            }
        }
        if (fSchema.getType().equals(Type.STRING)) {
            possibleValues.add(field.name());
        }
    }
    partitionKey.setPossibleValues(possibleValues);
    rowKey.setPossibleValues(possibleValues);
}
 

/**
 * Get easyly the Avro field type from its schema
 *
 * @param field concerned
 * @return Avro.Type
 */
public static Type getFieldType(Field field) {
    Schema convSchema = field.schema();
    Type type = field.schema().getType();
    if (convSchema.getType().equals(Type.UNION)) {
        for (Schema s : field.schema().getTypes()) {
            if (s.getType() != Type.NULL) {
                type = s.getType();
                break;
            }
        }
    }
    return type;
}
 

/**
 * Checks to see if an avro schema is a combination of
 * null and another object.
 * @param s The object to check
 * @return whether it's a nullable union
 */
public static boolean isNullableUnion(final Schema s) {
  return (
      s.getType() == Type.UNION
      && ((s.getTypes().size() == 1)
          || (s.getTypes().size() == 2
           && (s.getTypes().get(0).getType() == Type.NULL
              || s.getTypes().get(1).getType() == Type.NULL))));
}
 

/**
 * Given an input schema that is a union of an avro schema
 * and null (or just a union with one type), return the avro schema.
 * @param s The input schema object
 * @return The non-null part of the union
 */
public static Schema removeSimpleUnion(final Schema s) {
  if (s.getType() == Type.UNION) {
    List<Schema> types = s.getTypes();
    for (Schema t : types) {
      if (t.getType() != Type.NULL) {
        return t;
      }
    }
  }
  return s;
}
 

private static List<Schema> getNonNullSubSchemas(final Schema avroSchema) {
    final List<Schema> unionFieldSchemas = avroSchema.getTypes();
    if (unionFieldSchemas == null) {
        return Collections.emptyList();
    }

    final List<Schema> nonNullTypes = new ArrayList<>(unionFieldSchemas.size());
    for (final Schema fieldSchema : unionFieldSchemas) {
        if (fieldSchema.getType() != Type.NULL) {
            nonNullTypes.add(fieldSchema);
        }
    }

    return nonNullTypes;
}
 

public static boolean isNullable(final Schema schema) {
    final Type schemaType = schema.getType();
    if (schemaType == Type.UNION) {
        for (final Schema unionSchema : schema.getTypes()) {
            if (isNullable(unionSchema)) {
                return true;
            }
        }
    }

    return schemaType == Type.NULL;
}
 

private final Type checkTypeConstraintsAndGetType(final Schema schema) {
	final Type type = schema.getType();
	if (type == Type.RECORD) {
		throw new RuntimeException("The given Avro file contains complex data types which are not supported right now");
	}

	if (type == Type.UNION) {
		List<Schema> types = schema.getTypes();
		if (types.size() > 2) {
			throw new RuntimeException("The given Avro file contains a union that has more than two elements");
		}
		if (types.size() == 1 && types.get(0).getType() != Type.UNION) {
			return types.get(0).getType();
		}
		if (types.get(0).getType() == Type.UNION || types.get(1).getType() == Type.UNION) {
			throw new RuntimeException("The given Avro file contains a nested union");
		}
		if (types.get(0).getType() == Type.NULL) {
			return types.get(1).getType();
		} else {
			if (types.get(1).getType() != Type.NULL) {
				throw new RuntimeException("The given Avro file is contains a union with two non-null types.");
			}
			return types.get(0).getType();
		}
	}
	return type;
}
 

/**
 *
 */
private static void updateRecord(Field field, Type type, String providedValue, Record avroRecord) {
    if (Type.NULL != type) {
        Object value;
        if (Type.INT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, IntNode.class).getIntValue()
                    : Integer.parseInt(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.BOOLEAN == type) {
            value = null == providedValue
                    ? possiblyGetDefaultValue(field, BooleanNode.class).getBooleanValue()
                    : Boolean.parseBoolean(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.DOUBLE == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Double.parseDouble(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.FLOAT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Float.parseFloat(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.LONG == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, LongNode.class).getLongValue()
                    : Long.parseLong(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.STRING == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, TextNode.class).getTextValue()
                    : providedValue;
            avroRecord.put(field.name(), value);
        } else if (Type.BYTES == type) {
            value = encodeLogicalType(field, providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.UNION == type) {
            field.schema().getTypes()
                    .forEach(schema -> updateRecord(field, schema.getType(), providedValue, avroRecord));
        } else if (Type.ARRAY == type || Type.ENUM == type || Type.FIXED == type || Type.MAP == type
                || Type.NULL == type || Type.RECORD == type) {
            throw new IllegalArgumentException("The field type '" + type + "' is not supported at the moment");
        } else {
            avroRecord.put(field.name(), providedValue);
        }
    }
}
 

/**
 * Determines the pig object type of the Avro schema.
 * @param s The avro schema for which to determine the type
 * @return the byte representing the schema type
 * @throws ExecException
 * @see org.apache.avro.Schema.Type
 */
public static byte getPigType(final Schema s) throws ExecException {
  switch (s.getType()) {
  case ARRAY:
    return DataType.BAG;
  case BOOLEAN:
    return DataType.BOOLEAN;
  case BYTES:
    return DataType.BYTEARRAY;
  case DOUBLE:
    return DataType.DOUBLE;
  case ENUM:
    return DataType.CHARARRAY;
  case FIXED:
    return DataType.BYTEARRAY;
  case FLOAT:
    return DataType.FLOAT;
  case INT:
    return DataType.INTEGER;
  case LONG:
    return DataType.LONG;
  case MAP:
    return DataType.MAP;
  case NULL:
    return DataType.NULL;
  case RECORD:
    return DataType.TUPLE;
  case STRING:
    return DataType.CHARARRAY;
  case UNION:
    List<Schema> types = s.getTypes();
    if (types.size() == 1) {
      return getPigType(types.get(0));
    } else if (types.size() == 2 && types.get(0).getType() == Type.NULL) {
        return getPigType(types.get(1));
    } else if (types.size() == 2 && types.get(1).getType() == Type.NULL) {
        return getPigType(types.get(0));
    } else if (isUnionOfSimpleTypes(s)) {
        return DataType.BYTEARRAY;
    }
    throw new ExecException(
        "Currently only supports element unions of a type and null (" + s.toString() +")");
  default:
    throw new ExecException("Unknown type: " + s.getType().toString());
  }
}