Java Code Examples for org.kitesdk.data.ValidationException#check()

@Override
public void initialize() {
  Preconditions.checkState(state.equals(ReaderWriterState.NEW),
      "Unable to open a writer from state:%s", state);

  DatasetDescriptor descriptor = view.getDataset().getDescriptor();
  ValidationException.check(
      FileSystemWriter.isSupportedFormat(descriptor),
      "Not a supported format: %s", descriptor.getFormat());

  LOG.debug("Opening partitioned dataset writer w/strategy:{}",
    partitionStrategy);

  cachedWriters = CacheBuilder.newBuilder().maximumSize(maxWriters)
    .removalListener(new DatasetWriterCloser<E>())
    .build(createCacheLoader());

  state = ReaderWriterState.OPEN;
}
 

/**
 * Returns the nested {@link Schema} for the given field name.
 *
 * @param schema a record Schema
 * @param name a String field name
 * @return the nested Schema for the field
 */
public static Schema fieldSchema(Schema schema, String name) {
  Schema nested = unwrapNullable(schema);
  List<String> levels = Lists.newArrayList();
  for (String level : NAME_SPLITTER.split(name)) {
    levels.add(level);
    ValidationException.check(Schema.Type.RECORD == schema.getType(),
        "Cannot get schema for %s: %s is not a record schema: %s",
        name, NAME_JOINER.join(levels), nested.toString(true));
    Schema.Field field = nested.getField(level);
    ValidationException.check(field != null,
        "Cannot get schema for %s: %s is not a field",
        name, NAME_JOINER.join(levels));
    nested = unwrapNullable(field.schema());
  }
  return nested;
}
 

/**
 * Precondition-style validation that a dataset name is compatible.
 *
 * @param namespace a String namespace
 * @param name a String name
 */
public static void checkDatasetName(String namespace, String name) {
  Preconditions.checkNotNull(namespace, "Namespace cannot be null");
  Preconditions.checkNotNull(name, "Dataset name cannot be null");
  ValidationException.check(Compatibility.isCompatibleName(namespace),
      "Namespace %s is not alphanumeric (plus '_')",
      namespace);
  ValidationException.check(Compatibility.isCompatibleName(name),
      "Dataset name %s is not alphanumeric (plus '_')",
      name);
}
 

/**
 * Precondition-style validation that a {@link Schema} is compatible.
 *
 * @param schema an avro {@code Schema}
 */
public static void checkSchema(Schema schema) {
  Preconditions.checkNotNull(schema, "Schema cannot be null");
  List<String> incompatible = getIncompatibleNames(schema);
  ValidationException.check(incompatible.isEmpty(),
      "Field names are not alphanumeric (plus '_'): %s",
      Joiner.on(", ").join(incompatible));
}
 

private static void checkNotChanged(String what,
                                    @Nullable Object existing,
                                    @Nullable Object test) {
  ValidationException.check(
      (existing == test) || (existing != null && existing.equals(test)),
      "Dataset %s is not compatible with existing: %s != %s",
      what, String.valueOf(existing), String.valueOf(test));
}
 

public static void checkStrategyUpdate(PartitionStrategy existing,
                                       PartitionStrategy other,
                                       Schema schema) {
  List<FieldPartitioner> existingFields = Accessor.getDefault()
      .getFieldPartitioners(existing);
  List<FieldPartitioner> otherFields = Accessor.getDefault()
      .getFieldPartitioners(other);
  ValidationException.check(existingFields.size() == otherFields.size(),
      "Not compatible: cannot replace %s partitioners with %s partitioners",
      existingFields.size(), otherFields.size());

  for (int i = 0; i < existingFields.size(); i += 1) {
    FieldPartitioner fp = existingFields.get(i);
    FieldPartitioner replacement = otherFields.get(i);
    if (fp.equals(replacement)) {
      continue;
    }

    ValidationException.check(fp instanceof ProvidedFieldPartitioner,
        "Cannot replace partition %s: not a provided partitioner",
        fp.getName());
    ValidationException.check(fp.getName().equals(replacement.getName()),
        "Cannot change the name of partition %s (to %s)",
        fp.getName(), replacement.getName());
    Class<?> outputType = SchemaUtil.getPartitionType(replacement, schema);
    ValidationException.check(
        isCompatibleWithProvidedType(fp.getType(), outputType),
        "Cannot change the data type of partition %s", fp.getName());
  }
}
 

/**
 * Parses the FieldMapping from an annotated schema field.
 *
 * @param mappingNode
 *          The value of the "mapping" node
 * @return FieldMapping
 */
public static FieldMapping parseFieldMapping(JsonNode mappingNode) {
  ValidationException.check(mappingNode.isObject(),
      "A column mapping must be a JSON record");

  ValidationException.check(mappingNode.has(SOURCE),
      "Partitioners must have a %s.", SOURCE);
  String source = mappingNode.get("source").asText();

  return parseFieldMapping(source, mappingNode);
}
 

private static ColumnMapping buildColumnMapping(JsonNode node) {
  ValidationException.check(node.isArray(),
      "Must be a JSON array of column mappings");

  ColumnMapping.Builder builder = new ColumnMapping.Builder();
  for (Iterator<JsonNode> it = node.elements(); it.hasNext();) {
    builder.fieldMapping(parseFieldMapping(it.next()));
  }
  return builder.build();
}
 

@Override
public Object extractField(E entity, String fieldName) {
  // make sure the field is a direct child of the schema
  ValidationException.check(
      accessor.getReadSchema().getField(fieldName) != null,
      "No field named %s in schema %s", fieldName, accessor.getReadSchema());
  return accessor.get(entity, fieldName);
}
 

@Override
public int run() throws IOException {
  PartitionStrategy.Builder strategyBuilder = new PartitionStrategy.Builder();
  for (String partition : partitions) {
    Matcher m = PARTITION_FIELD.matcher(partition);
    if (m.matches()) {
      String fieldName = m.group(1);
      ValidationException.check(fieldName != null && !fieldName.isEmpty(),
          "Invalid field name: %s", String.valueOf(fieldName));
      String partitionerType = m.group(2);
      if ("hash".equals(partitionerType)) {
        String width = m.group(3);
        ValidationException.check(width != null,
            "Missing number of hash partitions: %s:hash[?]", fieldName);
        strategyBuilder.hash(fieldName, Integer.parseInt(width));
      } else if ("copy".equals(partitionerType)) {
        strategyBuilder.identity(fieldName);
      } else if ("year".equals(partitionerType)) {
        strategyBuilder.year(fieldName);
      } else if ("month".equals(partitionerType)) {
        strategyBuilder.month(fieldName);
      } else if ("day".equals(partitionerType)) {
        strategyBuilder.day(fieldName);
      } else if ("hour".equals(partitionerType)) {
        strategyBuilder.hour(fieldName);
      } else if ("minute".equals(partitionerType)) {
        strategyBuilder.minute(fieldName);
      } else if ("provided".equals(partitionerType)) {
        strategyBuilder.provided(fieldName);
      } else {
        throw new ValidationException(
            "Unknown partitioner type: " + partitionerType);
      }
    } else {
      throw new ValidationException(
          "Invalid partition <field:type>: " + partition);
    }
  }

  // building the descriptor validates the schema and strategy
  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
      .partitionStrategy(strategyBuilder.build())
      .schema(open(avroSchemaFile))
      .build();

  String strategy = descriptor.getPartitionStrategy().toString(!minimize);

  output(strategy, console, outputPath);

  return 0;
}
 

private static PartitionStrategy buildPartitionStrategy(JsonNode node) {
  ValidationException.check(node.isArray(),
      "A partition strategy must be a JSON array of partitioners");

  PartitionStrategy.Builder builder = new PartitionStrategy.Builder();
  for (Iterator<JsonNode> it = node.elements(); it.hasNext();) {
    JsonNode fieldPartitioner = it.next();
    ValidationException.check(fieldPartitioner.isObject(),
        "A partitioner must be a JSON record");

    ValidationException.check(fieldPartitioner.has(TYPE),
        "Partitioners must have a %s", TYPE);
    String type = fieldPartitioner.get(TYPE).asText();

    // only provided partitioners do not need a source field
    boolean isProvided = type.equals("provided");
    ValidationException.check(isProvided || fieldPartitioner.has(SOURCE),
        "Partitioners must have a %s", SOURCE);

    String source = null;
    // only provided has no source field
    if (!isProvided) {
      source = fieldPartitioner.get(SOURCE).asText();
    }

    String name = null;
    if (fieldPartitioner.has(NAME)) {
      name = fieldPartitioner.get(NAME).asText();
    }

    // Note: string range, int range, and list partitioners are not supported
    if (type.equals("identity")) {
      builder.identity(source, name);
    } else if (type.equals("hash")) {
      ValidationException.check(fieldPartitioner.has(BUCKETS),
          "Hash partitioner %s must have attribute %s",
          name == null ? source : name, BUCKETS);
      int buckets = fieldPartitioner.get(BUCKETS).asInt();
      ValidationException.check(buckets > 0,
          "Invalid number of buckets for hash partitioner %s: %s",
          name == null ? source : name,
          fieldPartitioner.get(BUCKETS).asText());
      builder.hash(source, name, buckets);
    } else if (type.equals("range")) {
      ValidationException.check(fieldPartitioner.has(SIZE),
          "Range partitioner %s must have attribute %s",
          name == null ? source : name, SIZE);
      long size = fieldPartitioner.get(SIZE).asLong();
      ValidationException.check(size > 0,
          "Invalid size for range partitioner %s: %s",
          name == null ? source : name,
          fieldPartitioner.get(SIZE).asText());
      builder.fixedSizeRange(source, name, size);
    } else if (type.equals("year")) {
      builder.year(source, name);
    } else if (type.equals("month")) {
      builder.month(source, name);
    } else if (type.equals("day")) {
      builder.day(source, name);
    } else if (type.equals("hour")) {
      builder.hour(source, name);
    } else if (type.equals("minute")) {
      builder.minute(source, name);
    } else if (type.equals("dateFormat")) {
      ValidationException.check(name != null,
          "Date format partitioner %s must have a %s.", source, NAME);
      ValidationException.check(fieldPartitioner.has(FORMAT),
          "Date format partitioner %s must have a %s.", name, FORMAT);
      String format = fieldPartitioner.get(FORMAT).asText();
      builder.dateFormat(source, name, format);
    } else if (isProvided) {
      ValidationException.check(name != null,
          "Provided partitioners must have a %s.", NAME);
      String valuesType = null;
      if (fieldPartitioner.has(VALUES)) {
        valuesType = fieldPartitioner.get(VALUES).asText();
      }
      builder.provided(name, valuesType);

    } else {
      throw new ValidationException("Invalid FieldPartitioner: " + type);
    }
  }
  return builder.build();
}
 

/**
 * Parses the FieldMapping from an annotated schema field.
 *
 * @param source
 *          The source field name for this mapping
 * @param mappingNode
 *          The value of the "mapping" node
 * @return FieldMapping
 */
public static FieldMapping parseFieldMapping(String source, JsonNode mappingNode) {
  ValidationException.check(mappingNode.isObject(),
      "A column mapping must be a JSON record");

  ValidationException.check(mappingNode.has(TYPE),
      "Column mappings must have a %s.", TYPE);
  String type = mappingNode.get(TYPE).asText();

  // return easy cases
  if ("occVersion".equals(type)) {
    return FieldMapping.version(source);
  } else if ("key".equals(type)) {
    return FieldMapping.key(source);
  }

  String family = null;
  String qualifier = null;
  String prefix = null;

  // for backward-compatibility, check for "value": "fam:qual"
  if (mappingNode.has(VALUE)) {
    // avoids String#split because of odd cases, like ":".split(":")
    String value = mappingNode.get(VALUE).asText();
    Iterator<String> values = VALUE_SPLITTER.split(value).iterator();
    if (values.hasNext()) {
      family = values.next();
    }
    if (values.hasNext()) {
      if ("keyAsColumn".equals(type)) {
        prefix = values.next();
        if (prefix.isEmpty()) {
          prefix = null;
        }
      } else {
        qualifier = values.next();
      }
    }
  }

  // replace any existing values with explicit family and qualifier
  if (mappingNode.has(FAMILY)) {
    family = mappingNode.get(FAMILY).textValue();
  }
  if (mappingNode.has(QUALIFIER)) {
    qualifier = mappingNode.get(QUALIFIER).textValue();
  }

  if ("column".equals(type)) {
    ValidationException.check(family != null && !family.isEmpty(),
        "Column mapping %s must have a %s", source, FAMILY);
    ValidationException.check(qualifier != null && !qualifier.isEmpty(),
        "Column mapping %s must have a %s", source, QUALIFIER);
    return FieldMapping.column(source, family, qualifier);

  } else if ("keyAsColumn".equals(type)) {
    ValidationException.check(family != null && !family.isEmpty(),
        "Column mapping %s must have a %s", source, FAMILY);
    ValidationException.check(qualifier == null,
        "Key-as-column mapping %s cannot have a %s", source, QUALIFIER);
    if (mappingNode.has(PREFIX)) {
      prefix = mappingNode.get(PREFIX).asText();
      if (prefix.isEmpty()) {
        prefix = null;
      }
    }
    return FieldMapping.keyAsColumn(source, family, prefix);

  } else if ("counter".equals(type)) {
    ValidationException.check(family != null && !family.isEmpty(),
        "Counter mapping %s must have a %s", source, FAMILY);
    ValidationException.check(qualifier != null && !qualifier.isEmpty(),
        "Counter mapping %s must have a %s", source, QUALIFIER);
    return FieldMapping.counter(source, family, qualifier);

  } else {
    throw new ValidationException("Invalid mapping type: " + type);
  }
}