org.kitesdk.morphline.api.MorphlineCompilationException Java Examples

/** Loads the given config file from the local file system */
public Config parse(File file, Config... overrides) throws IOException {
  if (file == null || file.getPath().trim().length() == 0) {
    throw new MorphlineCompilationException("Missing morphlineFile parameter", null);
  }
  if (!file.exists()) {
    throw new FileNotFoundException("File not found: " + file);
  }
  if (!file.canRead()) {
    throw new IOException("Insufficient permissions to read file: " + file);
  }
  Config config = ConfigFactory.parseFile(file);
  for (Config override : overrides) {
    config = override.withFallback(config);
  }
  
  synchronized (LOCK) {
    ConfigFactory.invalidateCaches();
    config = ConfigFactory.load(config);
    config.checkValid(ConfigFactory.defaultReference()); // eagerly validate aspects of tree config
  }
  return config;
}
 

private Expression parseExpression(String expr, Config config) {
  if (expr.equals("*")) {
    expr = "glob:*";
  }
  int i = expr.indexOf(':');
  if (i < 0) {
    throw new MorphlineCompilationException("Illegal match expression: " + expr, config);
  }
  String type = expr.substring(0, i);
  String pattern = expr.substring(i + 1, expr.length());
  if (type.equals("literal")) {
    return new LiteralExpression(pattern);
  } else if (type.equals("regex")) {
    if (pattern.equals(".*")) {
      return new MatchAllExpression(); // optimization
    }
    return new RegexExpression(Pattern.compile(pattern));
  } else if (type.equals("glob")) {
    if (pattern.equals("*")) {
      return new MatchAllExpression(); // optimization
    }
    return new GlobExpression(pattern);
  } else {
    throw new MorphlineCompilationException("Illegal match type: " + type, config);
  }
}
 

public SplitKeyValue(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.inputFieldName = getConfigs().getString(config, "inputField");      
  this.outputFieldPrefix = getConfigs().getString(config, "outputFieldPrefix", "");
  this.separator = getConfigs().getString(config, "separator", "=");
  if (separator.length() == 0) {
    throw new MorphlineCompilationException("separator must not be the empty string", config);
  }
  if (getConfigs().getBoolean(config, "isRegex", false)) {
    GrokDictionaries dict = new GrokDictionaries(config, getConfigs());
    this.regex = dict.compileExpression(separator).pattern().matcher("");
  } else {
    this.regex = null;
  }
  this.addEmptyStrings = getConfigs().getBoolean(config, "addEmptyStrings", false);      
  this.trim = getConfigs().getBoolean(config, "trim", true);
  validateArguments();
}
 

@Test (expected = MorphlineCompilationException.class)
public void morphlineCompilationError(
    final @Mocked Compiler compiler
) throws Exception {
  new Expectations() {{
    compiler.compile((File) any, anyString, (MorphlineContext) any, (Command) any); 
    result = new Exception("Compilation exception");
  }};

  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(MorphlineTranslator.ENCODING_KEY, "UTF-8");
  configMap.put(MorphlineTranslator.ENCODING_MSG, "UTF-8");
  configMap.put(MorphlineTranslator.MORPHLINE, getResourcePath(MORPHLINE_FILE));
  configMap.put(MorphlineTranslator.MORPHLINE_ID, "compiler-exception");
  configMap.put(MorphlineTranslator.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "flat");
  configMap.put(MorphlineTranslator.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG,
      Lists.newArrayList("bar"));
  configMap.put(MorphlineTranslator.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG,
      Lists.newArrayList("int"));
  Config config = ConfigFactory.parseMap(configMap);

  translator = new MorphlineTranslator();
  translator.configure(config);
  Row raw = TestingMessageFactory.get("The Key", DataTypes.StringType, "The Message", DataTypes.StringType);
  translator.translate(raw);
}
 

@Test (expected = MorphlineCompilationException.class)
public void invalidCommand() throws Exception {
  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(MorphlineTranslator.ENCODING_KEY, "UTF-8");
  configMap.put(MorphlineTranslator.ENCODING_MSG, "UTF-8");
  configMap.put(MorphlineTranslator.MORPHLINE, getResourcePath(MORPHLINE_FILE));
  configMap.put(MorphlineTranslator.MORPHLINE_ID, "invalid-command");
  configMap.put(MorphlineTranslator.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "flat");
  configMap.put(MorphlineTranslator.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG,
      Lists.newArrayList("int", "str", "float"));
  configMap.put(MorphlineTranslator.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG,
      Lists.newArrayList("int", "string", "float"));
  Config config = ConfigFactory.parseMap(configMap);

  translator.configure(config);
  Row raw = TestingMessageFactory.get("The Key", DataTypes.StringType,
      "The Message", DataTypes.StringType);
  translator.translate(raw);
}
 

@Test
public void testUnrecognizedPatternType() throws Exception {
  String str = "{" + 
    "\n metricFilter : {" +
    "\n includes : { " +
    "\n \"unRecognizedType:foo\" : \"glob:*\"" +
    "\n }" +
    "\n }}";

  Config config = ConfigFactory.parseString(str);
  try {
    PatternMetricFilter.parse(new Configs(), config);
    fail();
  } catch (MorphlineCompilationException e) {
    ; // expected
  }
}
 

@Test
public void testIllegalPatternType() throws Exception {
  String str = "{" + 
    "\n metricFilter : {" +
    "\n includes : { " +
    "\n \"ILLEGAL\" : \"glob:*\"" +
    "\n }" +
    "\n }}";

  Config config = ConfigFactory.parseString(str);
  try {
    PatternMetricFilter.parse(new Configs(), config);
    fail();
  } catch (MorphlineCompilationException e) {
    ; // expected
  }
}
 

/**
 * Validates that an enum of the given type with the given value exists, and that this enum is
 * contained in the given list of permitted choices; finally returns that enum object.
 */
public <T extends Enum<T>> T  validateEnum(Config config, String value, Class<T> type, T... choices) {
  if (choices.length == 0) {
    choices = type.getEnumConstants();
  }
  Preconditions.checkArgument(choices.length > 0);
  try {
    T result = Enum.valueOf(type, value);
    if (!Arrays.asList(choices).contains(result)) {
      throw new IllegalArgumentException();
    }
    return result;
  } catch (IllegalArgumentException e) {
    throw new MorphlineCompilationException(
      String.format("Invalid choice: '%s' (choose from {%s})", 
                    value, 
                    Joiner.on(",").join(choices)), 
      config);
  }
}
 

private void loadDictionary(Reader reader) throws IOException {
  for (String line : CharStreams.readLines(reader)) {
    line = line.trim();
    if (line.length() == 0) {
      continue; // ignore empty lines
    }
    if (line.startsWith("#")) {
      continue; // ignore comment lines
    }
    int i = line.indexOf(" ");
    if (i < 0) {
      throw new MorphlineCompilationException("Dictionary entry line must contain a space to separate name and value: " + line, getConfig());
    }
    if (i == 0) {
      throw new MorphlineCompilationException("Dictionary entry line must contain a name: " + line, getConfig());
    }
    String name = line.substring(0, i);
    String value = line.substring(i + 1, line.length()).trim();
    if (value.length() == 0) {
      throw new MorphlineCompilationException("Dictionary entry line must contain a value: " + line, getConfig());
    }
    dictionary.put(name, value);
  }      
}
 

public ReadAvro(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  String schemaString = getConfigs().getString(config, "writerSchemaString", null);
  if (schemaString != null) {
    this.writerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "writerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.writerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro writer schema file: " + schemaFile, config, e);
      }
    } else {
      this.writerSchema = null;
    }
  }
  
  this.isJson = getConfigs().getBoolean(config, "isJson", false);
  validateArguments();      
}
 

public IfThenElse(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  Command devNull = new DropRecordBuilder().build(null, this, null, context); // pipes into /dev/null
  List<Command> conditions = buildCommandChain(config, "conditions", devNull, true);
  if (conditions.size() == 0) {
    throw new MorphlineCompilationException("Missing conditions", config);
  } else {
    this.conditionChain = conditions.get(0);
  }

  List<Command> thenCommands = buildCommandChain(config, "then", child, true);
  if (thenCommands.size() > 0) {
    this.thenChain = thenCommands.get(0);
  }
  
  List<Command> elseCommands = buildCommandChain(config, "else", child, true);
  if (elseCommands.size() > 0) {
    this.elseChain = elseCommands.get(0);
  }
  validateArguments();
}
 

public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.outputFieldName = getConfigs().getString(config, "outputField");      
  String solrFieldType = getConfigs().getString(config, "solrFieldType");      
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  IndexSchema schema = locator.getIndexSchema();
  FieldType fieldType = schema.getFieldTypeByName(solrFieldType);
  if (fieldType == null) {
    throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config);
  }
  this.analyzer = fieldType.getIndexAnalyzer();
  Preconditions.checkNotNull(analyzer);
  // register CharTermAttribute for later (implicit) reuse
  this.token = analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class);
  Preconditions.checkNotNull(token);
  validateArguments();
}
 

public GeoIP(CommandBuilder builder, Config config, Command parent, 
                                   Command child, final MorphlineContext context) {
  
  super(builder, config, parent, child, context);      
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.databaseFile = new File(getConfigs().getString(config, "database", "GeoLite2-City.mmdb"));
  try {
    this.databaseReader = new Reader(databaseFile);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot read Maxmind database: " + databaseFile, config, e);
  }
  validateArguments();
}
 

private static SolrContentHandlerFactory getSolrContentHandlerFactory(
    Class<? extends SolrContentHandlerFactory> factoryClass, Collection<String> dateFormats, Config config) {
  try {
    return factoryClass.getConstructor(Collection.class).newInstance(dateFormats);
  } catch (NoSuchMethodException nsme) {
    throw new MorphlineCompilationException("Unable to find valid constructor of type "
      + factoryClass.getName() + " for creating SolrContentHandler", config, nsme);
  } catch (Exception e) {
    throw new MorphlineCompilationException("Unexpected exception when trying to create SolrContentHandlerFactory of type "
      + factoryClass.getName(), config, e);
  }
}
 

public SolrClient getSolrServer() {
  if (zkHost != null && zkHost.length() > 0) {
    if (collectionName == null || collectionName.length() == 0) {
      throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config);
    }
    CloudSolrClient cloudSolrClient = new Builder()
        .withZkHost(zkHost)
        .build();
    cloudSolrClient.setDefaultCollection(collectionName);
    cloudSolrClient.setZkClientTimeout(zkClientSessionTimeout); 
    cloudSolrClient.setZkConnectTimeout(zkClientConnectTimeout); 
    return cloudSolrClient;
  } else {
    if (solrUrl == null && solrHomeDir != null) {
      CoreContainer coreContainer = new CoreContainer(solrHomeDir);
      coreContainer.load();
      EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(coreContainer, collectionName);
      return embeddedSolrServer;
    }
    if (solrUrl == null || solrUrl.length() == 0) {
      throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config);
    }
    int solrServerNumThreads = 2;
    int solrServerQueueLength = solrServerNumThreads;
    SolrClient server = new SafeConcurrentUpdateSolrServer(solrUrl, solrServerQueueLength, solrServerNumThreads);
    return server;
  }
}
 

private void validateSchema(IndexSchema schema) {
  if (schema.getUniqueKeyField() == null) {
    throw new MorphlineCompilationException("Solr schema.xml is missing unique key field", config);
  }
  if (!schema.getUniqueKeyField().isRequired()) {
    throw new MorphlineCompilationException("Solr schema.xml must contain a required unique key field", config);
  }
}
 

public RCFileColumn(Config columnConfig, Configuration conf) {
  this.conf = conf;
  Configs configs = new Configs();
  this.inputField = configs.getInt(columnConfig, "inputField");
  if (inputField < 0) {
    throw new MorphlineCompilationException(
        "Invalid column inputField specified: " + inputField, columnConfig);
  }

  this.outputField = configs.getString(columnConfig, "outputField");
  String writableClassString = configs.getString(columnConfig, "writableClass");

  if (writableClassString == null || writableClassString.isEmpty()) {
    throw new MorphlineCompilationException(
        "No writableClass specified for column " + outputField, columnConfig);
  }
  try {
    Class clazz = Class.forName(writableClassString);
    if (!Writable.class.isAssignableFrom(clazz)) {
      throw new MorphlineCompilationException("writableClass provided "
          + writableClassString + " for column " + outputField
          + " does not implement " + Writable.class.getName(), columnConfig);
    }
    this.writableClass = clazz;
  } catch (ClassNotFoundException e) {
    throw new MorphlineCompilationException("Could not load class "
        + writableClassString + " definition", columnConfig, e);
  }
  configs.validateArguments(columnConfig);
}
 

public ExtractJsonPaths(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  ListMultimap<String, String> stepMultiMap = ArrayListMultimap.create();
  this.flatten = getConfigs().getBoolean(config, "flatten", true);
  Config paths = getConfigs().getConfig(config, "paths");
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(paths)) {
    String fieldName = entry.getKey();        
    String path = entry.getValue().toString().trim();
    if (path.contains("//")) {
      throw new MorphlineCompilationException("No support for descendant axis available yet", config);
    }
    if (path.startsWith("/")) {
      path = path.substring(1);
    }
    if (path.endsWith("/")) {
      path = path.substring(0, path.length() - 1);
    }
    path = path.trim();
    for (String step : path.split("/")) {
      step = step.trim();
      if (step.length() > ARRAY_TOKEN.length() && step.endsWith(ARRAY_TOKEN)) {
        step = step.substring(0,  step.length() - ARRAY_TOKEN.length());
        stepMultiMap.put(fieldName, normalize(step));
        stepMultiMap.put(fieldName, ARRAY_TOKEN);
      } else {
        stepMultiMap.put(fieldName, normalize(step));
      }
    }
  }
  this.stepMap = stepMultiMap.asMap();
  LOG.debug("stepMap: {}", stepMap);
  validateArguments();
}
 

public SaxonCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
    super(builder, config, parent, child, context);
    
    this.isTracing = getConfigs().getBoolean(config, "isTracing", false);
    boolean isLicensedSaxonEdition = getConfigs().getBoolean(config, "isLicensedSaxonEdition", false);
    this.processor = new Processor(isLicensedSaxonEdition);
    this.documentBuilder = processor.newDocumentBuilder();
    
    Config features = getConfigs().getConfig(config, "features", ConfigFactory.empty());
    for (Map.Entry<String, Object> entry : new Configs().getEntrySet(features)) {
      processor.setConfigurationProperty(entry.getKey(), entry.getValue());
    }
    
    for (String clazz : getConfigs().getStringList(config, "extensionFunctions", Collections.<String>emptyList())) {
      Object function;
      try {
        function = Class.forName(clazz).newInstance();
      } catch (Exception e) {
        throw new MorphlineCompilationException("Cannot instantiate extension function: " + clazz, config);
      }
      
      if (function instanceof ExtensionFunction) {
        processor.registerExtensionFunction((ExtensionFunction) function);              
//      }
//      else if (function instanceof ExtensionFunctionDefinition) {
//        processor.registerExtensionFunction((ExtensionFunctionDefinition) function);              
      } else {
        throw new MorphlineCompilationException("Extension function has wrong class: " + clazz, config);
      }
    }
  }
 

@Override
protected void validateArguments() {
  super.validateArguments();
  if (writerSchema == null) {
    throw new MorphlineCompilationException(
        "You must specify an external Avro writer schema because this is required to read containerless Avro", getConfig());
  }
}
 

public ExtractAvroPaths(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  ListMultimap<String, String> stepMultiMap = ArrayListMultimap.create();
  this.flatten = getConfigs().getBoolean(config, "flatten", true);
  Config paths = getConfigs().getConfig(config, "paths");
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(paths)) {
    String fieldName = entry.getKey();        
    String path = entry.getValue().toString().trim();
    if (path.contains("//")) {
      throw new MorphlineCompilationException("No support for descendant axis available yet", config);
    }
    if (path.startsWith("/")) {
      path = path.substring(1);
    }
    if (path.endsWith("/")) {
      path = path.substring(0, path.length() - 1);
    }
    path = path.trim();
    for (String step : path.split("/")) {
      step = step.trim();
      if (step.length() > ARRAY_TOKEN.length() && step.endsWith(ARRAY_TOKEN)) {
        step = step.substring(0,  step.length() - ARRAY_TOKEN.length());
        stepMultiMap.put(fieldName, normalize(step));
        stepMultiMap.put(fieldName, ARRAY_TOKEN);
      } else {
        stepMultiMap.put(fieldName, normalize(step));
      }
    }
  }
  this.stepMap = stepMultiMap.asMap();
  LOG.debug("stepMap: {}", stepMap);
  validateArguments();
}
 

public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {   
  super(builder, config, parent, child, context);

  String schemaString = getConfigs().getString(config, "readerSchemaString", null);
  if (schemaString != null) {
    this.readerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "readerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.readerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e);
      }
    } else {
      this.readerSchema = null;
    }
  }
  
  if (getClass() == ReadAvroContainer.class) {
    resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>(
        getConfigs().getInt(config, "schemaCacheCapacity", 100));
    
    validateArguments();
  } else {
    resolverCache = null;
  }
}
 

public ToAvroMap(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  InputStream in = null;
  try {
    in = getClass().getResourceAsStream("morphlineRecord.avsc");
    this.schema = new Schema.Parser().parse(in);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot parse morphlineRecord schema", config, e, builder);
  } finally {
    Closeables.closeQuietly(in);
  }
  
  validateArguments();
}
 

@Test
public void testReadAvroWithMissingExternalSchema() throws Exception {
  try {
    morphline = createMorphline("test-morphlines/readAvroWithMissingExternalSchema");
    fail();
  } catch (MorphlineCompilationException e) {
    assertTrue(e.getMessage().startsWith(
        "You must specify an external Avro writer schema because this is required to read containerless Avro"));
  }
}
 

@Override
public Command build(Config config, Command parent, Command child, MorphlineContext context) {
  try {
    return new DownloadHdfsFile(this, config, parent, child, context);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot compile", config, e);
  }
}
 

/**
 *
 * @param morphlineFile
 * @param morphlineId
 * @param collector
 * @param isProduction
 * @return
 */
public static Pipeline setPipeline(String morphlineFile, String morphlineId, Collector collector, boolean isProduction) {
  LOG.debug("Constructing Pipeline[{}#{}]", morphlineFile, morphlineId);

  // Set up the Morphline context and handler
  MorphlineContext context = new MorphlineContext.Builder()
      .setExceptionHandler(new FaultTolerance(isProduction, false))
      .build();

  // Compile the Morphline process
  Command morphline;
  try {
    morphline = new Compiler().compile(
        new File(morphlineFile),
        morphlineId,
        context,
        collector);
  } catch (Exception e) {
    throw new MorphlineCompilationException("Morphline compilation error", null, e);
  }

  // Create the pipeline wrapper
  Pipeline pipeline = new Pipeline(morphline, collector);

  // Ensure shutdown notification to Morphline commands esp in streaming environments
  JVMUtils.closeAtShutdown(pipeline);

  // Prep the pipeline
  Notifications.notifyBeginTransaction(pipeline.getMorphline());

  // Register the pipeline into the cache
  if (null == pipelineCache.get()) {
    pipelineCache.set(new HashMap<String, Pipeline>());
  }
  pipelineCache.get().put(morphlineFile + SEPARATOR + morphlineId, pipeline);

  LOG.trace("Pipeline[{}#{}] prepared", morphlineFile, morphlineId);
  return pipeline;
}
 

public ReadLine(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.charset = getConfigs().getCharset(config, "charset", null);
  this.ignoreFirstLine = getConfigs().getBoolean(config, "ignoreFirstLine", false);
  String cprefix = getConfigs().getString(config, "commentPrefix", "");
  if (cprefix.length() > 1) {
    throw new MorphlineCompilationException("commentPrefix must be at most one character long: " + cprefix, config);
  }
  this.commentPrefix = (cprefix.length() > 0 ? cprefix : null);
  validateArguments();
}
 

@Test (expected = RuntimeException.class)
public void deriveMorphlineMapperFunctionError(
    final @Mocked MorphlineUtils utils
) throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(MorphlineDeriver.STEP_NAME_CONFIG, "dep1");
  paramMap.put(MorphlineDeriver.MORPHLINE, "morphline");
  paramMap.put(MorphlineDeriver.MORPHLINE_ID, "id");
  paramMap.put(MorphlineDeriver.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME, "flat");
  paramMap.put(MorphlineDeriver.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG,
      Lists.newArrayList("bar"));
  paramMap.put(MorphlineDeriver.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG,
      Lists.newArrayList("int"));
  final Config config = ConfigFactory.parseMap(paramMap);

  new Expectations() {{
    MorphlineUtils.morphlineMapper(anyString, anyString, (StructType) any, true); result =
        new MorphlineCompilationException("Compile exception", config);
  }};

  Dataset<Row> dataFrame = Contexts.getSparkSession().createDataFrame(
      Lists.newArrayList(RowFactory.create(1)),
      DataTypes.createStructType(Lists.newArrayList(DataTypes.createStructField(
          "baz", DataTypes.IntegerType, false)))
  );

  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  dependencies.put("dep1", dataFrame);

  MorphlineDeriver deriver = new MorphlineDeriver();
  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  deriver.derive(dependencies);
}
 

/**
 * Validates that the given value is contained in the range [min, max]
 */
public void validateRange(Config config, T value, Comparable<T> min, Comparable<T>  max) {
  boolean isValid = min.compareTo(value) <= 0 && 0 <= max.compareTo(value);
  if (!isValid) {
    throw new MorphlineCompilationException(
      String.format("Invalid choice: '%s' (choose from {%s..%s})", 
                    value, 
                    min,
                    max), 
      config);
  }
}
 

public void validateArguments(Config config) {
  Set<String> recognizedArgs = getRecognizedArguments();
  for (String key : config.root().keySet()) {
    if (!recognizedArgs.contains(key)) {
      throw new MorphlineCompilationException("Unrecognized command argument: " + key + 
          ", recognized arguments: " + recognizedArgs, config);
    }
  }      
}