Java Code Examples for org.apache.hadoop.util.GenericOptionsParser#printGenericCommandUsage()

void printUsage() {
  // The CLI package should do this for us, but I can't figure out how
  // to make it print something reasonable.
  System.out.println("bin/hadoop pipes");
  System.out.println("  [-input <path>] // Input directory");
  System.out.println("  [-output <path>] // Output directory");
  System.out.println("  [-jar <jar file> // jar filename");
  System.out.println("  [-inputformat <class>] // InputFormat class");
  System.out.println("  [-map <class>] // Java Map class");
  System.out.println("  [-partitioner <class>] // Java Partitioner");
  System.out.println("  [-reduce <class>] // Java Reduce class");
  System.out.println("  [-writer <class>] // Java RecordWriter");
  System.out.println("  [-program <executable>] // executable URI");
  System.out.println("  [-reduces <num>] // number of reduces");
  System.out.println("  [-lazyOutput <true/false>] // createOutputLazily");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);
}
 

void printUsage() {
  // The CLI package should do this for us, but I can't figure out how
  // to make it print something reasonable.
  System.out.println("bin/hadoop pipes");
  System.out.println("  [-input <path>] // Input directory");
  System.out.println("  [-output <path>] // Output directory");
  System.out.println("  [-jar <jar file> // jar filename");
  System.out.println("  [-inputformat <class>] // InputFormat class");
  System.out.println("  [-map <class>] // Java Map class");
  System.out.println("  [-partitioner <class>] // Java Partitioner");
  System.out.println("  [-reduce <class>] // Java Reduce class");
  System.out.println("  [-writer <class>] // Java RecordWriter");
  System.out.println("  [-program <executable>] // executable URI");
  System.out.println("  [-reduces <num>] // number of reduces");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);
}
 

void printUsage() {
  // The CLI package should do this for us, but I can't figure out how
  // to make it print something reasonable.
  System.out.println("bin/hadoop pipes");
  System.out.println("  [-input <path>] // Input directory");
  System.out.println("  [-output <path>] // Output directory");
  System.out.println("  [-jar <jar file> // jar filename");
  System.out.println("  [-inputformat <class>] // InputFormat class");
  System.out.println("  [-map <class>] // Java Map class");
  System.out.println("  [-partitioner <class>] // Java Partitioner");
  System.out.println("  [-reduce <class>] // Java Reduce class");
  System.out.println("  [-writer <class>] // Java RecordWriter");
  System.out.println("  [-program <executable>] // executable URI");
  System.out.println("  [-reduces <num>] // number of reduces");
  System.out.println("  [-lazyOutput <true/false>] // createOutputLazily");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);
}
 

void printUsage() {
  // The CLI package should do this for us, but I can't figure out how
  // to make it print something reasonable.
  System.out.println("bin/hadoop pipes");
  System.out.println("  [-input <path>] // Input directory");
  System.out.println("  [-output <path>] // Output directory");
  System.out.println("  [-jar <jar file> // jar filename");
  System.out.println("  [-inputformat <class>] // InputFormat class");
  System.out.println("  [-map <class>] // Java Map class");
  System.out.println("  [-partitioner <class>] // Java Partitioner");
  System.out.println("  [-reduce <class>] // Java Reduce class");
  System.out.println("  [-writer <class>] // Java RecordWriter");
  System.out.println("  [-program <executable>] // executable URI");
  System.out.println("  [-reduces <num>] // number of reduces");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);
}
 

protected static int printUsage() {
  System.err.println(
  "Usage: [-m <maps>] [-r <reduces>]\n" +
  "       [-keepmap <percent>] [-keepred <percent>]\n" +
  "       [-indir <path>] [-outdir <path]\n" +
  "       [-inFormat[Indirect] <InputFormat>] [-outFormat <OutputFormat>]\n" +
  "       [-outKey <WritableComparable>] [-outValue <Writable>]\n");
  GenericOptionsParser.printGenericCommandUsage(System.err);
  return -1;
}
 

protected static int printUsage() {
  System.err.println(
  "Usage: [-m <maps>] [-r <reduces>]\n" +
  "       [-keepmap <percent>] [-keepred <percent>]\n" +
  "       [-indir <path>] [-outdir <path]\n" +
  "       [-inFormat[Indirect] <InputFormat>] [-outFormat <OutputFormat>]\n" +
  "       [-outKey <WritableComparable>] [-outValue <Writable>]\n");
  GenericOptionsParser.printGenericCommandUsage(System.err);
  return -1;
}
 

public int run(String[] args) throws Exception {

        if (args.length != 2) {
            System.err.println("Usage: hadoop jar crunch-1.0.0-SNAPSHOT-job.jar" + " [generic options] input output");
            System.err.println();
            GenericOptionsParser.printGenericCommandUsage(System.err);
            return 1;
        }

        String inputPath = args[0];
        String outputPath = args[1];

        // Create an object to coordinate pipeline creation and execution.
        Pipeline pipeline = new MRPipeline(WordCount.class, getConf());

        // Reference a given text file as a collection of Strings.
        PCollection<String> lines = pipeline.readTextFile(inputPath);

        // Define a function that splits each line in a PCollection of Strings into
        // a PCollection made up of the individual words in the file.
        // The second argument sets the serialization format.
        PCollection<String> words = lines.parallelDo(new Tokenizer(), Writables.strings());

        // Take the collection of words and remove known stop words.
        PCollection<String> noStopWords = words.filter(new StopWordFilter());

        // The count method applies a series of Crunch primitives and returns
        // a map of the unique words in the input PCollection to their counts.
        PTable<String, Long> counts = noStopWords.count();

        // Instruct the pipeline to write the resulting counts to a text file.
        pipeline.writeTextFile(counts, outputPath);

        // Execute the pipeline as a MapReduce.
        PipelineResult result = pipeline.done();

        return result.succeeded() ? 0 : 1;
    }
 

protected static int printUsage() {
  System.err.println(
  "Usage: [-m <maps>] [-r <reduces>]\n" +
  "       [-keepmap <percent>] [-keepred <percent>]\n" +
  "       [-indir <path>] [-outdir <path]\n" +
  "       [-inFormat[Indirect] <InputFormat>] [-outFormat <OutputFormat>]\n" +
  "       [-outKey <WritableComparable>] [-outValue <Writable>]\n");
  GenericOptionsParser.printGenericCommandUsage(System.err);
  return -1;
}
 

private static void printUsage(PrintStream err) {
  err.println("fetchdt retrieves delegation tokens from the NameNode");
  err.println();
  err.println("fetchdt <opts> <token file>");
  err.println("Options:");
  err.println("  --webservice <url>  Url to contact NN on");
  err.println("  --renewer <name>    Name of the delegation token renewer");
  err.println("  --cancel            Cancel the delegation token");
  err.println("  --renew             Renew the delegation token.  Delegation " 
  		+ "token must have been fetched using the --renewer <name> option.");
  err.println("  --print             Print the delegation token");
  err.println();
  GenericOptionsParser.printGenericCommandUsage(err);
  ExitUtil.terminate(1);    
}
 

protected static int printUsage() {
  System.err.println(
  "Usage: [-m <maps>] [-r <reduces>]\n" +
  "       [-keepmap <percent>] [-keepred <percent>]\n" +
  "       [-indir <path>] [-outdir <path]\n" +
  "       [-inFormat[Indirect] <InputFormat>] [-outFormat <OutputFormat>]\n" +
  "       [-outKey <WritableComparable>] [-outValue <Writable>]\n");
  GenericOptionsParser.printGenericCommandUsage(System.err);
  return -1;
}
 

protected static int printUsage() {
  System.err.println(
  "Usage: [-m <maps>] [-r <reduces>]\n" +
  "       [-keepmap <percent>] [-keepred <percent>]\n" +
  "       [-indir <path>] [-outdir <path]\n" +
  "       [-inFormat[Indirect] <InputFormat>] [-outFormat <OutputFormat>]\n" +
  "       [-outKey <WritableComparable>] [-outValue <Writable>]\n");
  GenericOptionsParser.printGenericCommandUsage(System.err);
  return -1;
}
 

private static void printUsage(PrintStream err) {
  err.println("fetchdt retrieves delegation tokens from the NameNode");
  err.println();
  err.println("fetchdt <opts> <token file>");
  err.println("Options:");
  err.println("  --webservice <url>  Url to contact NN on");
  err.println("  --renewer <name>    Name of the delegation token renewer");
  err.println("  --cancel            Cancel the delegation token");
  err.println("  --renew             Renew the delegation token.  Delegation " 
  		+ "token must have been fetched using the --renewer <name> option.");
  err.println("  --print             Print the delegation token");
  err.println();
  GenericOptionsParser.printGenericCommandUsage(err);
  ExitUtil.terminate(1);    
}
 

protected static int printUsage() {
  System.err.println(
  "Usage: [-m <maps>] [-r <reduces>]\n" +
  "       [-keepmap <percent>] [-keepred <percent>]\n" +
  "       [-indir <path>] [-outdir <path]\n" +
  "       [-inFormat[Indirect] <InputFormat>] [-outFormat <OutputFormat>]\n" +
  "       [-outKey <WritableComparable>] [-outValue <Writable>]\n");
  GenericOptionsParser.printGenericCommandUsage(System.err);
  return -1;
}
 

public void exitUsage(boolean detailed) {
  //         1         2         3         4         5         6         7
  //1234567890123456789012345678901234567890123456789012345678901234567890123456789
  
  System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar \\");
  System.out.println("          $HADOOP_HOME/hadoop-streaming.jar [options]");
  System.out.println("Options:");
  System.out.println("  -input    <path>     DFS input file(s) for the Map step");
  System.out.println("  -output   <path>     DFS output directory for the Reduce step");
  System.out.println("  -mapper   <cmd|JavaClassName>      The streaming command to run");
  System.out.println("  -combiner <cmd|JavaClassName>" + 
                     " The streaming command to run");
  System.out.println("  -reducer  <cmd|JavaClassName>      The streaming command to run");
  System.out.println("  -file     <file>     File/dir to be shipped in the Job jar file");
  System.out.println("  -inputformat TextInputFormat(default)|SequenceFileAsTextInputFormat|JavaClassName Optional.");
  System.out.println("  -outputformat TextOutputFormat(default)|JavaClassName  Optional.");
  System.out.println("  -partitioner JavaClassName  Optional.");
  System.out.println("  -numReduceTasks <num>  Optional.");
  System.out.println("  -inputreader <spec>  Optional.");
  System.out.println("  -cmdenv   <n>=<v>    Optional. Pass env.var to streaming commands");
  System.out.println("  -mapdebug <path>  Optional. " +
  "To run this script when a map task fails ");
  System.out.println("  -reducedebug <path>  Optional." +
  " To run this script when a reduce task fails ");
  System.out.println("  -verbose");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);

  if (!detailed) {
    System.out.println();      
    System.out.println("For more details about these options:");
    System.out.println("Use $HADOOP_HOME/bin/hadoop jar build/hadoop-streaming.jar -info");
    fail("");
  }
  System.out.println();
  System.out.println("In -input: globbing on <path> is supported and can have multiple -input");
  System.out.println("Default Map input format: a line is a record in UTF-8");
  System.out.println("  the key part ends at first TAB, the rest of the line is the value");
  System.out.println("Custom input format: -inputformat package.MyInputFormat ");
  System.out.println("Map output format, reduce input/output format:");
  System.out.println("  Format defined by what the mapper command outputs. Line-oriented");
  System.out.println();
  System.out.println("The files named in the -file argument[s] end up in the");
  System.out.println("  working directory when the mapper and reducer are run.");
  System.out.println("  The location of this working directory is unspecified.");
  System.out.println();
  System.out.println("To set the number of reduce tasks (num. of output files):");
  System.out.println("  -D mapred.reduce.tasks=10");
  System.out.println("To skip the sort/combine/shuffle/sort/reduce step:");
  System.out.println("  Use -numReduceTasks 0");
  System.out
    .println("  A Task's Map output then becomes a 'side-effect output' rather than a reduce input");
  System.out
    .println("  This speeds up processing, This also feels more like \"in-place\" processing");
  System.out.println("  because the input filename and the map input order are preserved");
  System.out.println("  This equivalent -reducer NONE");
  System.out.println();
  System.out.println("To speed up the last maps:");
  System.out.println("  -D mapred.map.tasks.speculative.execution=true");
  System.out.println("To speed up the last reduces:");
  System.out.println("  -D mapred.reduce.tasks.speculative.execution=true");
  System.out.println("To name the job (appears in the JobTracker Web UI):");
  System.out.println("  -D mapred.job.name='My Job' ");
  System.out.println("To change the local temp directory:");
  System.out.println("  -D dfs.data.dir=/tmp/dfs");
  System.out.println("  -D stream.tmpdir=/tmp/streaming");
  System.out.println("Additional local temp directories with -cluster local:");
  System.out.println("  -D mapred.local.dir=/tmp/local");
  System.out.println("  -D mapred.system.dir=/tmp/system");
  System.out.println("  -D mapred.temp.dir=/tmp/temp");
  System.out.println("To treat tasks with non-zero exit status as SUCCEDED:");    
  System.out.println("  -D stream.non.zero.exit.is.failure=false");
  System.out.println("Use a custom hadoopStreaming build along a standard hadoop install:");
  System.out.println("  $HADOOP_HOME/bin/hadoop jar /path/my-hadoop-streaming.jar [...]\\");
  System.out
    .println("    [...] -D stream.shipped.hadoopstreaming=/path/my-hadoop-streaming.jar");
  System.out.println("For more details about jobconf parameters see:");
  System.out.println("  http://wiki.apache.org/hadoop/JobConfFile");
  System.out.println("To set an environement variable in a streaming command:");
  System.out.println("   -cmdenv EXAMPLE_DIR=/home/example/dictionaries/");
  System.out.println();
  System.out.println("Shortcut:");
  System.out
    .println("   setenv HSTREAMING \"$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar\"");
  System.out.println();
  System.out.println("Example: $HSTREAMING -mapper \"/usr/local/bin/perl5 filter.pl\"");
  System.out.println("           -file /local/filter.pl -input \"/logs/0604*/*\" [...]");
  System.out.println("  Ships a script, invokes the non-shipped perl interpreter");
  System.out.println("  Shipped files go to the working directory so filter.pl is found by perl");
  System.out.println("  Input files are all the daily logs for days in month 2006-04");
  fail("");
}
 

static void printUsage() {
  System.err.println(
      "Usage: " + " FaultToleranceTestRunner [generic options] <dag-class-name> <test-conf-path>");
  GenericOptionsParser.printGenericCommandUsage(System.err);
}
 

static void printUsage() {
  System.err.println(
      "Usage: " + " FaultToleranceTestRunner [generic options] <dag-class-name> <test-conf-path>");
  GenericOptionsParser.printGenericCommandUsage(System.err);
}
 

public void exitUsage(boolean detailed) {
  //         1         2         3         4         5         6         7
  //1234567890123456789012345678901234567890123456789012345678901234567890123456789
  
  System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar \\");
  System.out.println("          $HADOOP_HOME/hadoop-streaming.jar [options]");
  System.out.println("Options:");
  System.out.println("  -input    <path>     DFS input file(s) for the Map step");
  System.out.println("  -output   <path>     DFS output directory for the Reduce step");
  System.out.println("  -mapper   <cmd|JavaClassName>      The streaming command to run");
  System.out.println("  -combiner <JavaClassName> Combiner has to be a Java class");
  System.out.println("  -reducer  <cmd|JavaClassName>      The streaming command to run");
  System.out.println("  -file     <file>     File/dir to be shipped in the Job jar file");
  System.out.println("  -inputformat TextInputFormat(default)|SequenceFileAsTextInputFormat|JavaClassName Optional.");
  System.out.println("  -outputformat TextOutputFormat(default)|JavaClassName  Optional.");
  System.out.println("  -partitioner JavaClassName  Optional.");
  System.out.println("  -numReduceTasks <num>  Optional.");
  System.out.println("  -inputreader <spec>  Optional.");
  System.out.println("  -cmdenv   <n>=<v>    Optional. Pass env.var to streaming commands");
  System.out.println("  -mapdebug <path>  Optional. " +
  "To run this script when a map task fails ");
  System.out.println("  -reducedebug <path>  Optional." +
  " To run this script when a reduce task fails ");
  System.out.println("  -verbose");
  System.out.println();
  GenericOptionsParser.printGenericCommandUsage(System.out);

  if (!detailed) {
    System.out.println();      
    System.out.println("For more details about these options:");
    System.out.println("Use $HADOOP_HOME/bin/hadoop jar build/hadoop-streaming.jar -info");
    fail("");
  }
  System.out.println();
  System.out.println("In -input: globbing on <path> is supported and can have multiple -input");
  System.out.println("Default Map input format: a line is a record in UTF-8");
  System.out.println("  the key part ends at first TAB, the rest of the line is the value");
  System.out.println("Custom input format: -inputformat package.MyInputFormat ");
  System.out.println("Map output format, reduce input/output format:");
  System.out.println("  Format defined by what the mapper command outputs. Line-oriented");
  System.out.println();
  System.out.println("The files named in the -file argument[s] end up in the");
  System.out.println("  working directory when the mapper and reducer are run.");
  System.out.println("  The location of this working directory is unspecified.");
  System.out.println();
  System.out.println("To set the number of reduce tasks (num. of output files):");
  System.out.println("  -D mapred.reduce.tasks=10");
  System.out.println("To skip the sort/combine/shuffle/sort/reduce step:");
  System.out.println("  Use -numReduceTasks 0");
  System.out
    .println("  A Task's Map output then becomes a 'side-effect output' rather than a reduce input");
  System.out
    .println("  This speeds up processing, This also feels more like \"in-place\" processing");
  System.out.println("  because the input filename and the map input order are preserved");
  System.out.println("  This equivalent -reducer NONE");
  System.out.println();
  System.out.println("To speed up the last maps:");
  System.out.println("  -D mapred.map.tasks.speculative.execution=true");
  System.out.println("To speed up the last reduces:");
  System.out.println("  -D mapred.reduce.tasks.speculative.execution=true");
  System.out.println("To name the job (appears in the JobTracker Web UI):");
  System.out.println("  -D mapred.job.name='My Job' ");
  System.out.println("To change the local temp directory:");
  System.out.println("  -D dfs.data.dir=/tmp/dfs");
  System.out.println("  -D stream.tmpdir=/tmp/streaming");
  System.out.println("Additional local temp directories with -cluster local:");
  System.out.println("  -D mapred.local.dir=/tmp/local");
  System.out.println("  -D mapred.system.dir=/tmp/system");
  System.out.println("  -D mapred.temp.dir=/tmp/temp");
  System.out.println("To treat tasks with non-zero exit status as SUCCEDED:");    
  System.out.println("  -D stream.non.zero.exit.is.failure=false");
  System.out.println("Use a custom hadoopStreaming build along a standard hadoop install:");
  System.out.println("  $HADOOP_HOME/bin/hadoop jar /path/my-hadoop-streaming.jar [...]\\");
  System.out
    .println("    [...] -D stream.shipped.hadoopstreaming=/path/my-hadoop-streaming.jar");
  System.out.println("For more details about jobconf parameters see:");
  System.out.println("  http://wiki.apache.org/hadoop/JobConfFile");
  System.out.println("To set an environement variable in a streaming command:");
  System.out.println("   -cmdenv EXAMPLE_DIR=/home/example/dictionaries/");
  System.out.println();
  System.out.println("Shortcut:");
  System.out
    .println("   setenv HSTREAMING \"$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar\"");
  System.out.println();
  System.out.println("Example: $HSTREAMING -mapper \"/usr/local/bin/perl5 filter.pl\"");
  System.out.println("           -file /local/filter.pl -input \"/logs/0604*/*\" [...]");
  System.out.println("  Ships a script, invokes the non-shipped perl interpreter");
  System.out.println("  Shipped files go to the working directory so filter.pl is found by perl");
  System.out.println("  Input files are all the daily logs for days in month 2006-04");
  fail("");
}