Java Code Examples for org.apache.hadoop.util.GenericOptionsParser#getConfiguration()

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    GenericOptionsParser optionparser = new GenericOptionsParser(conf, args);
    conf = optionparser.getConfiguration();

    Job job = Job.getInstance(conf, "leftjoin");
    job.setJarByClass(LeftJoin.class);
    FileInputFormat.addInputPaths(job, conf.get("input_dir"));
    Path out = new Path(conf.get("output_dir"));
    FileOutputFormat.setOutputPath(job, out);
    job.setNumReduceTasks(conf.getInt("reduce_num", 1));

    job.setMapperClass(LeftJoinMapper.class);
    job.setReducerClass(LeftJoinReduce.class);
    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    conf.set("mapred.textoutputformat.separator", ",");

    return (job.waitForCompletion(true) ? 0 : 1);
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = new Configuration();

	Config.bsp_mode = false;
	Config.spark_mode = false;
	Config.flink_mode = true;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new FlinkEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local", "-flink" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();
	
	Config.hadoop_mode = true;
	Config.testing = true;		
	Config.parse_args(args, conf);

	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = null;

	Config.bsp_mode = false;
	Config.spark_mode = false;
	Config.flink_mode = true;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new FlinkEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local", "-flink" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();

	Config.hadoop_mode = true;
	Config.testing = true;
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = null;

	Config.bsp_mode = false;
	Config.spark_mode = false;
	Config.flink_mode = false;
	Config.storm_mode = true;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new StormEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local", "-storm" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();

	Config.hadoop_mode = true;
	Config.testing = true;
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = new Configuration();

	Config.bsp_mode = false;
	Config.spark_mode = false;
	Config.flink_mode = false;
	Config.storm_mode = true;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new StormEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-dist", "-storm","-stream","5" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();
	
	Config.hadoop_mode = true;
	Config.testing = true;		
	Config.parse_args(args, conf);

	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException {
    // need to make a copy of the configuration because to make sure different temp dirs are used.
    GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args);
    Configuration newConf = opts.getConfiguration();
    args = opts.getRemainingArgs();
    Job job = Import.createSubmittableJob(newConf, args);
    job.waitForCompletion(false);
    return job.isSuccessful();
}
 

public static void main(String[] args)
    throws IOException, InterruptedException, ClassNotFoundException {

  // GenericOptionsParser is a utility to parse command line arguments generic to the Hadoop
  // framework. This example won't cover the specifics, but will recognize several standard
  // command line arguments, enabling applications to easily specify a namenode, a
  // ResourceManager, additional configuration resources etc.
  GenericOptionsParser parser = new GenericOptionsParser(args);
  args = parser.getRemainingArgs();

  // Make sure we have the right parameters.
  if (args.length != 3) {
    System.out.println(
        "Usage: hadoop jar bigquery_wordcount.jar [ProjectId] [QualifiedInputTableId] "
            + "[GcsOutputPath]\n"
            + "    ProjectId - Project under which to issue the BigQuery operations. Also "
            + "serves as the default project for table IDs which don't explicitly specify a "
            + "project for the table.\n"
            + "    QualifiedInputTableId - Input table ID of the form "
            + "(Optional ProjectId):[DatasetId].[TableId]\n"
            + "    OutputPath - The output path to write data, e.g. "
            + "gs://bucket/dir/");
    System.exit(1);
  }

  // Get the individual parameters from the command line.
  String projectId = args[0];
  String inputQualifiedTableId = args[1];
  String outputPath = args[2];

  // Create the job and get its configuration.
  Job job = new Job(parser.getConfiguration(), "wordcount");
  Configuration conf = job.getConfiguration();

  // Set the job-level projectId.
  conf.set(PROJECT_ID.getKey(), projectId);

  // Configure input and output.
  BigQueryConfiguration.configureBigQueryInput(conf, inputQualifiedTableId);

  // Set column and predicate filters
  conf.set(SELECTED_FIELDS.getKey(), "word,word_count");
  conf.set(SQL_FILTER.getKey(), "word >= 'A' AND word <= 'zzz'");
  conf.set(MRJobConfig.NUM_MAPS, "999");

  // This helps Hadoop identify the Jar which contains the mapper and reducer by specifying a
  // class in that Jar. This is required if the jar is being passed on the command line to Hadoop.
  job.setJarByClass(DirectBigQueryWordCount.class);

  // Tell the job what the output will be.
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(LongWritable.class);

  job.setMapperClass(Map.class);
  job.setReducerClass(Reduce.class);

  job.setInputFormatClass(DirectBigQueryInputFormat.class);
  job.setOutputFormatClass(TextOutputFormat.class);

  FileOutputFormat.setOutputPath(job, new Path(outputPath));

  job.waitForCompletion(true);
}
 

public static void main(String[] args)
    throws IOException, InterruptedException, ClassNotFoundException {
  GenericOptionsParser parser = new GenericOptionsParser(args);
  String[] customArgs = parser.getRemainingArgs();
  Configuration config = parser.getConfiguration();

  if (customArgs.length != 5) {
    System.out.println(
        "Usage: hadoop jar wikipedia_bytes_deploy.jar "
            + "[projectId] [inputDatasetId] [inputTableId] [exportGcsBucket] [jobOutputPath]");
    System.exit(1);
  }

  String projectId = customArgs[0];
  String inputDatasetId = customArgs[1];
  String inputTableId = customArgs[2];
  String exportGcsBucket = customArgs[3];
  String jobOutputPath = customArgs[4];

  JobConf conf = new JobConf(config, WikipediaRequestBytes.class);
  BigQueryConfiguration.configureBigQueryInput(conf, projectId, inputDatasetId, inputTableId);
  conf.set(BigQueryConfiguration.GCS_BUCKET.getKey(), exportGcsBucket);

  Job job = new Job(conf, "WikipediaRequestBytes");
  job.setJarByClass(WikipediaRequestBytes.class);

  job.setMapperClass(TitleBytesMapper.class);
  job.setCombinerClass(TitleBytesSumReducer.class);
  job.setReducerClass(TitleBytesSumReducer.class);

  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(LongWritable.class);
  FileOutputFormat.setOutputPath(job, new Path(jobOutputPath));

  // Read from BigQuery, write with plan TextOutputFormat to provided 'Path'.
  job.setInputFormatClass(GsonBigQueryInputFormat.class);
  job.setOutputFormatClass(TextOutputFormat.class);

  job.waitForCompletion(true);

  // Make sure to clean up the GCS export paths if desired, and possibly an intermediate input
  // table if we did sharded export and thus didn't clean it up at setup time.
  GsonBigQueryInputFormat.cleanupJob(job.getConfiguration(), job.getJobID());
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = null;

	Config.bsp_mode = false;
	Config.spark_mode = true;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new SparkEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local", "-spark" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();

	Config.hadoop_mode = true;
	Config.testing = true;
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = new Configuration();

	Config.bsp_mode = false;
	Config.spark_mode = true;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new SparkEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local", "-spark" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();
	
	Config.hadoop_mode = true;
	Config.testing = true;		
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);

	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = null;

	Config.bsp_mode = true;
	Config.spark_mode = false;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new BSPEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local", "-bsp" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();
	
	Config.hadoop_mode = true;
	Config.testing = true;
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = new Configuration();

	Config.bsp_mode = true;
	Config.spark_mode = false;
	Config.map_reduce_mode = false;

	Evaluator.evaluator = new BSPEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local", "-bsp" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();

	Config.hadoop_mode = true;
	Config.testing = true;
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = new Configuration();

	Config.bsp_mode = false;
	Config.spark_mode = false;
	Config.map_reduce_mode = true;

	Evaluator.evaluator = new MapReduceEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();

	Config.hadoop_mode = true;
	Config.testing = true;
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);
	
	return Evaluator.evaluator;
}
 

@Override
protected Evaluator createEvaluator() throws Exception {
	Configuration conf = null;

	Config.bsp_mode = false;
	Config.spark_mode = false;
	Config.map_reduce_mode = true;

	Evaluator.evaluator = new MapReduceEvaluator();

	Config.quiet_execution = true;

	String[] args = new String[] { "-local" };

	conf = Evaluator.evaluator.new_configuration();
	GenericOptionsParser gop = new GenericOptionsParser(conf, args);
	conf = gop.getConfiguration();

	args = gop.getRemainingArgs();

	Config.hadoop_mode = true;
	Config.testing = true;
	Config.parse_args(args, conf);
	
	Evaluator.evaluator.init(conf);

	return Evaluator.evaluator;
}
 

public static void main (String [] args) throws Exception {

		GenericOptionsParser parser = new GenericOptionsParser(args);

		String[] remArgs = parser.getRemainingArgs();
		String adjacencyFile = remArgs[0];
		String resultFile = remArgs[1];
		int numVertices = Integer.valueOf(remArgs[2]);
		int numIterations = Integer.valueOf(remArgs[3]);


		Configuration conf = parser.getConfiguration();
		RANDOM_JUMP = 1.0 / ((double) numVertices);
		conf.set("random_jump", String.valueOf(RANDOM_JUMP));
		conf.set("dampening_factor", String.valueOf(DAMPENING_FACTOR));

		FileSystem fs = FileSystem.get(conf);
		String adjacencySeq = adjacencyFile + "_seq";

		assignInitialRanks(conf, fs, adjacencyFile, adjacencySeq, numVertices);



		String inputFile = adjacencySeq;
		String outputFile = null;

		for (int iteration = 0; iteration < numIterations; iteration++) {
			outputFile = "/pageranks_iteration_" + iteration;
			calculateNextRanks(conf, fs, inputFile, outputFile);
			inputFile = outputFile;
		}




		printFinalRanks(conf, fs, outputFile, resultFile);
	}