org.apache.flink.api.java.hadoop.mapred.HadoopInputFormat Java Examples

The following examples show how to use org.apache.flink.api.java.hadoop.mapred.HadoopInputFormat. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HadoopInputs.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath, Job job) throws IOException {
	// set input path in Job
	org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapreduceInputFormat, key, value, job);
}
Example #2
Source File: HadoopMapredCompatWordCount.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
	if (args.length < 2) {
		System.err.println("Usage: WordCount <input path> <result path>");
		return;
	}

	final String inputPath = args[0];
	final String outputPath = args[1];

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// Set up the Hadoop Input Format
	HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
	TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));

	// Create a Flink job with it
	DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);

	DataSet<Tuple2<Text, LongWritable>> words =
			text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
				.groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));

	// Set up Hadoop Output Format
	HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
			new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
	hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
	TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));

	// Output & Execute
	words.output(hadoopOutputFormat).setParallelism(1);
	env.execute("Hadoop Compat WordCount");
}
Example #3
Source File: HadoopInputs.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath, Job job) throws IOException {
	// set input path in Job
	org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapreduceInputFormat, key, value, job);
}
Example #4
Source File: HadoopInputs.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath, JobConf job) {
	// set input path in JobConf
	org.apache.hadoop.mapred.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapredInputFormat, key, value, job);
}
Example #5
Source File: HadoopMapredCompatWordCount.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
	if (args.length < 2) {
		System.err.println("Usage: WordCount <input path> <result path>");
		return;
	}

	final String inputPath = args[0];
	final String outputPath = args[1];

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// Set up the Hadoop Input Format
	HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
	TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));

	// Create a Flink job with it
	DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);

	DataSet<Tuple2<Text, LongWritable>> words =
			text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
				.groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));

	// Set up Hadoop Output Format
	HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
			new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
	hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
	TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));

	// Output & Execute
	words.output(hadoopOutputFormat).setParallelism(1);
	env.execute("Hadoop Compat WordCount");
}
Example #6
Source File: HadoopInputs.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath, JobConf job) {
	// set input path in JobConf
	org.apache.hadoop.mapred.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapredInputFormat, key, value, job);
}
Example #7
Source File: HadoopInputs.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath, JobConf job) {
	// set input path in JobConf
	org.apache.hadoop.mapred.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapredInputFormat, key, value, job);
}
Example #8
Source File: HadoopMapredCompatWordCount.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
	if (args.length < 2) {
		System.err.println("Usage: WordCount <input path> <result path>");
		return;
	}

	final String inputPath = args[0];
	final String outputPath = args[1];

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// Set up the Hadoop Input Format
	HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
	TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));

	// Create a Flink job with it
	DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);

	DataSet<Tuple2<Text, LongWritable>> words =
			text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
				.groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));

	// Set up Hadoop Output Format
	HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
			new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
	hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
	TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));

	// Output & Execute
	words.output(hadoopOutputFormat).setParallelism(1);
	env.execute("Hadoop Compat WordCount");
}
Example #9
Source File: HadoopInputs.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath, Job job) throws IOException {
	// set input path in Job
	org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapreduceInputFormat, key, value, job);
}
Example #10
Source File: HadoopInputs.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath) {
	return readHadoopFile(mapredInputFormat, key, value, inputPath, new JobConf());
}
Example #11
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> createHadoopInput(
		org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
	return new org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<>(mapreduceInputFormat, key, value, job);
}
Example #12
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance());
}
Example #13
Source File: HadoopInputs.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} to read a Hadoop sequence file for the given key and value classes.
 *
 * @return A Flink InputFormat that wraps a Hadoop SequenceFileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readSequenceFile(Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(new org.apache.hadoop.mapred.SequenceFileInputFormat<K, V>(), key, value, inputPath);
}
Example #14
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> createHadoopInput(org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
	return new HadoopInputFormat<>(mapredInputFormat, key, value, job);
}
Example #15
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} to read a Hadoop sequence file for the given key and value classes.
 *
 * @return A Flink InputFormat that wraps a Hadoop SequenceFileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readSequenceFile(Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(new org.apache.hadoop.mapred.SequenceFileInputFormat<K, V>(), key, value, inputPath);
}
Example #16
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath) {
	return readHadoopFile(mapredInputFormat, key, value, inputPath, new JobConf());
}
Example #17
Source File: HadoopInputs.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> createHadoopInput(org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
	return new HadoopInputFormat<>(mapredInputFormat, key, value, job);
}
Example #18
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> createHadoopInput(
		org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
	return new org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<>(mapreduceInputFormat, key, value, job);
}
Example #19
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance());
}
Example #20
Source File: HadoopInputs.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance());
}
Example #21
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> createHadoopInput(org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
	return new HadoopInputFormat<>(mapredInputFormat, key, value, job);
}
Example #22
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} to read a Hadoop sequence file for the given key and value classes.
 *
 * @return A Flink InputFormat that wraps a Hadoop SequenceFileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readSequenceFile(Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(new org.apache.hadoop.mapred.SequenceFileInputFormat<K, V>(), key, value, inputPath);
}
Example #23
Source File: HadoopInputs.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath) {
	return readHadoopFile(mapredInputFormat, key, value, inputPath, new JobConf());
}
Example #24
Source File: HadoopInputs.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> createHadoopInput(
		org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
	return new org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<>(mapreduceInputFormat, key, value, job);
}