Java Code Examples for org.apache.flink.api.java.DataSet#write()

The following examples show how to use org.apache.flink.api.java.DataSet#write() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AvroOutputFormatITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath)
		.fieldDelimiter("|")
		.types(String.class, Integer.class, String.class);

	//output the data with AvroOutputFormat for specific user type
	DataSet<User> specificUser = input.map(new ConvertToUser());
	AvroOutputFormat<User> avroOutputFormat = new AvroOutputFormat<>(User.class);
	avroOutputFormat.setCodec(Codec.SNAPPY); // FLINK-4771: use a codec
	avroOutputFormat.setSchema(User.SCHEMA$); //FLINK-3304: Ensure the OF is properly serializing the schema
	specificUser.write(avroOutputFormat, outputPath1);

	//output the data with AvroOutputFormat for reflect user type
	DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
	reflectiveUser.write(new AvroOutputFormat<>(ReflectiveUser.class), outputPath2);

	env.execute();
}
Example 2
Source File: AvroOutputFormatITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath)
		.fieldDelimiter("|")
		.types(String.class, Integer.class, String.class);

	//output the data with AvroOutputFormat for specific user type
	DataSet<User> specificUser = input.map(new ConvertToUser());
	AvroOutputFormat<User> avroOutputFormat = new AvroOutputFormat<>(User.class);
	avroOutputFormat.setCodec(Codec.SNAPPY); // FLINK-4771: use a codec
	avroOutputFormat.setSchema(User.SCHEMA$); //FLINK-3304: Ensure the OF is properly serializing the schema
	specificUser.write(avroOutputFormat, outputPath1);

	//output the data with AvroOutputFormat for reflect user type
	DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
	reflectiveUser.write(new AvroOutputFormat<>(ReflectiveUser.class), outputPath2);

	env.execute();
}
Example 3
Source File: AvroOutputFormatITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath)
		.fieldDelimiter("|")
		.types(String.class, Integer.class, String.class);

	//output the data with AvroOutputFormat for specific user type
	DataSet<User> specificUser = input.map(new ConvertToUser());
	AvroOutputFormat<User> avroOutputFormat = new AvroOutputFormat<>(User.class);
	avroOutputFormat.setCodec(Codec.SNAPPY); // FLINK-4771: use a codec
	avroOutputFormat.setSchema(User.SCHEMA$); //FLINK-3304: Ensure the OF is properly serializing the schema
	specificUser.write(avroOutputFormat, outputPath1);

	//output the data with AvroOutputFormat for reflect user type
	DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
	reflectiveUser.write(new AvroOutputFormat<>(ReflectiveUser.class), outputPath2);

	env.execute();
}
Example 4
Source File: DataSetSerializer.java    From toolbox with Apache License 2.0 5 votes vote down vote up 
public static <T> void serializeDataSet(DataSet<T> dataSet, String pathFile){
    DataSet<byte[]> tmp = dataSet.map(new RichMapFunction<T, byte[]>() {
        @Override
        public byte[] map(T value) throws Exception {
            return Serialization.serializeObject(value);
        }
    });
    tmp.write(new SerializedOutputFormat(), pathFile, FileSystem.WriteMode.OVERWRITE);
}
Example 5
Source File: Prepare.java    From flink-perf with Apache License 2.0 5 votes vote down vote up 
public static void main(final String[] args) throws Exception {
	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> text = env.readTextFile(args[0]);
	DataSet<AvroLineitem> avro = text.map(new AvroLineItemMapper());
	avro.write(new AvroOutputFormat<AvroLineitem>(AvroLineitem.class), args[1]);
	env.execute("Lineitem Text 2 Avro converter");
}