org.apache.flink.api.common.io.FileOutputFormat Java Examples

@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 

@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 

@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 

@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 

@Test
public void testNullAllow() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.setAllowNullValues(true);
		csvOutputFormat.open(0, 1);
		csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
	}
	finally {
		csvOutputFormat.close();
	}

	java.nio.file.Path p = Paths.get(path);
	Assert.assertTrue(Files.exists(p));
	List<String> lines = Files.readAllLines(Paths.get(path), StandardCharsets.UTF_8);
	Assert.assertEquals(1, lines.size());
	Assert.assertEquals("One,,8", lines.get(0));
}
 

@Test
public void testNullDisallowOnDefault() throws Exception {
	final CsvOutputFormat<Tuple3<String, String, Integer>> csvOutputFormat = new CsvOutputFormat<>(new Path(path));
	try {
		csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
		csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
		csvOutputFormat.open(0, 1);
		try {
			csvOutputFormat.writeRecord(new Tuple3<String, String, Integer>("One", null, 8));
			fail("should fail with an exception");
		} catch (RuntimeException e) {
			// expected
		}

	}
	finally {
		csvOutputFormat.close();
	}
}
 

public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 

public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 

public void registerFileOutputTask(
	Class<? extends FileOutputFormat<Record>> stubClass,
	String outPath,
	Configuration formatParams) {

	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath, formatParams);
}
 

@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

@Before
public void setUp() throws Exception {
	path = tmpFolder.newFile().getAbsolutePath();
	csvOutputFormat = new ScalaCsvOutputFormat<>(new Path(path));
	csvOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	csvOutputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.PARONLY);
	csvOutputFormat.open(0, 1);
}
 

/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
    try {
        final Path path = getOutputFilePath();
        final FileSystem fs = path.getFileSystem();

        // only distributed file systems can be initialized at start-up time.
        if (fs.isDistributedFS()) {

            final FileSystem.WriteMode writeMode = getWriteMode();
            final FileOutputFormat.OutputDirectoryMode outDirMode = getOutputDirectoryMode();

            if (parallelism == 1 && outDirMode == FileOutputFormat.OutputDirectoryMode.PARONLY) {
                // output is not written in parallel and should be written to a single file.
                // prepare distributed output path
                if (!fs.initOutPathDistFS(path, writeMode, false)) {
                    // output preparation failed! Cancel task.
                    throw new IOException("Output path could not be initialized.");
                }

            } else {
                // output should be written to a directory

                // only distributed file systems can be initialized at start-up time.
                if (!fs.initOutPathDistFS(path, writeMode, true)) {
                    throw new IOException("Output directory could not be created.");
                }
            }
        }
    }catch (Exception e){
        throw new RheemException(e);
    }
}
 

public void setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode mode) {
    if (mode == null) {
        throw new NullPointerException();
    }

    this.outputDirectoryMode = mode;
}
 

protected Path getAcutalFilePath() {
	try {
		Field field = FileOutputFormat.class.getDeclaredField("actualFilePath");
		field.setAccessible(true);
		return (Path) field.get(this);
	} catch (NoSuchFieldException | IllegalAccessException e) {
		throw new RuntimeException("Get actual file path failed!", e);
	}
}
 

public void registerFileOutputTask(
	Class<? extends FileOutputFormat<Record>> stubClass,
	String outPath,
	Configuration formatParams) {

	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath, formatParams);
}
 

@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

public void registerFileOutputTask(FileOutputFormat<Record> outputFormat, String outPath) {
	TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	dsConfig.setStubWrapper(new UserCodeObjectWrapper<>(outputFormat));
}
 

@Test
public void testChangingFileNames() {
	org.apache.hadoop.fs.Path hdfsPath = new org.apache.hadoop.fs.Path(hdfsURI + "/hdfsTest");
	Path path = new Path(hdfsPath.toString());

	String type = "one";
	TextOutputFormat<String> outputFormat = new TextOutputFormat<>(path);

	outputFormat.setWriteMode(FileSystem.WriteMode.NO_OVERWRITE);
	outputFormat.setOutputDirectoryMode(FileOutputFormat.OutputDirectoryMode.ALWAYS);

	try {
		outputFormat.open(0, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		outputFormat.open(1, 2);
		outputFormat.writeRecord(type);
		outputFormat.close();

		assertTrue("No result file present", hdfs.exists(hdfsPath));
		FileStatus[] files = hdfs.listStatus(hdfsPath);
		Assert.assertEquals(2, files.length);
		for (FileStatus file : files) {
			assertTrue("1".equals(file.getPath().getName()) || "2".equals(file.getPath().getName()));
		}

	} catch (IOException e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 

public FileOutputFormat.OutputDirectoryMode getOutputDirectoryMode() {
    return this.outputDirectoryMode;
}
 

public void registerFileOutputTask(Class<? extends FileOutputFormat<Record>> stubClass, String outPath) {
	registerFileOutputTask(InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath);
}
 

private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 

private void initializeIOFormatClasses(Configuration configuration) {
	// TODO: That we still have to call something like this is a crime against humanity
	FileOutputFormat.initDefaultsFromConfiguration(configuration);
}
 

/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 

/**
 * Initialize defaults for output format. Needs to be a static method because it is configured for local
 * cluster execution, see LocalFlinkMiniCluster.
 * @param configuration The configuration to load defaults from
 */
public static void initDefaultsFromConfiguration(Configuration configuration) {

    DEFAULT_WRITE_MODE = FileSystem.WriteMode.OVERWRITE ;


    DEFAULT_OUTPUT_DIRECTORY_MODE =  FileOutputFormat.OutputDirectoryMode.PARONLY;
}
 

/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}
 

/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 

/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	return output(outputFormat);
}
 

/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}
 

/**
 * Writes a DataSet using a {@link FileOutputFormat} to a specified location.
 * This method adds a data sink to the program.
 *
 * @param outputFormat The FileOutputFormat to write the DataSet.
 * @param filePath The path to the location where the DataSet is written.
 * @param writeMode The mode of writing, indicating whether to overwrite existing files.
 * @return The DataSink that writes the DataSet.
 *
 * @see FileOutputFormat
 */
public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
	Preconditions.checkNotNull(filePath, "File path must not be null.");
	Preconditions.checkNotNull(writeMode, "Write mode must not be null.");
	Preconditions.checkNotNull(outputFormat, "Output format must not be null.");

	outputFormat.setOutputFilePath(new Path(filePath));
	outputFormat.setWriteMode(writeMode);
	return output(outputFormat);
}