Java Code Examples for org.apache.flink.core.fs.FileSystem#initOutPathDistFS()

The following examples show how to use org.apache.flink.core.fs.FileSystem#initOutPathDistFS() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileOutputFormat.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
	final Path path = getOutputFilePath();
	final FileSystem fs = path.getFileSystem();
	
	// only distributed file systems can be initialized at start-up time.
	if (fs.isDistributedFS()) {
		
		final WriteMode writeMode = getWriteMode();
		final OutputDirectoryMode outDirMode = getOutputDirectoryMode();

		if (parallelism == 1 && outDirMode == OutputDirectoryMode.PARONLY) {
			// output is not written in parallel and should be written to a single file.
			// prepare distributed output path
			if(!fs.initOutPathDistFS(path, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path could not be initialized.");
			}

		} else {
			// output should be written to a directory

			// only distributed file systems can be initialized at start-up time.
			if(!fs.initOutPathDistFS(path, writeMode, true)) {
				throw new IOException("Output directory could not be created.");
			}
		}
	}
}
Example 2
Source File: FileOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
	final Path path = getOutputFilePath();
	final FileSystem fs = path.getFileSystem();
	
	// only distributed file systems can be initialized at start-up time.
	if (fs.isDistributedFS()) {
		
		final WriteMode writeMode = getWriteMode();
		final OutputDirectoryMode outDirMode = getOutputDirectoryMode();

		if (parallelism == 1 && outDirMode == OutputDirectoryMode.PARONLY) {
			// output is not written in parallel and should be written to a single file.
			// prepare distributed output path
			if(!fs.initOutPathDistFS(path, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path could not be initialized.");
			}

		} else {
			// output should be written to a directory

			// only distributed file systems can be initialized at start-up time.
			if(!fs.initOutPathDistFS(path, writeMode, true)) {
				throw new IOException("Output directory could not be created.");
			}
		}
	}
}
Example 3
Source File: RheemFileOutputFormat.java    From rheem with Apache License 2.0 5 votes vote down vote up 
/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
    try {
        final Path path = getOutputFilePath();
        final FileSystem fs = path.getFileSystem();

        // only distributed file systems can be initialized at start-up time.
        if (fs.isDistributedFS()) {

            final FileSystem.WriteMode writeMode = getWriteMode();
            final FileOutputFormat.OutputDirectoryMode outDirMode = getOutputDirectoryMode();

            if (parallelism == 1 && outDirMode == FileOutputFormat.OutputDirectoryMode.PARONLY) {
                // output is not written in parallel and should be written to a single file.
                // prepare distributed output path
                if (!fs.initOutPathDistFS(path, writeMode, false)) {
                    // output preparation failed! Cancel task.
                    throw new IOException("Output path could not be initialized.");
                }

            } else {
                // output should be written to a directory

                // only distributed file systems can be initialized at start-up time.
                if (!fs.initOutPathDistFS(path, writeMode, true)) {
                    throw new IOException("Output directory could not be created.");
                }
            }
        }
    }catch (Exception e){
        throw new RheemException(e);
    }
}
Example 4
Source File: FileOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Initialization of the distributed file system if it is used.
 *
 * @param parallelism The task parallelism.
 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
	final Path path = getOutputFilePath();
	final FileSystem fs = path.getFileSystem();
	
	// only distributed file systems can be initialized at start-up time.
	if (fs.isDistributedFS()) {
		
		final WriteMode writeMode = getWriteMode();
		final OutputDirectoryMode outDirMode = getOutputDirectoryMode();

		if (parallelism == 1 && outDirMode == OutputDirectoryMode.PARONLY) {
			// output is not written in parallel and should be written to a single file.
			// prepare distributed output path
			if(!fs.initOutPathDistFS(path, writeMode, false)) {
				// output preparation failed! Cancel task.
				throw new IOException("Output path could not be initialized.");
			}

		} else {
			// output should be written to a directory

			// only distributed file systems can be initialized at start-up time.
			if(!fs.initOutPathDistFS(path, writeMode, true)) {
				throw new IOException("Output directory could not be created.");
			}
		}
	}
}