Java Code Examples for org.apache.flink.core.fs.Path#toUri()

/**
 * Checks the validity of the path's scheme and path.
 *
 * @param path The path to check.
 * @return The URI as a Path.
 *
 * @throws IllegalArgumentException Thrown, if the URI misses scheme or path.
 */
private static Path validatePath(Path path) {
	final URI uri = path.toUri();
	final String scheme = uri.getScheme();
	final String pathPart = uri.getPath();

	// some validity checks
	if (scheme == null) {
		throw new IllegalArgumentException("The scheme (hdfs://, file://, etc) is null. " +
				"Please specify the file system scheme explicitly in the URI.");
	}
	if (pathPart == null) {
		throw new IllegalArgumentException("The path to store the checkpoint data in is null. " +
				"Please specify a directory path for the checkpoint data.");
	}
	if (pathPart.length() == 0 || pathPart.equals("/")) {
		throw new IllegalArgumentException("Cannot use the root directory for checkpoints.");
	}

	return path;
}
 

@Override
public String map(Integer value) throws Exception {
	final Path actualFile = new Path(getRuntimeContext().getDistributedCache().getFile("test_data").toURI());

	Path path = new Path(actualFile.toUri());
	assertFalse(path.getFileSystem().isDistributedFS());

	DataInputStream in = new DataInputStream(actualFile.getFileSystem().open(actualFile));
	String contents = in.readUTF();

	assertEquals(testFileContent, contents);

	final Path actualDir = new Path(getRuntimeContext().getDistributedCache().getFile("test_dir").toURI());
	FileStatus fileStatus = actualDir.getFileSystem().getFileStatus(actualDir);
	assertTrue(fileStatus.isDir());
	FileStatus[] fileStatuses = actualDir.getFileSystem().listStatus(actualDir);
	assertEquals(2, fileStatuses.length);

	return contents;
}
 

/**
 * Checks the validity of the path's scheme and path.
 *
 * @param path The path to check.
 * @return The URI as a Path.
 *
 * @throws IllegalArgumentException Thrown, if the URI misses scheme or path.
 */
private static Path validatePath(Path path) {
	final URI uri = path.toUri();
	final String scheme = uri.getScheme();
	final String pathPart = uri.getPath();

	// some validity checks
	if (scheme == null) {
		throw new IllegalArgumentException("The scheme (hdfs://, file://, etc) is null. " +
				"Please specify the file system scheme explicitly in the URI.");
	}
	if (pathPart == null) {
		throw new IllegalArgumentException("The path to store the checkpoint data in is null. " +
				"Please specify a directory path for the checkpoint data.");
	}
	if (pathPart.length() == 0 || pathPart.equals("/")) {
		throw new IllegalArgumentException("Cannot use the root directory for checkpoints.");
	}

	return path;
}
 

@Override
public String map(Integer value) throws Exception {
	final Path actualFile = new Path(getRuntimeContext().getDistributedCache().getFile("test_data").toURI());

	Path path = new Path(actualFile.toUri());
	assertFalse(path.getFileSystem().isDistributedFS());

	DataInputStream in = new DataInputStream(actualFile.getFileSystem().open(actualFile));
	String contents = in.readUTF();

	assertEquals(testFileContent, contents);

	final Path actualDir = new Path(getRuntimeContext().getDistributedCache().getFile("test_dir").toURI());
	FileStatus fileStatus = actualDir.getFileSystem().getFileStatus(actualDir);
	assertTrue(fileStatus.isDir());
	FileStatus[] fileStatuses = actualDir.getFileSystem().listStatus(actualDir);
	assertEquals(2, fileStatuses.length);

	return contents;
}
 

/**
 * Checks the validity of the path's scheme and path.
 *
 * @param path The path to check.
 * @return The URI as a Path.
 *
 * @throws IllegalArgumentException Thrown, if the URI misses scheme or path.
 */
private static Path validatePath(Path path) {
	final URI uri = path.toUri();
	final String scheme = uri.getScheme();
	final String pathPart = uri.getPath();

	// some validity checks
	if (scheme == null) {
		throw new IllegalArgumentException("The scheme (hdfs://, file://, etc) is null. " +
				"Please specify the file system scheme explicitly in the URI.");
	}
	if (pathPart == null) {
		throw new IllegalArgumentException("The path to store the checkpoint data in is null. " +
				"Please specify a directory path for the checkpoint data.");
	}
	if (pathPart.length() == 0 || pathPart.equals("/")) {
		throw new IllegalArgumentException("Cannot use the root directory for checkpoints.");
	}

	return path;
}
 

@Override
public String map(Integer value) throws Exception {
	final Path actualFile = new Path(getRuntimeContext().getDistributedCache().getFile("test_data").toURI());

	Path path = new Path(actualFile.toUri());
	assertFalse(path.getFileSystem().isDistributedFS());

	DataInputStream in = new DataInputStream(actualFile.getFileSystem().open(actualFile));
	String contents = in.readUTF();

	assertEquals(testFileContent, contents);

	final Path actualDir = new Path(getRuntimeContext().getDistributedCache().getFile("test_dir").toURI());
	FileStatus fileStatus = actualDir.getFileSystem().getFileStatus(actualDir);
	assertTrue(fileStatus.isDir());
	FileStatus[] fileStatuses = actualDir.getFileSystem().listStatus(actualDir);
	assertEquals(2, fileStatuses.length);

	return contents;
}
 

private static void ensureLocalFileDeleted(Path path) {
	URI uri = path.toUri();
	if ("file".equals(uri.getScheme())) {
		File file = new File(uri.getPath());
		assertFalse("file not properly deleted", file.exists());
	}
	else {
		throw new IllegalArgumentException("not a local path");
	}
}
 

private static void ensureLocalFileDeleted(Path path) {
	URI uri = path.toUri();
	if ("file".equals(uri.getScheme())) {
		File file = new File(uri.getPath());
		assertFalse("file not properly deleted", file.exists());
	}
	else {
		throw new IllegalArgumentException("not a local path");
	}
}
 

private static void ensureLocalFileDeleted(Path path) {
	URI uri = path.toUri();
	if ("file".equals(uri.getScheme())) {
		File file = new File(uri.getPath());
		assertFalse("file not properly deleted", file.exists());
	}
	else {
		throw new IllegalArgumentException("not a local path");
	}
}
 

public static org.apache.hadoop.fs.Path toHadoopPath(Path path) {
	return new org.apache.hadoop.fs.Path(path.toUri());
}
 

@Override
public void open(String uId) throws Exception {
	Path path = new Path(resultPath + "/" + uId);
	FileSystem.get(new URI("file:///")).create(path, false);
	internalWriter = new PrintWriter(new File(path.toUri()));
}
 

/**
 * Util for generating partitioned {@link ParquetColumnarRowSplitReader}.
 */
public static ParquetColumnarRowSplitReader genPartColumnarRowReader(
		boolean utcTimestamp,
		boolean caseSensitive,
		Configuration conf,
		String[] fullFieldNames,
		DataType[] fullFieldTypes,
		Map<String, Object> partitionSpec,
		int[] selectedFields,
		int batchSize,
		Path path,
		long splitStart,
		long splitLength) throws IOException {
	List<String> nonPartNames = Arrays.stream(fullFieldNames)
			.filter(n -> !partitionSpec.containsKey(n))
			.collect(Collectors.toList());

	List<String> selNonPartNames = Arrays.stream(selectedFields)
			.mapToObj(i -> fullFieldNames[i])
			.filter(nonPartNames::contains).collect(Collectors.toList());

	int[] selParquetFields = selNonPartNames.stream()
			.mapToInt(nonPartNames::indexOf)
			.toArray();

	ParquetColumnarRowSplitReader.ColumnBatchGenerator gen = readVectors -> {
		// create and initialize the row batch
		ColumnVector[] vectors = new ColumnVector[selectedFields.length];
		for (int i = 0; i < vectors.length; i++) {
			String name = fullFieldNames[selectedFields[i]];
			LogicalType type = fullFieldTypes[selectedFields[i]].getLogicalType();
			vectors[i] = partitionSpec.containsKey(name) ?
					createVectorFromConstant(type, partitionSpec.get(name), batchSize) :
					readVectors[selNonPartNames.indexOf(name)];
		}
		return new VectorizedColumnBatch(vectors);
	};

	return new ParquetColumnarRowSplitReader(
			utcTimestamp,
			caseSensitive,
			conf,
			Arrays.stream(selParquetFields)
					.mapToObj(i -> fullFieldTypes[i].getLogicalType())
					.toArray(LogicalType[]::new),
			selNonPartNames.toArray(new String[0]),
			gen,
			batchSize,
			new org.apache.hadoop.fs.Path(path.toUri()),
			splitStart,
			splitLength);
}
 

public static org.apache.hadoop.fs.Path toHadoopPath(Path path) {
	return new org.apache.hadoop.fs.Path(path.toUri());
}
 

public static org.apache.hadoop.fs.Path toHadoopPath(Path path) {
	return new org.apache.hadoop.fs.Path(path.toUri());
}
 

/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 */
public FsStateBackend(Path checkpointDataUri) {
	this(checkpointDataUri.toUri());
}
 

/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(Path checkpointDataUri, boolean asynchronousSnapshots) {
	this(checkpointDataUri.toUri(), asynchronousSnapshots);
}
 

/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(Path checkpointDataUri, boolean asynchronousSnapshots) {
	this(checkpointDataUri.toUri(), asynchronousSnapshots);
}
 

/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 */
public FsStateBackend(Path checkpointDataUri) {
	this(checkpointDataUri.toUri());
}
 

/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(Path checkpointDataUri, boolean asynchronousSnapshots) {
	this(checkpointDataUri.toUri(), asynchronousSnapshots);
}
 

/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 */
public FsStateBackend(Path checkpointDataUri) {
	this(checkpointDataUri.toUri());
}