org.apache.flink.core.fs.FileSystemKind Java Examples

/**
 * Gets the kind of the file system from its scheme.
 *
 * <p>Implementation note: Initially, especially within the Flink 1.3.x line
 * (in order to not break backwards compatibility), we must only label file systems
 * as 'inconsistent' or as 'not proper filesystems' if we are sure about it.
 * Otherwise, we cause regression for example in the performance and cleanup handling
 * of checkpoints.
 * For that reason, we initially mark some filesystems as 'eventually consistent' or
 * as 'object stores', and leave the others as 'consistent file systems'.
 */
static FileSystemKind getKindForScheme(String scheme) {
	scheme = scheme.toLowerCase(Locale.US);

	if (scheme.startsWith("s3") || scheme.startsWith("emr") || scheme.startsWith("oss") || scheme.startsWith("wasb")) {
		// the Amazon S3 storage or Aliyun OSS storage or Azure Blob Storage
		return FileSystemKind.OBJECT_STORE;
	}
	else if (scheme.startsWith("http") || scheme.startsWith("ftp")) {
		// file servers instead of file systems
		// they might actually be consistent, but we have no hard guarantees
		// currently to rely on that
		return FileSystemKind.OBJECT_STORE;
	}
	else {
		// the remainder should include hdfs, kosmos, ceph, ...
		// this also includes federated HDFS (viewfs).
		return FileSystemKind.FILE_SYSTEM;
	}
}
 

/**
 * Gets the kind of the file system from its scheme.
 *
 * <p>Implementation note: Initially, especially within the Flink 1.3.x line
 * (in order to not break backwards compatibility), we must only label file systems
 * as 'inconsistent' or as 'not proper filesystems' if we are sure about it.
 * Otherwise, we cause regression for example in the performance and cleanup handling
 * of checkpoints.
 * For that reason, we initially mark some filesystems as 'eventually consistent' or
 * as 'object stores', and leave the others as 'consistent file systems'.
 */
static FileSystemKind getKindForScheme(String scheme) {
	scheme = scheme.toLowerCase(Locale.US);

	if (scheme.startsWith("s3") || scheme.startsWith("emr") || scheme.startsWith("oss")) {
		// the Amazon S3 storage or Aliyun OSS storage
		return FileSystemKind.OBJECT_STORE;
	}
	else if (scheme.startsWith("http") || scheme.startsWith("ftp")) {
		// file servers instead of file systems
		// they might actually be consistent, but we have no hard guarantees
		// currently to rely on that
		return FileSystemKind.OBJECT_STORE;
	}
	else {
		// the remainder should include hdfs, kosmos, ceph, ...
		// this also includes federated HDFS (viewfs).
		return FileSystemKind.FILE_SYSTEM;
	}
}
 

/**
 * Gets the kind of the file system from its scheme.
 *
 * <p>Implementation note: Initially, especially within the Flink 1.3.x line
 * (in order to not break backwards compatibility), we must only label file systems
 * as 'inconsistent' or as 'not proper filesystems' if we are sure about it.
 * Otherwise, we cause regression for example in the performance and cleanup handling
 * of checkpoints.
 * For that reason, we initially mark some filesystems as 'eventually consistent' or
 * as 'object stores', and leave the others as 'consistent file systems'.
 */
static FileSystemKind getKindForScheme(String scheme) {
	scheme = scheme.toLowerCase(Locale.US);

	if (scheme.startsWith("s3") || scheme.startsWith("emr") || scheme.startsWith("oss") || scheme.startsWith("wasb")) {
		// the Amazon S3 storage or Aliyun OSS storage or Azure Blob Storage
		return FileSystemKind.OBJECT_STORE;
	}
	else if (scheme.startsWith("http") || scheme.startsWith("ftp")) {
		// file servers instead of file systems
		// they might actually be consistent, but we have no hard guarantees
		// currently to rely on that
		return FileSystemKind.OBJECT_STORE;
	}
	else {
		// the remainder should include hdfs, kosmos, ceph, ...
		// this also includes federated HDFS (viewfs).
		return FileSystemKind.FILE_SYSTEM;
	}
}
 

@Test
public void testS3fileSystemSchemes() {
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3n"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3a"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("EMRFS"));
}
 

@Test
public void testMapRFsKind() throws Exception {
	final Path path = new Path("maprfs:///my/path");

	final FileSystem fs = path.getFileSystem();

	assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind());
}
 

@Override
public FileSystemKind getKind() {
	if (fsKind == null) {
		fsKind = getKindForScheme(this.fs.getUri().getScheme());
	}
	return fsKind;
}
 

@Test
public void testMapRFsKind() throws Exception {
	final Path path = new Path("maprfs:///my/path");

	final FileSystem fs = path.getFileSystem();

	assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind());
}
 

@Override
public FileSystemKind getKind() {
	if (fsKind == null) {
		fsKind = getKindForScheme(this.fs.getUri().getScheme());
	}
	return fsKind;
}
 

@Test
public void testS3fileSystemSchemes() {
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3n"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3a"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("EMRFS"));
}
 

@Test
public void testS3aKind() throws IOException {
	try {
		Class.forName("org.apache.hadoop.fs.s3a.S3AFileSystem");
	} catch (ClassNotFoundException ignored) {
		// not in the classpath, cannot run this test
		log.info("Skipping test 'testS3aKind()' because the S3AFileSystem is not in the class path");
		return;
	}

	final FileSystem s3a = new Path("s3a://myId:mySecret@bucket/some/bucket/some/object").getFileSystem();
	assertEquals(FileSystemKind.OBJECT_STORE, s3a.getKind());
}
 

@Test
public void testS3nKind() throws IOException {
	try {
		Class.forName("org.apache.hadoop.fs.s3native.NativeS3FileSystem");
	} catch (ClassNotFoundException ignored) {
		// not in the classpath, cannot run this test
		log.info("Skipping test 'testS3nKind()' because the Native S3 file system is not in the class path");
		return;
	}

	final FileSystem s3n = new Path("s3n://myId:mySecret@bucket/some/bucket/some/object").getFileSystem();
	assertEquals(FileSystemKind.OBJECT_STORE, s3n.getKind());
}
 

@Test
public void testS3Kind() throws IOException {
	try {
		Class.forName("org.apache.hadoop.fs.s3.S3FileSystem");
	} catch (ClassNotFoundException ignored) {
		// not in the classpath, cannot run this test
		log.info("Skipping test 'testS3Kind()' because the S3 file system is not in the class path");
		return;
	}

	final FileSystem s3 = new Path("s3://myId:mySecret@bucket/some/bucket/some/object").getFileSystem();
	assertEquals(FileSystemKind.OBJECT_STORE, s3.getKind());
}
 

@Override
public FileSystemKind getKind() {
	if (fsKind == null) {
		fsKind = getKindForScheme(this.fs.getUri().getScheme());
	}
	return fsKind;
}
 

@Test
public void testS3fileSystemSchemes() {
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3n"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("s3a"));
	assertEquals(FileSystemKind.OBJECT_STORE, HadoopFileSystem.getKindForScheme("EMRFS"));
}
 

@Test
public void testS3Kind() throws IOException {
	try {
		Class.forName("org.apache.hadoop.fs.s3.S3FileSystem");
	} catch (ClassNotFoundException ignored) {
		// not in the classpath, cannot run this test
		log.info("Skipping test 'testS3Kind()' because the S3 file system is not in the class path");
		return;
	}

	final FileSystem s3 = new Path("s3://myId:mySecret@bucket/some/bucket/some/object").getFileSystem();
	assertEquals(FileSystemKind.OBJECT_STORE, s3.getKind());
}
 

@Test
public void testS3nKind() throws IOException {
	try {
		Class.forName("org.apache.hadoop.fs.s3native.NativeS3FileSystem");
	} catch (ClassNotFoundException ignored) {
		// not in the classpath, cannot run this test
		log.info("Skipping test 'testS3nKind()' because the Native S3 file system is not in the class path");
		return;
	}

	final FileSystem s3n = new Path("s3n://myId:mySecret@bucket/some/bucket/some/object").getFileSystem();
	assertEquals(FileSystemKind.OBJECT_STORE, s3n.getKind());
}
 

@Test
public void testS3aKind() throws IOException {
	try {
		Class.forName("org.apache.hadoop.fs.s3a.S3AFileSystem");
	} catch (ClassNotFoundException ignored) {
		// not in the classpath, cannot run this test
		log.info("Skipping test 'testS3aKind()' because the S3AFileSystem is not in the class path");
		return;
	}

	final FileSystem s3a = new Path("s3a://myId:mySecret@bucket/some/bucket/some/object").getFileSystem();
	assertEquals(FileSystemKind.OBJECT_STORE, s3a.getKind());
}
 

@Override
public FileSystemKind getKind() {
	return FileSystemKind.OBJECT_STORE;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	return FileSystemKind.OBJECT_STORE;
}
 

@Test
public void testViewFs() {
	assertEquals(FileSystemKind.FILE_SYSTEM, HadoopFileSystem.getKindForScheme("viewfs"));
}
 

@Override
public FileSystemKind getKind() {
	return FileSystemKind.FILE_SYSTEM;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	return FileSystemKind.OBJECT_STORE;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	return FileSystemKind.FILE_SYSTEM;
}
 

@Test
public void testKind() {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind());
}
 

@Override
public FileSystemKind getKind() {
	return FileSystemKind.OBJECT_STORE;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	return FileSystemKind.OBJECT_STORE;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	return FileSystemKind.OBJECT_STORE;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	return FileSystemKind.FILE_SYSTEM;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	return FileSystemKind.FILE_SYSTEM;
}
 

@Override
public FileSystemKind getFileSystemKind() {
	// this tests tests only HDFS, so it should be a file system
	return FileSystemKind.FILE_SYSTEM;
}