Java Code Examples for org.apache.flink.core.fs.Path#getPath()

@Override
public boolean filterPath(Path filePath) {
	if (getIncludeMatchers().isEmpty() && getExcludeMatchers().isEmpty()) {
		return false;
	}

	// compensate for the fact that Flink paths are slashed
	final String path = filePath.hasWindowsDrive() ?
			filePath.getPath().substring(1) :
			filePath.getPath();

	final java.nio.file.Path nioPath = Paths.get(path);

	for (PathMatcher matcher : getIncludeMatchers()) {
		if (matcher.matches(nioPath)) {
			return shouldExclude(nioPath);
		}
	}

	return true;
}
 

@Override
public boolean filterPath(Path filePath) {
	if (getIncludeMatchers().isEmpty() && getExcludeMatchers().isEmpty()) {
		return false;
	}

	// compensate for the fact that Flink paths are slashed
	final String path = filePath.hasWindowsDrive() ?
			filePath.getPath().substring(1) :
			filePath.getPath();

	final java.nio.file.Path nioPath = Paths.get(path);

	for (PathMatcher matcher : getIncludeMatchers()) {
		if (matcher.matches(nioPath)) {
			return shouldExclude(nioPath);
		}
	}

	return true;
}
 

/**
 * This test checks that the expected mkdirs action for checkpoint storage,
 * only be called when initializeBaseLocations and not called when resolveCheckpointStorageLocation.
 */
@Test
public void testStorageLocationMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir = new File(storage.getCheckpointsDirectory().getPath());
	assertFalse(baseDir.exists());

	// mkdirs would only be called when initializeBaseLocations
	storage.initializeBaseLocations();
	assertTrue(baseDir.exists());

	// mkdir would not be called when resolveCheckpointStorageLocation
	storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 

/**
 * Reads the given archive file and returns a {@link Collection} of contained {@link ArchivedJson}.
 *
 * @param file archive to extract
 * @return collection of archived jsons
 * @throws IOException if the file can't be opened, read or doesn't contain valid json
 */
public static Collection<ArchivedJson> getArchivedJsons(Path file) throws IOException {
	try (FSDataInputStream input = file.getFileSystem().open(file);
		ByteArrayOutputStream output = new ByteArrayOutputStream()) {
		IOUtils.copyBytes(input, output);

		try {
			JsonNode archive = mapper.readTree(output.toByteArray());

			Collection<ArchivedJson> archives = new ArrayList<>();
			for (JsonNode archivePart : archive.get(ARCHIVE)) {
				String path = archivePart.get(PATH).asText();
				String json = archivePart.get(JSON).asText();
				archives.add(new ArchivedJson(path, json));
			}
			return archives;
		} catch (NullPointerException npe) {
			// occurs if the archive is empty or any of the expected fields are not present
			throw new IOException("Job archive (" + file.getPath() + ") did not conform to expected format.");
		}
	}
}
 

@Override
public boolean filterPath(Path filePath) {
	if (getIncludeMatchers().isEmpty() && getExcludeMatchers().isEmpty()) {
		return false;
	}

	// compensate for the fact that Flink paths are slashed
	final String path = filePath.hasWindowsDrive() ?
			filePath.getPath().substring(1) :
			filePath.getPath();

	final java.nio.file.Path nioPath = Paths.get(path);

	for (PathMatcher matcher : getIncludeMatchers()) {
		if (matcher.matches(nioPath)) {
			return shouldExclude(nioPath);
		}
	}

	return true;
}
 

@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
}
 

@Override
public void open(int taskNumber, int numTasks) throws IOException {
	super.open(taskNumber, numTasks);
	if (config != null) {
		TSFileConfigUtil.setGlobalTSFileConfig(config);
	}
	// Use TsFile API to write instead of FSDataOutputStream.
	this.stream.close();
	Path actualFilePath = getAcutalFilePath();
	TsFileOutput out;
	try {
		if (actualFilePath.getFileSystem().isDistributedFS()) {
			// HDFS
			out = new HDFSOutput(
				new org.apache.hadoop.fs.Path(new URI(actualFilePath.getPath())), hadoopConf, true);
		} else {
			// Local File System
			out = new LocalTsFileOutput(new FileOutputStream(actualFilePath.getPath()));
		}
	} catch (URISyntaxException e) {
		throw new RuntimeException(e);
	}
	writer = new TsFileWriter(out, schema);
}
 

/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 

/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 

@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
}
 

/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 

/**
 * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from
 * a local state.
 */
private void restoreInstanceDirectoryFromPath(Path source, String instanceRocksDBPath) throws IOException {

	FileSystem fileSystem = source.getFileSystem();

	final FileStatus[] fileStatuses = fileSystem.listStatus(source);

	if (fileStatuses == null) {
		throw new IOException("Cannot list file statues. Directory " + source + " does not exist.");
	}

	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		File restoreFile = new File(source.getPath(), fileName);
		File targetFile = new File(instanceRocksDBPath, fileName);
		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			// hardlink'ing the immutable sst-files.
			Files.createLink(targetFile.toPath(), restoreFile.toPath());
		} else {
			// true copy for all other files.
			Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
		}
	}
}
 

@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
}
 

@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		final java.nio.file.Path directory = temporaryFolder.newFolder("zipArchive").toPath();
		final java.nio.file.Path containedFile = directory.resolve("cacheFile");
		Files.copy(new ByteArrayInputStream(testFileContent.getBytes(StandardCharsets.UTF_8)), containedFile);
		Path zipPath = FileUtils.compressDirectory(new Path(directory.toString()), new Path(directory + ".zip"));
		return new File(zipPath.getPath());
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 

@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		final java.nio.file.Path directory = temporaryFolder.newFolder("zipArchive").toPath();
		final java.nio.file.Path containedFile = directory.resolve("cacheFile");
		Files.copy(new ByteArrayInputStream(testFileContent.getBytes(StandardCharsets.UTF_8)), containedFile);
		Path zipPath = FileUtils.compressDirectory(new Path(directory.toString()), new Path(directory + ".zip"));
		return new File(zipPath.getPath());
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 

@Test
public void testProgram() throws Exception {
	Path testEntryPoint = new Path(getBaseTestPythonDir(), "run_all_tests.py");
	List<String> testFiles = findTestFiles();

	Preconditions.checkState(testFiles.size() > 0, "No test files were found in {}.", getBaseTestPythonDir());

	String[] arguments = new String[1 + 1 + testFiles.size()];
	arguments[0] = testEntryPoint.getPath();
	arguments[1] = findUtilsModule().getPath();
	int index = 2;
	for (String testFile : testFiles) {
		arguments[index] = testFile;
		index++;
	}
	try {
		new PythonStreamBinder(new Configuration())
			.runPlan(arguments);
	} catch (PyException e) {
		if (e.getCause() instanceof JobExecutionException) {
			// JobExecutionExceptions are wrapped again by the jython interpreter resulting in horrible stacktraces
			throw (JobExecutionException) e.getCause();
		} else {
			// probably caused by some issue in the main script itself
			throw e;
		}
	}
}
 

@Override
public File getFile(JobID jobId, PermanentBlobKey key) throws IOException {
	if (key.equals(permanentBlobKey)) {
		final java.nio.file.Path directory = temporaryFolder.newFolder("zipArchive").toPath();
		final java.nio.file.Path containedFile = directory.resolve("cacheFile");
		Files.copy(new ByteArrayInputStream(testFileContent.getBytes(StandardCharsets.UTF_8)), containedFile);
		Path zipPath = FileUtils.compressDirectory(new Path(directory.toString()), new Path(directory + ".zip"));
		return new File(zipPath.getPath());
	} else {
		throw new IllegalArgumentException("This service contains only entry for " + permanentBlobKey);
	}
}
 

private int readSplitAndCheck(
		int start,
		long seekToRow,
		Path testPath,
		long splitStart,
		long splitLength,
		List<Integer> values) throws IOException {
	LogicalType[] fieldTypes = new LogicalType[]{
			new VarCharType(VarCharType.MAX_LENGTH),
			new BooleanType(),
			new TinyIntType(),
			new SmallIntType(),
			new IntType(),
			new BigIntType(),
			new FloatType(),
			new DoubleType(),
			new TimestampType(9),
			new DecimalType(5, 0),
			new DecimalType(15, 0),
			new DecimalType(20, 0),
			new DecimalType(5, 0),
			new DecimalType(15, 0),
			new DecimalType(20, 0)};

	ParquetColumnarRowSplitReader reader = new ParquetColumnarRowSplitReader(
			false,
			true,
			new Configuration(),
			fieldTypes,
			new String[] {
					"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
					"f8", "f9", "f10", "f11", "f12", "f13", "f14"},
			VectorizedColumnBatch::new,
			500,
			new org.apache.hadoop.fs.Path(testPath.getPath()),
			splitStart,
			splitLength);
	reader.seekToRow(seekToRow);

	int i = start;
	while (!reader.reachedEnd()) {
		ColumnarRowData row = reader.nextRecord();
		Integer v = values.get(i);
		if (v == null) {
			assertTrue(row.isNullAt(0));
			assertTrue(row.isNullAt(1));
			assertTrue(row.isNullAt(2));
			assertTrue(row.isNullAt(3));
			assertTrue(row.isNullAt(4));
			assertTrue(row.isNullAt(5));
			assertTrue(row.isNullAt(6));
			assertTrue(row.isNullAt(7));
			assertTrue(row.isNullAt(8));
			assertTrue(row.isNullAt(9));
			assertTrue(row.isNullAt(10));
			assertTrue(row.isNullAt(11));
			assertTrue(row.isNullAt(12));
			assertTrue(row.isNullAt(13));
			assertTrue(row.isNullAt(14));
		} else {
			assertEquals("" + v, row.getString(0).toString());
			assertEquals(v % 2 == 0, row.getBoolean(1));
			assertEquals(v.byteValue(), row.getByte(2));
			assertEquals(v.shortValue(), row.getShort(3));
			assertEquals(v.intValue(), row.getInt(4));
			assertEquals(v.longValue(), row.getLong(5));
			assertEquals(v.floatValue(), row.getFloat(6), 0);
			assertEquals(v.doubleValue(), row.getDouble(7), 0);
			assertEquals(
					toDateTime(v),
					row.getTimestamp(8, 9).toLocalDateTime());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(9, 5, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(10, 15, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(11, 20, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(12, 5, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(13, 15, 0).toBigDecimal());
			assertEquals(BigDecimal.valueOf(v), row.getDecimal(14, 20, 0).toBigDecimal());
		}
		i++;
	}
	reader.close();
	return i - start;
}
 

/**
 * Converts the given Path to a File for this file system.
 */
public File pathToFile(Path path) {
	return new File(path.getPath());
}