Java Code Examples for org.apache.flink.util.FileUtils#readFileUtf8()

The following examples show how to use org.apache.flink.util.FileUtils#readFileUtf8() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JsonRowSchemaConverterTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
Example 2
Source File: SQLClientHBaseITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void before() throws Exception {
	DOWNLOAD_CACHE.before();
	Path tmpPath = tmp.getRoot().toPath();
	LOG.info("The current temporary path: {}", tmpPath);

	// Prepare all hadoop jars to mock HADOOP_CLASSPATH, use hadoop.classpath which contains all hadoop jars
	File hadoopClasspathFile = new File(hadoopClasspath.toAbsolutePath().toString());

	if (!hadoopClasspathFile.exists()) {
		throw new FileNotFoundException("File that contains hadoop classpath " + hadoopClasspath.toString()
			+ " does not exist.");
	}

	String classPathContent = FileUtils.readFileUtf8(hadoopClasspathFile);
	hadoopClasspathJars = Arrays.stream(classPathContent.split(":"))
		.map(jar -> Paths.get(jar))
		.collect(Collectors.toList());
}
Example 3
Source File: FileCacheDirectoriesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void testDirectoryDownloaded(DistributedCache.DistributedCacheEntry entry) throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	// copy / create the file
	final String fileName = "test_file";
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}
Example 4
Source File: JsonRowSchemaConverterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
Example 5
Source File: FileCacheDirectoriesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}
Example 6
Source File: JsonRowSchemaConverterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
Example 7
Source File: FileCacheDirectoriesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDirectoryDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey),
		true);
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final FileSystem fs = dstPath.getFileSystem();
	final FileStatus fileStatus = fs.getFileStatus(dstPath);
	assertTrue(fileStatus.isDir());

	final Path cacheFile = new Path(dstPath, "cacheFile");
	assertTrue(fs.exists(cacheFile));
	final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
	assertEquals(testFileContent, actualContent);
}
Example 8
Source File: EnvironmentFileUtil.java    From pulsar-flink with Apache License 2.0 5 votes vote down vote up 
public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
    final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
    Objects.requireNonNull(url);
    String schema = FileUtils.readFileUtf8(new File(url.getFile()));

    for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
        schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
    }

    return Environment.parse(schema);
}
Example 9
Source File: EnvironmentFileUtil.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
	final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
	Objects.requireNonNull(url);
	String schema = FileUtils.readFileUtf8(new File(url.getFile()));

	for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
		schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
	}

	return Environment.parse(schema);
}
Example 10
Source File: JsonRowSchemaConverterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}
Example 11
Source File: EnvironmentFileUtil.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
	final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
	Objects.requireNonNull(url);
	String schema = FileUtils.readFileUtf8(new File(url.getFile()));

	for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
		schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
	}

	return Environment.parse(schema);
}
Example 12
Source File: JsonRowSchemaConverterTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}
Example 13
Source File: JsonRowSchemaConverterTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}
Example 14
Source File: EnvironmentFileUtil.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static Environment parseModified(String fileName, Map<String, String> replaceVars) throws IOException {
	final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName);
	Objects.requireNonNull(url);
	String schema = FileUtils.readFileUtf8(new File(url.getFile()));

	for (Map.Entry<String, String> replaceVar : replaceVars.entrySet()) {
		schema = schema.replace(replaceVar.getKey(), replaceVar.getValue());
	}

	return Environment.parse(schema);
}
Example 15
Source File: TupleGenerator.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		s.close();
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}
Example 16
Source File: TupleGenerator.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}
Example 17
Source File: TupleGenerator.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}
Example 18
Source File: TupleGenerator.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		s.close();
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}
Example 19
Source File: TupleGenerator.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		s.close();
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}
Example 20
Source File: TupleGenerator.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static void insertCodeIntoFile(String code, File file) throws IOException {
	String fileContent = FileUtils.readFileUtf8(file);

	try (Scanner s = new Scanner(fileContent)) {
		StringBuilder sb = new StringBuilder();
		String line;

		boolean indicatorFound = false;

		// add file beginning
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
			if (line.contains(BEGIN_INDICATOR)) {
				indicatorFound = true;
				break;
			}
		}

		if (!indicatorFound) {
			System.out.println("No indicator found in '" + file + "'. Will skip code generation.");
			s.close();
			return;
		}

		// add generator signature
		sb.append("\t// GENERATED FROM ").append(TupleGenerator.class.getName()).append(".\n");

		// add tuple dependent code
		sb.append(code).append("\n");

		// skip generated code
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			if (line.contains(END_INDICATOR)) {
				sb.append(line).append("\n");
				break;
			}
		}

		// add file ending
		while (s.hasNextLine() && (line = s.nextLine()) != null) {
			sb.append(line).append("\n");
		}
		FileUtils.writeFileUtf8(file, sb.toString());
	}
}