Java Code Examples for org.apache.flink.api.java.ExecutionEnvironment#registerCachedFile()

public static void main(String[] args) throws Exception {
    ExecutionEnvironment environment = ExecutionEnvironment.getExecutionEnvironment();
    String filePath = "file:\\D:\\imooc\\新一代大数据计算引擎 Flink从入门到实战-v\\input\\hello.txt";

    //1. 注册一个本地文件
    environment.registerCachedFile(filePath, "java-cf");
    DataSource<String> data = environment.fromElements("hadoop", "spark", "flink", "pyspark", "storm");

    data.map(new RichMapFunction<String, String>() {
        List<String> list = new ArrayList<>();
        @Override
        public void open(Configuration parameters) throws Exception {
            File file = getRuntimeContext().getDistributedCache().getFile("java-cf");
            List<String> lines = FileUtils.readLines(file);
            for (String line : lines) {
                System.out.println("line: " + line);
            }

        }

        @Override
        public String map(String value) throws Exception {
            return value;
        }
    }).print();
}
 

@Test
public void testDistributedCacheWithIterations() throws Exception{
	final String testString = "Et tu, Brute?";
	final String testName = "testing_caesar";

	final File folder = tempFolder.newFolder();
	final File resultFile = new File(folder, UUID.randomUUID().toString());

	String testPath = resultFile.toString();
	String resultPath = resultFile.toURI().toString();

	File tempFile = new File(testPath);
	try (FileWriter writer = new FileWriter(tempFile)) {
		writer.write(testString);
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(resultPath, testName);

	IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
	solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {
		@Override
		public void open(Configuration parameters) throws Exception{
			File file = getRuntimeContext().getDistributedCache().getFile(testName);
			BufferedReader reader = new BufferedReader(new FileReader(file));
			String output = reader.readLine();
			reader.close();
			assertEquals(output, testString);
		}

		@Override
		public boolean filter(Long value) throws Exception {
			return false;
		}
	}).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
	env.execute();
}
 

@Test
public void testBatchDistributedCache() throws Exception {
	String textPath = createTempFile("count.txt", DATA);
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(textPath, "cache_test");
	env.readTextFile(textPath).flatMap(new WordChecker()).count();
}
 

@Test
public void testDistributedCacheWithIterations() throws Exception{
	final String testString = "Et tu, Brute?";
	final String testName = "testing_caesar";

	final File folder = tempFolder.newFolder();
	final File resultFile = new File(folder, UUID.randomUUID().toString());

	String testPath = resultFile.toString();
	String resultPath = resultFile.toURI().toString();

	File tempFile = new File(testPath);
	try (FileWriter writer = new FileWriter(tempFile)) {
		writer.write(testString);
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(resultPath, testName);

	IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
	solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {
		@Override
		public void open(Configuration parameters) throws Exception{
			File file = getRuntimeContext().getDistributedCache().getFile(testName);
			BufferedReader reader = new BufferedReader(new FileReader(file));
			String output = reader.readLine();
			reader.close();
			assertEquals(output, testString);
		}

		@Override
		public boolean filter(Long value) throws Exception {
			return false;
		}
	}).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
	env.execute();
}
 

@Test
public void testBatchDistributedCache() throws Exception {
	String textPath = createTempFile("count.txt", DATA);
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(textPath, "cache_test");
	env.readTextFile(textPath).flatMap(new WordChecker()).count();
}
 

@Test
public void testDistributedCacheWithIterations() throws Exception{
	final String testString = "Et tu, Brute?";
	final String testName = "testing_caesar";

	final File folder = tempFolder.newFolder();
	final File resultFile = new File(folder, UUID.randomUUID().toString());

	String testPath = resultFile.toString();
	String resultPath = resultFile.toURI().toString();

	File tempFile = new File(testPath);
	try (FileWriter writer = new FileWriter(tempFile)) {
		writer.write(testString);
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(resultPath, testName);

	IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
	solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {
		@Override
		public void open(Configuration parameters) throws Exception{
			File file = getRuntimeContext().getDistributedCache().getFile(testName);
			BufferedReader reader = new BufferedReader(new FileReader(file));
			String output = reader.readLine();
			reader.close();
			assertEquals(output, testString);
		}

		@Override
		public boolean filter(Long value) throws Exception {
			return false;
		}
	}).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
	env.execute();
}
 

@Test
public void testBatchDistributedCache() throws Exception {
	String textPath = createTempFile("count.txt", DATA);
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(textPath, "cache_test");
	env.readTextFile(textPath).flatMap(new WordChecker()).count();
}
 

@Test
public void testGenerate() {

	final String fileA = "fileA";
	final String fileB = "fileB";

	final Map<String, DistributedCache.DistributedCacheEntry> originalArtifacts = Stream.of(
			Tuple2.of(fileA, new DistributedCache.DistributedCacheEntry("test1", true)),
			Tuple2.of(fileB, new DistributedCache.DistributedCacheEntry("test2", false))
	).collect(Collectors.toMap(x -> x.f0, x -> x.f1));

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(10);
	env.registerCachedFile("test1", fileA, true);
	env.registerCachedFile("test2", fileB, false);

	env.fromElements(1, 3, 5)
			.map((MapFunction<Integer, String>) value -> String.valueOf(value + 1))
			.writeAsText("/tmp/csv");

	final Plan generatedPlanUnderTest = env.createProgramPlan("test");

	final Map<String, DistributedCache.DistributedCacheEntry> retrievedArtifacts =
			generatedPlanUnderTest
					.getCachedFiles()
					.stream()
					.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

	assertEquals(1, generatedPlanUnderTest.getDataSinks().size());
	assertEquals(10, generatedPlanUnderTest.getDefaultParallelism());
	assertEquals(env.getConfig(), generatedPlanUnderTest.getExecutionConfig());
	assertEquals("test", generatedPlanUnderTest.getJobName());

	assertEquals(originalArtifacts.size(), retrievedArtifacts.size());
	assertEquals(originalArtifacts.get(fileA), retrievedArtifacts.get(fileA));
	assertEquals(originalArtifacts.get(fileB), retrievedArtifacts.get(fileB));
}