org.apache.flink.test.util.SuccessException Java Examples

@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = PackagedProgram.newBuilder()
		.setJarFile(new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH))
		.setArguments(new String[] { checkpointDir.toURI().toString(), outputDir.toURI().toString()})
		.build();

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
		Collections.emptyList());

	try {
		program.invokeInteractiveModeForExecution();
		fail("exception should happen");
	} catch (ProgramInvocationException e) {
		assertTrue(ExceptionUtils.findThrowable(e, SuccessException.class).isPresent());
	}
}
 

protected static void tryExecutePropagateExceptions(StreamExecutionEnvironment see, String name) throws Exception {
	try {
		see.execute(name);
	}
	catch (ProgramInvocationException | JobExecutionException root) {
		Throwable cause = root.getCause();

		// search for nested SuccessExceptions
		int depth = 0;
		while (!(cause instanceof SuccessException)) {
			if (cause == null || depth++ == 20) {
				throw root;
			}
			else {
				cause = cause.getCause();
			}
		}
	}
}
 

@Override
public void invoke(Integer value) throws Exception {
	numElements++;

	if (duplicateChecker.get(value)) {
		throw new Exception("Received a duplicate: " + value);
	}
	duplicateChecker.set(value);
	if (numElements == numElementsTotal) {
		// validate
		if (duplicateChecker.cardinality() != numElementsTotal) {
			throw new Exception("Duplicate checker has wrong cardinality");
		}
		else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
			throw new Exception("Received sparse sequence");
		}
		else {
			throw new SuccessException();
		}
	}
}
 

protected static void tryExecutePropagateExceptions(StreamExecutionEnvironment see, String name) throws Exception {
	try {
		see.execute(name);
	}
	catch (ProgramInvocationException | JobExecutionException root) {
		Throwable cause = root.getCause();

		// search for nested SuccessExceptions
		int depth = 0;
		while (!(cause instanceof SuccessException)) {
			if (cause == null || depth++ == 20) {
				throw root;
			}
			else {
				cause = cause.getCause();
			}
		}
	}
}
 

@Override
public void invoke(Row value) throws Exception {
    numElements++;

    int v = (Integer) value.getField(0);

    if (duplicateChecker.get(v)) {
        throw new Exception("Received a duplicate: " + v);
    }
    duplicateChecker.set(v);
    if (numElements == numElementsTotal) {
        // validate
        if (duplicateChecker.cardinality() != numElementsTotal) {
            throw new Exception("Duplicate checker has wrong cardinality");
        } else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
            throw new Exception("Received sparse sequence");
        } else {
            throw new SuccessException();
        }
    }
}
 

@Override
public void flatMap(String value, Collector<String> out) throws Exception {
	LOG.info("Consumed {}", value);

	int id = Integer.parseInt(value.split("-")[0]);
	if (validator.get(id)) {
		throw new RuntimeException("Saw id " + id + " twice!");
	}
	validator.set(id);
	if (id > totalEventCount - 1) {
		throw new RuntimeException("Out of bounds ID observed");
	}

	if (validator.nextClearBit(0) == totalEventCount) {
		throw new SuccessException();
	}
}
 

@Override
public void invoke(Integer value) throws Exception {
	numElements++;

	if (duplicateChecker.get(value)) {
		throw new Exception("Received a duplicate: " + value);
	}
	duplicateChecker.set(value);
	if (numElements == numElementsTotal) {
		// validate
		if (duplicateChecker.cardinality() != numElementsTotal) {
			throw new Exception("Duplicate checker has wrong cardinality");
		}
		else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
			throw new Exception("Received sparse sequence");
		}
		else {
			throw new SuccessException();
		}
	}
}
 

@Override
public void flatMap(String value, Collector<String> out) throws Exception {
	LOG.info("Consumed {}", value);

	int id = Integer.parseInt(value.split("-")[0]);
	if (validator.get(id)) {
		throw new RuntimeException("Saw id " + id + " twice!");
	}
	validator.set(id);
	if (id > totalEventCount - 1) {
		throw new RuntimeException("Out of bounds ID observed");
	}

	if (validator.nextClearBit(0) == totalEventCount) {
		throw new SuccessException();
	}
}
 

@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = new PackagedProgram(
		new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH),
		new String[] {
			checkpointDir.toURI().toString(),
			outputDir.toURI().toString()
		});

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
		Collections.<URL>emptyList());

	expectedException.expectCause(
		Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class)));

	program.invokeInteractiveModeForExecution();
}
 

@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = new PackagedProgram(
		new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH),
		new String[] {
			checkpointDir.toURI().toString(),
			outputDir.toURI().toString()
		});

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
		Collections.<URL>emptyList());

	expectedException.expectCause(
		Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class)));

	program.invokeInteractiveModeForExecution();
}
 

private void createConsumerTopology(StreamExecutionEnvironment env, AMQSourceConfig<String> config) {
    AMQSource<String> source = new AMQSource<>(config);

    env.addSource(source)
        .addSink(new SinkFunction<String>() {
            final HashSet<Integer> set = new HashSet<>();
            @Override
            public void invoke(String value, Context context) throws Exception {
                int val = Integer.parseInt(value.split("-")[1]);
                set.add(val);

                if (set.size() == MESSAGES_NUM) {
                    throw new SuccessException();
                }
            }
        });
}
 

@Override
public void flatMap(String value, Collector<String> out) throws Exception {
	LOG.info("Consumed {}", value);

	int id = Integer.parseInt(value.split("-")[0]);
	if (validator.get(id)) {
		throw new RuntimeException("Saw id " + id + " twice!");
	}
	validator.set(id);
	if (id > totalEventCount - 1) {
		throw new RuntimeException("Out of bounds ID observed");
	}

	if (validator.nextClearBit(0) == totalEventCount) {
		throw new SuccessException();
	}
}
 

@Override
public void invoke(Integer value) throws Exception {
	numElements++;

	if (duplicateChecker.get(value)) {
		throw new Exception("Received a duplicate: " + value);
	}
	duplicateChecker.set(value);
	if (numElements == numElementsTotal) {
		// validate
		if (duplicateChecker.cardinality() != numElementsTotal) {
			throw new Exception("Duplicate checker has wrong cardinality");
		}
		else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
			throw new Exception("Received sparse sequence");
		}
		else {
			throw new SuccessException();
		}
	}
}
 

protected static void tryExecutePropagateExceptions(StreamExecutionEnvironment see, String name) throws Exception {
	try {
		see.execute(name);
	}
	catch (ProgramInvocationException | JobExecutionException root) {
		Throwable cause = root.getCause();

		// search for nested SuccessExceptions
		int depth = 0;
		while (!(cause instanceof SuccessException)) {
			if (cause == null || depth++ == 20) {
				throw root;
			}
			else {
				cause = cause.getCause();
			}
		}
	}
}
 

/**
 * Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)}
 * followed by the checks in {@link #postSubmit}.
 */
@Test
public void runCheckpointedProgram() throws Exception {
	try {
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(PARALLELISM);
		env.enableCheckpointing(500);
		env.getConfig().disableSysoutLogging();
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));

		testProgram(env);

		JobGraph jobGraph = env.getStreamGraph().getJobGraph();
		try {
			cluster.getClusterClient().submitJob(jobGraph, getClass().getClassLoader()).getJobExecutionResult();
		}
		catch (ProgramInvocationException root) {
			Throwable cause = root.getCause();

			// search for nested SuccessExceptions
			int depth = 0;
			while (!(cause instanceof SuccessException)) {
				if (cause == null || depth++ == 20) {
					root.printStackTrace();
					fail("Test failed: " + root.getMessage());
				}
				else {
					cause = cause.getCause();
				}
			}
		}

		postSubmit();
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

@Override
public Tuple3<Integer, Long, Integer> map(Tuple3<Integer, Long, Integer> element) throws Exception {
	counter++;

	if (counter == totalCount) {
		throw new SuccessException();
	}

	return element;
}
 

@Override
public void invoke(String value) throws Exception {
	int fileIdx = getFileIdx(value);

	Set<String> content = actualContent.get(fileIdx);
	if (content == null) {
		content = new HashSet<>();
		actualContent.put(fileIdx, content);
	}

	// detect duplicate lines.
	if (!content.add(value + "\n")) {
		fail("Duplicate line: " + value);
		System.exit(0);
	}

	elementCounter++;

	// this is termination
	if (elementCounter >= NO_OF_FILES * LINES_PER_FILE) {
		actualCollectedContent = actualContent;
		throw new SuccessException();
	}

	// add some latency so that we have at least two checkpoint in
	if (!hasRestoredAfterFailure && successfulCheckpoints < 2) {
		Thread.sleep(5);
	}

	// simulate a node failure
	if (!hasRestoredAfterFailure && successfulCheckpoints >= 2 && elementCounter >= elementsToFailure) {
		throw new Exception("Task Failure @ elem: " + elementCounter + " / " + elementsToFailure);
	}
}
 

@Override
public void invoke(String value) throws Exception {
	int fileIdx = getFileIdx(value);

	Set<String> content = actualContent.get(fileIdx);
	if (content == null) {
		content = new HashSet<>();
		actualContent.put(fileIdx, content);
	}

	// detect duplicate lines.
	if (!content.add(value + "\n")) {
		fail("Duplicate line: " + value);
		System.exit(0);
	}

	elementCounter++;

	// this is termination
	if (elementCounter >= NO_OF_FILES * LINES_PER_FILE) {
		actualCollectedContent = actualContent;
		throw new SuppressRestartsException(new SuccessException());
	}

	// add some latency so that we have at least two checkpoint in
	if (!hasRestoredAfterFailure && successfulCheckpoints < 2) {
		Thread.sleep(5);
	}

	// simulate a node failure
	if (!hasRestoredAfterFailure && successfulCheckpoints >= 2 && elementCounter >= elementsToFailure) {
		throw new Exception("Task Failure @ elem: " + elementCounter + " / " + elementsToFailure);
	}
}
 

@Override
public void invoke(Integer value) throws Exception {
	valuesPerPartition[value]++;

	boolean missing = false;
	for (int i : valuesPerPartition) {
		if (i < 100) {
			missing = true;
			break;
		}
	}
	if (!missing) {
		throw new SuccessException();
	}
}
 

@Override
public void invoke(RowData value, Context context) throws Exception {
	rows.add(value.toString());
	if (rows.size() >= expectedSize) {
		// job finish
		throw new SuccessException();
	}
}
 

protected static boolean isCausedByJobFinished(Throwable e) {
	if (e instanceof SuccessException) {
		return true;
	} else if (e.getCause() != null) {
		return isCausedByJobFinished(e.getCause());
	} else {
		return false;
	}
}
 

@Override
public void flatMap(Tuple2<Integer, Integer> value, Collector<Integer> out) throws Exception {
	kvState.add(value.f1);

	if (atLeastOneSnapshotComplete) {
		if (restored) {
			throw new SuccessException();
		} else {
			throw new RuntimeException("Intended failure, to trigger restore");
		}
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
	// this sink can only work with DOP 1
	assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
	if (usingProcessingTime && resultChecker.checkResult(windowCounts)) {
		throw new SuccessException();
	}
}
 

@Override
public void close() {
	if (resultChecker.checkResult(windowCounts)) {
		if (usingProcessingTime) {
			throw new SuccessException();
		}
	} else {
		throw new AssertionError("Test failed check.");
	}
}
 

@Override
public void invoke(T value, Context context) throws Exception {
	countUpdater.updateCount(value, windowCounts);
	if (usingProcessingTime && resultChecker.checkResult(windowCounts)) {
		throw new SuccessException();
	}
}
 

/**
 * Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)}
 * followed by the checks in {@link #postSubmit}.
 */
@Test
public void runCheckpointedProgram() throws Exception {
	try {
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(PARALLELISM);
		env.enableCheckpointing(500);
					env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));

		testProgram(env);

		JobGraph jobGraph = env.getStreamGraph().getJobGraph();
		try {
			ClientUtils.submitJobAndWaitForResult(cluster.getClusterClient(), jobGraph, getClass().getClassLoader()).getJobExecutionResult();
		} catch (ProgramInvocationException root) {
			Throwable cause = root.getCause();

			// search for nested SuccessExceptions
			int depth = 0;
			while (!(cause instanceof SuccessException)) {
				if (cause == null || depth++ == 20) {
					root.printStackTrace();
					fail("Test failed: " + root.getMessage());
				}
				else {
					cause = cause.getCause();
				}
			}
		}

		postSubmit();
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

@Override
public void close() {
	if (resultChecker.checkResult(windowCounts)) {
		if (usingProcessingTime) {
			throw new SuccessException();
		}
	} else {
		throw new AssertionError("Test failed check.");
	}
}
 

@Override
public void invoke(Integer value) throws Exception {
	valuesPerPartition[value]++;

	boolean missing = false;
	for (int i : valuesPerPartition) {
		if (i < 100) {
			missing = true;
			break;
		}
	}
	if (!missing) {
		throw new SuccessException();
	}
}
 

@Override
public void flatMap(Tuple2<Integer, Integer> value, Collector<Integer> out) throws Exception {
	kvState.add(value.f1);

	if (atLeastOneSnapshotComplete) {
		if (restored) {
			throw new SuccessException();
		} else {
			throw new RuntimeException("Intended failure, to trigger restore");
		}
	}
}
 

@Override
public void open(Configuration parameters) throws Exception {
	// this sink can only work with DOP 1
	assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
	if (usingProcessingTime && resultChecker.checkResult(windowCounts)) {
		throw new SuccessException();
	}
}