org.apache.flink.test.util.SuccessException Java Examples
The following examples show how to use
org.apache.flink.test.util.SuccessException.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ClassLoaderITCase.java From flink with Apache License 2.0 | 6 votes |
@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
File checkpointDir = FOLDER.newFolder();
File outputDir = FOLDER.newFolder();
final PackagedProgram program = PackagedProgram.newBuilder()
.setJarFile(new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH))
.setArguments(new String[] { checkpointDir.toURI().toString(), outputDir.toURI().toString()})
.build();
TestStreamEnvironment.setAsContext(
miniClusterResource.getMiniCluster(),
parallelism,
Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
Collections.emptyList());
try {
program.invokeInteractiveModeForExecution();
fail("exception should happen");
} catch (ProgramInvocationException e) {
assertTrue(ExceptionUtils.findThrowable(e, SuccessException.class).isPresent());
}
}
Example #2
Source File: KafkaTestBase.java From flink with Apache License 2.0 | 6 votes |
protected static void tryExecutePropagateExceptions(StreamExecutionEnvironment see, String name) throws Exception {
try {
see.execute(name);
}
catch (ProgramInvocationException | JobExecutionException root) {
Throwable cause = root.getCause();
// search for nested SuccessExceptions
int depth = 0;
while (!(cause instanceof SuccessException)) {
if (cause == null || depth++ == 20) {
throw root;
}
else {
cause = cause.getCause();
}
}
}
}
Example #3
Source File: ValidatingExactlyOnceSink.java From flink with Apache License 2.0 | 6 votes |
@Override
public void invoke(Integer value) throws Exception {
numElements++;
if (duplicateChecker.get(value)) {
throw new Exception("Received a duplicate: " + value);
}
duplicateChecker.set(value);
if (numElements == numElementsTotal) {
// validate
if (duplicateChecker.cardinality() != numElementsTotal) {
throw new Exception("Duplicate checker has wrong cardinality");
}
else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
throw new Exception("Received sparse sequence");
}
else {
throw new SuccessException();
}
}
}
Example #4
Source File: KafkaTestBase.java From flink with Apache License 2.0 | 6 votes |
protected static void tryExecutePropagateExceptions(StreamExecutionEnvironment see, String name) throws Exception {
try {
see.execute(name);
}
catch (ProgramInvocationException | JobExecutionException root) {
Throwable cause = root.getCause();
// search for nested SuccessExceptions
int depth = 0;
while (!(cause instanceof SuccessException)) {
if (cause == null || depth++ == 20) {
throw root;
}
else {
cause = cause.getCause();
}
}
}
}
Example #5
Source File: ValidatingExactlyOnceSink.java From pulsar-flink with Apache License 2.0 | 6 votes |
@Override
public void invoke(Row value) throws Exception {
numElements++;
int v = (Integer) value.getField(0);
if (duplicateChecker.get(v)) {
throw new Exception("Received a duplicate: " + v);
}
duplicateChecker.set(v);
if (numElements == numElementsTotal) {
// validate
if (duplicateChecker.cardinality() != numElementsTotal) {
throw new Exception("Duplicate checker has wrong cardinality");
} else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
throw new Exception("Received sparse sequence");
} else {
throw new SuccessException();
}
}
}
Example #6
Source File: ExactlyOnceValidatingConsumerThread.java From flink with Apache License 2.0 | 6 votes |
@Override
public void flatMap(String value, Collector<String> out) throws Exception {
LOG.info("Consumed {}", value);
int id = Integer.parseInt(value.split("-")[0]);
if (validator.get(id)) {
throw new RuntimeException("Saw id " + id + " twice!");
}
validator.set(id);
if (id > totalEventCount - 1) {
throw new RuntimeException("Out of bounds ID observed");
}
if (validator.nextClearBit(0) == totalEventCount) {
throw new SuccessException();
}
}
Example #7
Source File: ValidatingExactlyOnceSink.java From flink with Apache License 2.0 | 6 votes |
@Override
public void invoke(Integer value) throws Exception {
numElements++;
if (duplicateChecker.get(value)) {
throw new Exception("Received a duplicate: " + value);
}
duplicateChecker.set(value);
if (numElements == numElementsTotal) {
// validate
if (duplicateChecker.cardinality() != numElementsTotal) {
throw new Exception("Duplicate checker has wrong cardinality");
}
else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
throw new Exception("Received sparse sequence");
}
else {
throw new SuccessException();
}
}
}
Example #8
Source File: ExactlyOnceValidatingConsumerThread.java From flink with Apache License 2.0 | 6 votes |
@Override
public void flatMap(String value, Collector<String> out) throws Exception {
LOG.info("Consumed {}", value);
int id = Integer.parseInt(value.split("-")[0]);
if (validator.get(id)) {
throw new RuntimeException("Saw id " + id + " twice!");
}
validator.set(id);
if (id > totalEventCount - 1) {
throw new RuntimeException("Out of bounds ID observed");
}
if (validator.nextClearBit(0) == totalEventCount) {
throw new SuccessException();
}
}
Example #9
Source File: ClassLoaderITCase.java From flink with Apache License 2.0 | 6 votes |
@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
File checkpointDir = FOLDER.newFolder();
File outputDir = FOLDER.newFolder();
final PackagedProgram program = new PackagedProgram(
new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH),
new String[] {
checkpointDir.toURI().toString(),
outputDir.toURI().toString()
});
TestStreamEnvironment.setAsContext(
miniClusterResource.getMiniCluster(),
parallelism,
Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
Collections.<URL>emptyList());
expectedException.expectCause(
Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class)));
program.invokeInteractiveModeForExecution();
}
Example #10
Source File: ClassLoaderITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
File checkpointDir = FOLDER.newFolder();
File outputDir = FOLDER.newFolder();
final PackagedProgram program = new PackagedProgram(
new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH),
new String[] {
checkpointDir.toURI().toString(),
outputDir.toURI().toString()
});
TestStreamEnvironment.setAsContext(
miniClusterResource.getMiniCluster(),
parallelism,
Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
Collections.<URL>emptyList());
expectedException.expectCause(
Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class)));
program.invokeInteractiveModeForExecution();
}
Example #11
Source File: ActiveMQConnectorITCase.java From bahir-flink with Apache License 2.0 | 6 votes |
private void createConsumerTopology(StreamExecutionEnvironment env, AMQSourceConfig<String> config) {
AMQSource<String> source = new AMQSource<>(config);
env.addSource(source)
.addSink(new SinkFunction<String>() {
final HashSet<Integer> set = new HashSet<>();
@Override
public void invoke(String value, Context context) throws Exception {
int val = Integer.parseInt(value.split("-")[1]);
set.add(val);
if (set.size() == MESSAGES_NUM) {
throw new SuccessException();
}
}
});
}
Example #12
Source File: ExactlyOnceValidatingConsumerThread.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override
public void flatMap(String value, Collector<String> out) throws Exception {
LOG.info("Consumed {}", value);
int id = Integer.parseInt(value.split("-")[0]);
if (validator.get(id)) {
throw new RuntimeException("Saw id " + id + " twice!");
}
validator.set(id);
if (id > totalEventCount - 1) {
throw new RuntimeException("Out of bounds ID observed");
}
if (validator.nextClearBit(0) == totalEventCount) {
throw new SuccessException();
}
}
Example #13
Source File: ValidatingExactlyOnceSink.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override
public void invoke(Integer value) throws Exception {
numElements++;
if (duplicateChecker.get(value)) {
throw new Exception("Received a duplicate: " + value);
}
duplicateChecker.set(value);
if (numElements == numElementsTotal) {
// validate
if (duplicateChecker.cardinality() != numElementsTotal) {
throw new Exception("Duplicate checker has wrong cardinality");
}
else if (duplicateChecker.nextClearBit(0) != numElementsTotal) {
throw new Exception("Received sparse sequence");
}
else {
throw new SuccessException();
}
}
}
Example #14
Source File: KafkaTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
protected static void tryExecutePropagateExceptions(StreamExecutionEnvironment see, String name) throws Exception {
try {
see.execute(name);
}
catch (ProgramInvocationException | JobExecutionException root) {
Throwable cause = root.getCause();
// search for nested SuccessExceptions
int depth = 0;
while (!(cause instanceof SuccessException)) {
if (cause == null || depth++ == 20) {
throw root;
}
else {
cause = cause.getCause();
}
}
}
}
Example #15
Source File: StreamFaultToleranceTestBase.java From flink with Apache License 2.0 | 5 votes |
/**
* Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)}
* followed by the checks in {@link #postSubmit}.
*/
@Test
public void runCheckpointedProgram() throws Exception {
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.enableCheckpointing(500);
env.getConfig().disableSysoutLogging();
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));
testProgram(env);
JobGraph jobGraph = env.getStreamGraph().getJobGraph();
try {
cluster.getClusterClient().submitJob(jobGraph, getClass().getClassLoader()).getJobExecutionResult();
}
catch (ProgramInvocationException root) {
Throwable cause = root.getCause();
// search for nested SuccessExceptions
int depth = 0;
while (!(cause instanceof SuccessException)) {
if (cause == null || depth++ == 20) {
root.printStackTrace();
fail("Test failed: " + root.getMessage());
}
else {
cause = cause.getCause();
}
}
}
postSubmit();
}
catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
Example #16
Source File: KafkaShuffleTestBase.java From flink with Apache License 2.0 | 5 votes |
@Override
public Tuple3<Integer, Long, Integer> map(Tuple3<Integer, Long, Integer> element) throws Exception {
counter++;
if (counter == totalCount) {
throw new SuccessException();
}
return element;
}
Example #17
Source File: ContinuousFileProcessingCheckpointITCase.java From flink with Apache License 2.0 | 5 votes |
@Override
public void invoke(String value) throws Exception {
int fileIdx = getFileIdx(value);
Set<String> content = actualContent.get(fileIdx);
if (content == null) {
content = new HashSet<>();
actualContent.put(fileIdx, content);
}
// detect duplicate lines.
if (!content.add(value + "\n")) {
fail("Duplicate line: " + value);
System.exit(0);
}
elementCounter++;
// this is termination
if (elementCounter >= NO_OF_FILES * LINES_PER_FILE) {
actualCollectedContent = actualContent;
throw new SuccessException();
}
// add some latency so that we have at least two checkpoint in
if (!hasRestoredAfterFailure && successfulCheckpoints < 2) {
Thread.sleep(5);
}
// simulate a node failure
if (!hasRestoredAfterFailure && successfulCheckpoints >= 2 && elementCounter >= elementsToFailure) {
throw new Exception("Task Failure @ elem: " + elementCounter + " / " + elementsToFailure);
}
}
Example #18
Source File: ContinuousFileProcessingCheckpointITCase.java From flink with Apache License 2.0 | 5 votes |
@Override
public void invoke(String value) throws Exception {
int fileIdx = getFileIdx(value);
Set<String> content = actualContent.get(fileIdx);
if (content == null) {
content = new HashSet<>();
actualContent.put(fileIdx, content);
}
// detect duplicate lines.
if (!content.add(value + "\n")) {
fail("Duplicate line: " + value);
System.exit(0);
}
elementCounter++;
// this is termination
if (elementCounter >= NO_OF_FILES * LINES_PER_FILE) {
actualCollectedContent = actualContent;
throw new SuppressRestartsException(new SuccessException());
}
// add some latency so that we have at least two checkpoint in
if (!hasRestoredAfterFailure && successfulCheckpoints < 2) {
Thread.sleep(5);
}
// simulate a node failure
if (!hasRestoredAfterFailure && successfulCheckpoints >= 2 && elementCounter >= elementsToFailure) {
throw new Exception("Task Failure @ elem: " + elementCounter + " / " + elementsToFailure);
}
}
Example #19
Source File: KafkaProducerTestBase.java From flink with Apache License 2.0 | 5 votes |
@Override
public void invoke(Integer value) throws Exception {
valuesPerPartition[value]++;
boolean missing = false;
for (int i : valuesPerPartition) {
if (i < 100) {
missing = true;
break;
}
}
if (!missing) {
throw new SuccessException();
}
}
Example #20
Source File: KafkaTableTestBase.java From flink with Apache License 2.0 | 5 votes |
@Override
public void invoke(RowData value, Context context) throws Exception {
rows.add(value.toString());
if (rows.size() >= expectedSize) {
// job finish
throw new SuccessException();
}
}
Example #21
Source File: KafkaTableTestBase.java From flink with Apache License 2.0 | 5 votes |
protected static boolean isCausedByJobFinished(Throwable e) {
if (e instanceof SuccessException) {
return true;
} else if (e.getCause() != null) {
return isCausedByJobFinished(e.getCause());
} else {
return false;
}
}
Example #22
Source File: CheckpointingCustomKvStateProgram.java From flink with Apache License 2.0 | 5 votes |
@Override
public void flatMap(Tuple2<Integer, Integer> value, Collector<Integer> out) throws Exception {
kvState.add(value.f1);
if (atLeastOneSnapshotComplete) {
if (restored) {
throw new SuccessException();
} else {
throw new RuntimeException("Intended failure, to trigger restore");
}
}
}
Example #23
Source File: ValidatingSink.java From flink with Apache License 2.0 | 5 votes |
@Override
public void open(Configuration parameters) throws Exception {
// this sink can only work with DOP 1
assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
if (usingProcessingTime && resultChecker.checkResult(windowCounts)) {
throw new SuccessException();
}
}
Example #24
Source File: ValidatingSink.java From flink with Apache License 2.0 | 5 votes |
@Override
public void close() {
if (resultChecker.checkResult(windowCounts)) {
if (usingProcessingTime) {
throw new SuccessException();
}
} else {
throw new AssertionError("Test failed check.");
}
}
Example #25
Source File: ValidatingSink.java From flink with Apache License 2.0 | 5 votes |
@Override
public void invoke(T value, Context context) throws Exception {
countUpdater.updateCount(value, windowCounts);
if (usingProcessingTime && resultChecker.checkResult(windowCounts)) {
throw new SuccessException();
}
}
Example #26
Source File: StreamFaultToleranceTestBase.java From flink with Apache License 2.0 | 5 votes |
/**
* Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)}
* followed by the checks in {@link #postSubmit}.
*/
@Test
public void runCheckpointedProgram() throws Exception {
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARALLELISM);
env.enableCheckpointing(500);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));
testProgram(env);
JobGraph jobGraph = env.getStreamGraph().getJobGraph();
try {
ClientUtils.submitJobAndWaitForResult(cluster.getClusterClient(), jobGraph, getClass().getClassLoader()).getJobExecutionResult();
} catch (ProgramInvocationException root) {
Throwable cause = root.getCause();
// search for nested SuccessExceptions
int depth = 0;
while (!(cause instanceof SuccessException)) {
if (cause == null || depth++ == 20) {
root.printStackTrace();
fail("Test failed: " + root.getMessage());
}
else {
cause = cause.getCause();
}
}
}
postSubmit();
}
catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
Example #27
Source File: ValidatingSink.java From flink with Apache License 2.0 | 5 votes |
@Override
public void close() {
if (resultChecker.checkResult(windowCounts)) {
if (usingProcessingTime) {
throw new SuccessException();
}
} else {
throw new AssertionError("Test failed check.");
}
}
Example #28
Source File: KafkaProducerTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override
public void invoke(Integer value) throws Exception {
valuesPerPartition[value]++;
boolean missing = false;
for (int i : valuesPerPartition) {
if (i < 100) {
missing = true;
break;
}
}
if (!missing) {
throw new SuccessException();
}
}
Example #29
Source File: CheckpointingCustomKvStateProgram.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override
public void flatMap(Tuple2<Integer, Integer> value, Collector<Integer> out) throws Exception {
kvState.add(value.f1);
if (atLeastOneSnapshotComplete) {
if (restored) {
throw new SuccessException();
} else {
throw new RuntimeException("Intended failure, to trigger restore");
}
}
}
Example #30
Source File: ValidatingSink.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override
public void open(Configuration parameters) throws Exception {
// this sink can only work with DOP 1
assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
if (usingProcessingTime && resultChecker.checkResult(windowCounts)) {
throw new SuccessException();
}
}