org.apache.flink.runtime.operators.testutils.ExpectedTestException Java Examples

@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Test
public void testFailingReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockFailingReduceStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException eetex) {
		// Good!
	} catch (Exception e) {
		LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
		Assert.fail("Test caused exception: " + e.getMessage());
	}
	
	this.outList.clear();
}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int valCnt = 0;
	
	for (Record next : records) {
		element = next;
		valCnt++;
	}
	
	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}
	
	element.getField(0, this.key);
	this.value.setValue(valCnt - this.key.getValue());
	element.setField(1, this.value);
	out.collect(element);
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Override
public void combine(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int sum = 0;

	for (Record next : records) {
		element = next;
		element.getField(1, this.combineValue);

		sum += this.combineValue.getValue();
	}

	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}

	this.combineValue.setValue(sum);
	element.setField(1, this.combineValue);
	out.collect(element);
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Override
public void combine(Iterable<Tuple2<Integer, Integer>> records, Collector<Tuple2<Integer, Integer>> out) {
	int key = 0;
	int sum = 0;

	for (Tuple2<Integer, Integer> next : records) {
		key = next.f0;
		sum += next.f1;
	}
	
	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}

	int resultValue = sum - key;
	out.collect(new Tuple2<>(key, resultValue));
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int valCnt = 0;
	
	for (Record next : records) {
		element = next;
		valCnt++;
	}
	
	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}
	
	element.getField(0, this.key);
	this.value.setValue(valCnt - this.key.getValue());
	element.setField(1, this.value);
	out.collect(element);
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 

private void testFailToConfirmCheckpointMessage(Consumer<StreamTask<?, ?>> consumer) throws Exception {
	StreamMap<Integer, Integer> streamMap = new StreamMap<>(new FailOnNotifyCheckpointMapper<>());
	MultipleInputStreamTaskTestHarnessBuilder<Integer> builder =
		new MultipleInputStreamTaskTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.INT_TYPE_INFO)
			.addInput(BasicTypeInfo.INT_TYPE_INFO);
	StreamTaskMailboxTestHarness<Integer> harness = builder
		.setupOutputForSingletonOperatorChain(streamMap)
		.build();

	try {
		consumer.accept(harness.streamTask);
		harness.streamTask.runMailboxStep();
		fail();
	} catch (ExpectedTestException expected) {
		// expected exception
	}
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Test
public void testFailingReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockFailingReduceStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException eetex) {
		// Good!
	} catch (Exception e) {
		LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
		Assert.fail("Test caused exception: " + e.getMessage());
	}
	
	this.outList.clear();
}
 

@Override
public void combine(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int sum = 0;

	for (Record next : records) {
		element = next;
		element.getField(1, this.combineValue);

		sum += this.combineValue.getValue();
	}

	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}

	this.combineValue.setValue(sum);
	element.setField(1, this.combineValue);
	out.collect(element);
}
 

@Override
public void combine(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int sum = 0;

	for (Record next : records) {
		element = next;
		element.getField(1, this.combineValue);

		sum += this.combineValue.getValue();
	}

	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}

	this.combineValue.setValue(sum);
	element.setField(1, this.combineValue);
	out.collect(element);
}
 

@Test
public void testTempLibFolderDeletedOnFail() throws Exception {
	PowerMockito.spy(RocksDB.class);
	PowerMockito.when(RocksDB.class, "loadLibrary").thenThrow(new ExpectedTestException());

	File tempFolder = temporaryFolder.newFolder();
	try {
		RocksDBStateBackend.ensureRocksDBIsLoaded(tempFolder.getAbsolutePath());
		fail("Not throwing expected exception.");
	} catch (IOException ignored) {
		// ignored
	}
	File[] files = tempFolder.listFiles();
	Assert.assertNotNull(files);
	Assert.assertEquals(0, files.length);
}
 

@Override
public void combine(Iterable<Tuple2<Integer, Integer>> records, Collector<Tuple2<Integer, Integer>> out) {
	int key = 0;
	int sum = 0;

	for (Tuple2<Integer, Integer> next : records) {
		key = next.f0;
		sum += next.f1;
	}
	
	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}

	int resultValue = sum - key;
	out.collect(new Tuple2<>(key, resultValue));
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Override
public void reduce(Iterable<Record> records, Collector<Record> out) {
	Record element = null;
	int valCnt = 0;
	
	for (Record next : records) {
		element = next;
		valCnt++;
	}
	
	if (++this.cnt >= 10) {
		throw new ExpectedTestException();
	}
	
	element.getField(0, this.key);
	this.value.setValue(valCnt - this.key.getValue());
	element.setField(1, this.value);
	out.collect(element);
}
 

@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
 

@Test
public void testFailingReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
	try {
		testDriver(testTask, MockFailingReduceStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException eetex) {
		// Good!
	} catch (Exception e) {
		LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
		Assert.fail("Test caused exception: " + e.getMessage());
	}
	
	this.outList.clear();
}
 

@Test
public void testCleanUpExceptionSuppressing() throws Exception {
	OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
		OneInputStreamTask::new,
		BasicTypeInfo.STRING_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setStreamOperator(new FailingTwiceOperator());
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<>("Doesn't matter", 0));

	try {
		testHarness.waitForTaskCompletion();
	}
	catch (Exception ex) {
		// make sure the original exception is the cause and not wrapped
		if (!(ex.getCause() instanceof ExpectedTestException)) {
			throw ex;
		}
		// make sure DisposeException is the only suppressed exception
		if (ex.getCause().getSuppressed().length != 1) {
			throw ex;
		}
		if (!(ex.getCause().getSuppressed()[0] instanceof FailingTwiceOperator.DisposeException)) {
			throw ex;
		}
	}
}
 

@Override
public RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot(
	long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
	if (checkpointId == DECLINE_CHECKPOINT_ID) {
		return ExceptionallyDoneFuture.of(new ExpectedTestException());
	} else {
		return DoneFuture.of(SnapshotResult.empty());
	}
}
 

private FileInputFormat<String> failingFormat() {
	return new FileInputFormat<String>() {
		@Override
		public boolean reachedEnd() {
			return false;
		}

		@Override
		public String nextRecord(String reuse) {
			throw new ExpectedTestException();
		}

		@Override
		public void open(FileInputSplit fileSplit) {
			throw new ExpectedTestException();
		}

		@Override
		public void close() {
			throw new ExpectedTestException();
		}

		@Override
		public void configure(Configuration parameters) {
		}
	};
}
 

@Override
public void cancel() {
	if (throwOnCancel) {
		throw new ExpectedTestException();
	}
	cancelled = true;
}
 

@Test
public void testFailingSortCoGroupTask() {
	int keyCnt1 = 100;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 1;
	
	setOutput(this.output);
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		testDriver(testTask, MockFailingCoGroupStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
}
 

@Override
public void coGroup(Iterable<Record> records1, Iterable<Record> records2, Collector<Record> out) {
	int val1Cnt = 0;
	
	for (@SuppressWarnings("unused") Record r : records1) { 
		val1Cnt++;
	}
	
	for (Record record2 : records2) { 
		if (val1Cnt == 0) {
			
			if(++this.cnt>=10) {
				throw new ExpectedTestException();
			}
			
			out.collect(record2);
		} else {
			for (int i=0; i<val1Cnt; i++) {
				
				if(++this.cnt>=10) {
					throw new ExpectedTestException();
				}
				
				out.collect(record2);
			}
		}
	}
}
 

@Test
public void testDecliningCheckpointStreamOperator() throws Exception {
	DeclineDummyEnvironment declineDummyEnvironment = new DeclineDummyEnvironment();

	// mock the returned snapshots
	OperatorSnapshotFutures operatorSnapshotResult1 = mock(OperatorSnapshotFutures.class);
	OperatorSnapshotFutures operatorSnapshotResult2 = mock(OperatorSnapshotFutures.class);

	final Exception testException = new ExpectedTestException();

	RunningTask<MockStreamTask> task = runTask(() -> createMockStreamTask(
		declineDummyEnvironment,
		operatorChain(
			streamOperatorWithSnapshotException(testException),
			streamOperatorWithSnapshot(operatorSnapshotResult1),
			streamOperatorWithSnapshot(operatorSnapshotResult2)
			)));
	MockStreamTask streamTask = task.streamTask;

	waitTaskIsRunning(streamTask, task.invocationFuture);

	streamTask.triggerCheckpointAsync(
		new CheckpointMetaData(42L, 1L),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		false);

	try {
		task.waitForTaskCompletion(false);
	}
	catch (Exception ex) {
		if (!ExceptionUtils.findThrowable(ex, ExpectedTestException.class).isPresent()) {
			throw ex;
		}
	}

	verify(operatorSnapshotResult1).cancel();
	verify(operatorSnapshotResult2).cancel();
}
 

@Test
public void testFailingHashFirstMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockFailingMatchStub.class, 3);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}