Java Code Examples for org.apache.flink.core.testutils.BlockerSync#releaseBlocker()

@Test
public void testSlowIODoesNotBlockRelease() throws Exception {
	BlockerSync sync = new BlockerSync();
	ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {
		@Override
		public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
			sync.blockNonInterruptible();
			super.releasePartition(partitionId, cause);
		}
	};

	NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder()
		.setResultPartitionManager(blockingResultPartitionManager)
		.setIoExecutor(Executors.newFixedThreadPool(1))
		.build();

	shuffleEnvironment.releasePartitionsLocally(Collections.singleton(new ResultPartitionID()));
	sync.awaitBlocker();
	sync.releaseBlocker();
}
 

@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 

@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 

@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 

@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

@Test
public void testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor() throws Exception {
	BlockerSync sync = new BlockerSync();
	ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {
		@Override
		public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
			sync.blockNonInterruptible();
			super.releasePartition(partitionId, cause);
		}
	};

	NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder()
		.setResultPartitionManager(blockingResultPartitionManager)
		.setIoExecutor(TEST_EXECUTOR_SERVICE_RESOURCE.getExecutor())
		.build();

	final CompletableFuture<ResultPartitionID> startTrackingFuture = new CompletableFuture<>();
	final TaskExecutorPartitionTracker partitionTracker = new TaskExecutorPartitionTrackerImpl(shuffleEnvironment) {
		@Override
		public void startTrackingPartition(JobID producingJobId, TaskExecutorPartitionInfo partitionInfo) {
			super.startTrackingPartition(producingJobId, partitionInfo);
			startTrackingFuture.complete(partitionInfo.getResultPartitionId());
		}
	};

	try {
		internalTestPartitionRelease(
			partitionTracker,
			shuffleEnvironment,
			startTrackingFuture,
			(jobId, resultPartitionDeploymentDescriptor, taskExecutor, taskExecutorGateway) -> {
				final IntermediateDataSetID dataSetId = resultPartitionDeploymentDescriptor.getResultId();

				taskExecutorGateway.releaseClusterPartitions(Collections.singleton(dataSetId), timeout);

				// execute some operation to check whether the TaskExecutor is blocked
				taskExecutorGateway.canBeReleased().get(5, TimeUnit.SECONDS);
			}
		);
	} finally {
		sync.releaseBlocker();
	}
}