Java Code Examples for org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups#createUnregisteredTaskMetricGroup()

@Test
public void testReturnsUnsigned() throws Throwable {
	RocksDBResource localRocksDBResource = new RocksDBResource();
	localRocksDBResource.before();

	SimpleMetricRegistry registry = new SimpleMetricRegistry();
	GenericMetricGroup group = new GenericMetricGroup(
		registry,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		OPERATOR_NAME
	);

	RocksDBNativeMetricOptions options = new RocksDBNativeMetricOptions();
	options.enableSizeAllMemTables();

	RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
		options,
		group,
		localRocksDBResource.getRocksDB()
	);

	ColumnFamilyHandle handle = rocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
	monitor.registerColumnFamily(COLUMN_FAMILY_NAME, handle);
	RocksDBNativeMetricMonitor.RocksDBNativeMetricView view = registry.metrics.get(0);

	view.setValue(-1);
	BigInteger result = view.getValue();

	localRocksDBResource.after();

	Assert.assertEquals("Failed to interpret RocksDB result as an unsigned long", 1, result.signum());
}
 

@Test
public void testReturnsUnsigned() throws Throwable {
	RocksDBResource localRocksDBResource = new RocksDBResource();
	localRocksDBResource.before();

	SimpleMetricRegistry registry = new SimpleMetricRegistry();
	GenericMetricGroup group = new GenericMetricGroup(
		registry,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		OPERATOR_NAME
	);

	RocksDBNativeMetricOptions options = new RocksDBNativeMetricOptions();
	options.enableSizeAllMemTables();

	RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
		options,
		group,
		localRocksDBResource.getRocksDB()
	);

	ColumnFamilyHandle handle = rocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
	monitor.registerColumnFamily(COLUMN_FAMILY_NAME, handle);
	RocksDBNativeMetricMonitor.RocksDBNativeMetricView view = registry.metrics.get(0);

	view.setValue(-1);
	BigInteger result = view.getValue();

	localRocksDBResource.after();

	Assert.assertEquals("Failed to interpret RocksDB result as an unsigned long", 1, result.signum());
}
 

@Test
public void testClosedGaugesDontRead() {
	SimpleMetricRegistry registry = new SimpleMetricRegistry();
	GenericMetricGroup group = new GenericMetricGroup(
		registry,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		OPERATOR_NAME
	);

	RocksDBNativeMetricOptions options = new RocksDBNativeMetricOptions();
	options.enableSizeAllMemTables();

	RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
		options,
		group,
		rocksDBResource.getRocksDB()
	);

	ColumnFamilyHandle handle = rocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
	monitor.registerColumnFamily(COLUMN_FAMILY_NAME, handle);

	RocksDBNativeMetricMonitor.RocksDBNativeMetricView view = registry.metrics.get(0);

	view.close();
	view.update();

	Assert.assertEquals("Closed gauge still queried RocksDB", BigInteger.ZERO, view.getValue());
}
 

@Test
public void testClosedGaugesDontRead() {
	SimpleMetricRegistry registry = new SimpleMetricRegistry();
	GenericMetricGroup group = new GenericMetricGroup(
		registry,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		OPERATOR_NAME
	);

	RocksDBNativeMetricOptions options = new RocksDBNativeMetricOptions();
	options.enableSizeAllMemTables();

	RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
		options,
		group,
		rocksDBResource.getRocksDB()
	);

	ColumnFamilyHandle handle = rocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
	monitor.registerColumnFamily(COLUMN_FAMILY_NAME, handle);

	RocksDBNativeMetricMonitor.RocksDBNativeMetricView view = registry.metrics.get(0);

	view.close();
	view.update();

	Assert.assertEquals("Closed gauge still queried RocksDB", BigInteger.ZERO, view.getValue());
}
 

public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

@Test
public void testMetricMonitorLifecycle() throws Throwable {
	//We use a local variable here to manually control the life-cycle.
	// This allows us to verify that metrics do not try to access
	// RocksDB after the monitor was closed.
	RocksDBResource localRocksDBResource = new RocksDBResource();
	localRocksDBResource.before();

	SimpleMetricRegistry registry = new SimpleMetricRegistry();
	GenericMetricGroup group = new GenericMetricGroup(
		registry,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		OPERATOR_NAME
	);

	RocksDBNativeMetricOptions options = new RocksDBNativeMetricOptions();
	// always returns a non-zero
	// value since empty memtables
	// have overhead.
	options.enableSizeAllMemTables();

	RocksDBNativeMetricMonitor monitor = new RocksDBNativeMetricMonitor(
		options,
		group,
		localRocksDBResource.getRocksDB()
	);

	ColumnFamilyHandle handle = localRocksDBResource.createNewColumnFamily(COLUMN_FAMILY_NAME);
	monitor.registerColumnFamily(COLUMN_FAMILY_NAME, handle);

	Assert.assertEquals("Failed to register metrics for column family", 1, registry.metrics.size());

	RocksDBNativeMetricMonitor.RocksDBNativeMetricView view = registry.metrics.get(0);

	view.update();

	Assert.assertNotEquals("Failed to pull metric from RocksDB", BigInteger.ZERO, view.getValue());

	view.setValue(0L);

	//After the monitor is closed no metric should be accessing RocksDB anymore.
	//If they do, then this test will likely fail with a segmentation fault.
	monitor.close();

	localRocksDBResource.after();

	view.update();

	Assert.assertEquals("Failed to release RocksDB reference", BigInteger.ZERO, view.getValue());
}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		TestingClassLoaderLease.newBuilder().build(),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
			new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"Job Name",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"Test Task",
			1,
			1,
			invokableClass.getName(),
			new Configuration());

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			mock(CheckpointResponder.class),
			new TestGlobalAggregateManager(),
			blobService,
			libCache,
			mock(FileCache.class),
			new TestingTaskManagerRuntimeInfo(),
			taskMetricGroup,
			consumableNotifier,
			partitionProducerStateChecker,
			executor);
}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

@Override
public TaskMetricGroup getMetricGroup() {
	return UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(new TestUserCodeClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

public Task build() throws Exception {
	final JobID jobId = new JobID();
	final JobVertexID jobVertexId = new JobVertexID();
	final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		new Configuration());

	final BlobCacheService blobCacheService = new BlobCacheService(
		mock(PermanentBlobCache.class),
		mock(TransientBlobCache.class));

	final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		new AllocationID(),
		0,
		0,
		resultPartitions,
		inputGates,
		0,
		new MemoryManager(1024 * 1024, 1),
		mock(IOManager.class),
		shuffleEnvironment,
		kvStateService,
		new BroadcastVariableManager(),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new TestGlobalAggregateManager(),
		blobCacheService,
		libraryCacheManager,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new NoOpTaskOperatorEventGateway(),
			new TestGlobalAggregateManager(),
			TestingClassLoaderLease.newBuilder().build(),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				VoidPermanentBlobService.INSTANCE),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	final TestingClassLoaderLease classLoaderHandle = TestingClassLoaderLease.newBuilder()
		.setGetOrResolveClassLoaderFunction((permanentBlobKeys, urls) -> new TestUserCodeClassLoader())
		.build();

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		classLoaderHandle,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build();
				final IOManager ioManager = new IOManagerAsync();

				final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

				final Configuration copiedConf = new Configuration(taskManagerConfig);
				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration
					.fromConfiguration(
						taskManagerConfig,
						TaskExecutorResourceUtils.resourceSpecFromConfigForLocalExecution(copiedConf),
						InetAddress.getLoopbackAddress().getHostAddress());

				final Executor executor = Executors.newCachedThreadPool();

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						shuffleEnvironment,
						new KvStateService(new KvStateRegistry(), null, null),
						new BroadcastVariableManager(),
						new TaskEventDispatcher(),
						ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						NoOpCheckpointResponder.INSTANCE,
						new NoOpTaskOperatorEventGateway(),
						new TestGlobalAggregateManager(),
						TestingClassLoaderLease.newBuilder().build(),
						new FileCache(tmInfo.getTmpDirectories(), VoidPermanentBlobService.INSTANCE),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 

public Task build() throws Exception {
	final JobVertexID jobVertexId = new JobVertexID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig);

	final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		allocationID,
		0,
		0,
		resultPartitions,
		inputGates,
		0,
		MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build(),
		mock(IOManager.class),
		shuffleEnvironment,
		kvStateService,
		new BroadcastVariableManager(),
		new TaskEventDispatcher(),
		externalResourceInfoProvider,
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		classLoaderHandle,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	final NetworkEnvironment networkEnv = mock(NetworkEnvironment.class);
	when(networkEnv.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnv.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		networkEnv,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = new MemoryManager(1024 * 1024, 1);
				final IOManager ioManager = new IOManagerAsync();

				final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
				when(networkEnvironment.createKvStateTaskRegistry(jid, jobVertexId)).thenReturn(mock(TaskKvStateRegistry.class));
				TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
				when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration.fromConfiguration(taskManagerConfig);

				final Executor executor = Executors.newCachedThreadPool();

				BlobCacheService blobService =
					new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						networkEnvironment,
						new BroadcastVariableManager(),
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						new NoOpCheckpointResponder(),
						new TestGlobalAggregateManager(),
						blobService,
						new BlobLibraryCacheManager(
							blobService.getPermanentBlobService(),
							FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
							new String[0]),
						new FileCache(tmInfo.getTmpDirectories(), blobService.getPermanentBlobService()),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 

@Override
public TaskMetricGroup getMetricGroup() {
	return UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
}