org.apache.flink.runtime.execution.Environment Java Examples

/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 

/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 

SubtaskCheckpointCoordinatorImpl(
		CheckpointStorageWorkerView checkpointStorage,
		String taskName,
		StreamTaskActionExecutor actionExecutor,
		CloseableRegistry closeableRegistry,
		ExecutorService executorService,
		Environment env,
		AsyncExceptionHandler asyncExceptionHandler,
		boolean unalignedCheckpointEnabled,
		BiFunctionWithException<ChannelStateWriter, Long, CompletableFuture<Void>, IOException> prepareInputSnapshot) throws IOException {
	this(checkpointStorage,
		taskName,
		actionExecutor,
		closeableRegistry,
		executorService,
		env,
		asyncExceptionHandler,
		unalignedCheckpointEnabled,
		prepareInputSnapshot,
		DEFAULT_MAX_RECORD_ABORTED_CHECKPOINTS);
}
 

/**
 * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(0);

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 

private static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>();
	List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
	Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

	for (int i = 0; i < outEdgesInOrder.size(); i++) {
		StreamEdge edge = outEdgesInOrder.get(i);
		recordWriters.add(
			createRecordWriter(
				edge,
				i,
				environment,
				environment.getTaskInfo().getTaskName(),
				chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
	}
	return recordWriters;
}
 

private <K> AbstractKeyedStateBackend<K> restoreKeyedBackend(
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	List<KeyedStateHandle> state,
	Environment env) throws Exception {
	AbstractKeyedStateBackend<K> backend = getStateBackend().createKeyedStateBackend(
		env,
		new JobID(),
		"test_op",
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		env.getTaskKvStateRegistry(),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		state,
		new CloseableRegistry());
	return backend;
}
 

private StreamOperatorStateContext getStreamOperatorStateContext(Environment environment) throws IOException {
	StreamTaskStateInitializer initializer = new StreamTaskStateInitializerImpl(
		environment,
		stateBackend);

	try {
		return initializer.streamOperatorStateContext(
			operatorState.getOperatorID(),
			operatorState.getOperatorID().toString(),
			new NeverFireProcessingTimeService(),
			operator,
			operator.getKeyType().createSerializer(environment.getExecutionConfig()),
			registry,
			getRuntimeContext().getMetricGroup());
	} catch (Exception e) {
		throw new IOException("Failed to restore state backend", e);
	}
}
 

@Override
public OperatorStateBackend createOperatorStateBackend(
	Environment env,
	String operatorIdentifier,
	@Nonnull Collection<OperatorStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws Exception {

	//the default for RocksDB; eventually there can be a operator state backend based on RocksDB, too.
	final boolean asyncSnapshots = true;
	return new DefaultOperatorStateBackendBuilder(
		env.getUserClassLoader(),
		env.getExecutionConfig(),
		asyncSnapshots,
		stateHandles,
		cancelStreamRegistry).build();
}
 

/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and specified localRecoveryConfig.
 */
public OneInputStreamTaskTestHarness(
		Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
		int numInputGates,
		int numInputChannelsPerGate,
		TypeInformation<IN> inputType,
		TypeInformation<OUT> outputType,
		LocalRecoveryConfig localRecoveryConfig) {

	super(taskFactory, outputType, localRecoveryConfig);

	this.inputType = inputType;
	inputSerializer = inputType.createSerializer(executionConfig);

	this.numInputGates = numInputGates;
	this.numInputChannelsPerGate = numInputChannelsPerGate;
}
 

protected <K> AbstractKeyedStateBackend<K> createKeyedBackend(
		TypeSerializer<K> keySerializer,
		int numberOfKeyGroups,
		KeyGroupRange keyGroupRange,
		Environment env) throws Exception {

	AbstractKeyedStateBackend<K> backend = getStateBackend().createKeyedStateBackend(
		env,
		new JobID(),
		"test_op",
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		env.getTaskKvStateRegistry(),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());

	return backend;
}
 

public StreamTaskStateInitializerImpl(
	Environment environment,
	StateBackend stateBackend,
	ProcessingTimeService processingTimeService) {

	this.environment = environment;
	this.taskStateManager = Preconditions.checkNotNull(environment.getTaskStateManager());
	this.stateBackend = Preconditions.checkNotNull(stateBackend);
	this.processingTimeService = processingTimeService;
}
 

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {
	throw new SuccessException();
}
 

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws Exception {

	return backend.createKeyedStateBackend(
		env,
		jobID,
		operatorIdentifier,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		kvStateRegistry,
		this.ttlTimeProvider,
		metricGroup,
		stateHandles,
		cancelStreamRegistry);
}
 

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws Exception {

	return backend.createKeyedStateBackend(
		env,
		jobID,
		operatorIdentifier,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		kvStateRegistry,
		this.ttlTimeProvider,
		metricGroup,
		stateHandles,
		cancelStreamRegistry);
}
 

@Override
public OperatorStateBackend createOperatorStateBackend(
	Environment env,
	String operatorIdentifier,
	@Nonnull Collection<OperatorStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws Exception {

	return new DefaultOperatorStateBackendBuilder(
		env.getUserClassLoader(),
		env.getExecutionConfig(),
		isUsingAsynchronousSnapshots(),
		stateHandles,
		cancelStreamRegistry).build();
}
 

@Override
public OperatorStateBackend createOperatorStateBackend(
	Environment env,
	String operatorIdentifier,
	@Nonnull Collection<OperatorStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws Exception {

	return new DefaultOperatorStateBackendBuilder(
		env.getUserClassLoader(),
		env.getExecutionConfig(),
		isUsingAsynchronousSnapshots(),
		stateHandles,
		cancelStreamRegistry).build();
}
 

@Override
public OperatorStateBackend createOperatorStateBackend(
	Environment env,
	String operatorIdentifier,
	@Nonnull Collection<OperatorStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	throw new UnsupportedOperationException();
}
 

protected StreamTaskStateInitializer createStreamTaskStateManager(
	Environment env,
	StateBackend stateBackend,
	TtlTimeProvider ttlTimeProvider) {
	return new StreamTaskStateInitializerImpl(
		env,
		stateBackend,
		ttlTimeProvider);
}
 

/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and local recovery disabled.
 */
public OneInputStreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	int numInputGates,
	int numInputChannelsPerGate,
	TypeInformation<IN> inputType,
	TypeInformation<OUT> outputType) {
	this(taskFactory, numInputGates, numInputChannelsPerGate, inputType, outputType, TestLocalRecoveryConfig.disabled());
}
 

/**
 * This test checks the behavior for basic setting of local DB directories.
 */
@Test
public void testSetDbPath() throws Exception {
	final RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(tempFolder.newFolder().toURI().toString());

	final String testDir1 = tempFolder.newFolder().getAbsolutePath();
	final String testDir2 = tempFolder.newFolder().getAbsolutePath();

	assertNull(rocksDbBackend.getDbStoragePaths());

	rocksDbBackend.setDbStoragePath(testDir1);
	assertArrayEquals(new String[] { testDir1 }, rocksDbBackend.getDbStoragePaths());

	rocksDbBackend.setDbStoragePath(null);
	assertNull(rocksDbBackend.getDbStoragePaths());

	rocksDbBackend.setDbStoragePaths(testDir1, testDir2);
	assertArrayEquals(new String[] { testDir1, testDir2 }, rocksDbBackend.getDbStoragePaths());

	final Environment env = getMockEnvironment(tempFolder.newFolder());
	final RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(rocksDbBackend, env);

	try {
		File instanceBasePath = keyedBackend.getInstanceBasePath();
		assertThat(instanceBasePath.getAbsolutePath(), anyOf(startsWith(testDir1), startsWith(testDir2)));

		//noinspection NullArgumentToVariableArgMethod
		rocksDbBackend.setDbStoragePaths(null);
		assertNull(rocksDbBackend.getDbStoragePaths());
	}
	finally {
		IOUtils.closeQuietly(keyedBackend);
		keyedBackend.dispose();
	}
}
 

public SometimesInstantiationErrorSender(Environment environment) {
	super(environment);

	if (failingSenders.contains(this.getIndexInSubtaskGroup())) {
		throw new RuntimeException("Test exception in constructor");
	}
}
 

@VisibleForTesting
public StreamTaskStateInitializerImpl(
	Environment environment,
	StateBackend stateBackend,
	TtlTimeProvider ttlTimeProvider) {

	this.environment = environment;
	this.taskStateManager = Preconditions.checkNotNull(environment.getTaskStateManager());
	this.stateBackend = Preconditions.checkNotNull(stateBackend);
	this.ttlTimeProvider = ttlTimeProvider;
}
 

private <K> AbstractKeyedStateBackend<K> createKeyedBackend(TypeSerializer<K> keySerializer, Environment env) throws Exception {
	return createKeyedBackend(
		keySerializer,
		10,
		new KeyGroupRange(0, 9),
		env);
}
 

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
 

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return null;
}
 

@Override
public OperatorStateBackend createOperatorStateBackend(
	Environment env,
	String operatorIdentifier,
	@Nonnull Collection<OperatorStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws Exception {

	return new DefaultOperatorStateBackendBuilder(
		env.getUserClassLoader(),
		env.getExecutionConfig(),
		isUsingAsynchronousSnapshots(),
		stateHandles,
		cancelStreamRegistry).build();
}
 

@Test
public void testCheckConcurrencyProblemWhenPerformingCheckpointAsync() throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();
	Environment env = new DummyEnvironment();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);

	ExecutorService executorService = Executors.newScheduledThreadPool(1);
	try {
		long checkpointID = 0;
		List<Future> futureList = new ArrayList();
		for (int i = 0; i < 10; ++i) {
			ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>("id" + i, IntSerializer.INSTANCE);
			ValueState<Integer> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
			((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
			backend.setCurrentKey(i);
			state.update(i);

			futureList.add(runSnapshotAsync(executorService,
				backend.snapshot(checkpointID++, System.currentTimeMillis(), streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())));
		}

		for (Future future : futureList) {
			future.get(20, TimeUnit.SECONDS);
		}
	} catch (Exception e) {
		fail();
	} finally {
		backend.dispose();
		executorService.shutdown();
	}
}
 

@Override
public OperatorStateBackend createOperatorStateBackend(
	Environment env,
	String operatorIdentifier,
	@Nonnull Collection<OperatorStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws Exception {
	return backend.createOperatorStateBackend(env, operatorIdentifier, stateHandles, cancelStreamRegistry);
}
 

@Override
public OperatorStateBackend createOperatorStateBackend(
	Environment env,
	String operatorIdentifier,
	@Nonnull Collection<OperatorStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	return new DefaultOperatorStateBackendBuilder(
		env.getUserClassLoader(),
		env.getExecutionConfig(),
		isUsingAsynchronousSnapshots(),
		stateHandles,
		cancelStreamRegistry).build();
}
 

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return null;
}