org.apache.flink.streaming.runtime.tasks.StreamTask Java Examples

@Override
public final void initializeState(StreamTaskStateInitializer streamTaskStateManager) throws Exception {

	final TypeSerializer<?> keySerializer = config.getStateKeySerializer(getUserCodeClassloader());

	final StreamTask<?, ?> containingTask =
		Preconditions.checkNotNull(getContainingTask());
	final CloseableRegistry streamTaskCloseableRegistry =
		Preconditions.checkNotNull(containingTask.getCancelables());

	final StreamOperatorStateContext context =
		streamTaskStateManager.streamOperatorStateContext(
			getOperatorID(),
			getClass().getSimpleName(),
			getProcessingTimeService(),
			this,
			keySerializer,
			streamTaskCloseableRegistry,
			metrics);

	stateHandler = new StreamOperatorStateHandler(context, getExecutionConfig(), streamTaskCloseableRegistry);
	timeServiceManager = context.internalTimerServiceManager();
	stateHandler.initializeOperatorState(this);
	runtimeContext.setKeyedStateStore(stateHandler.getKeyedStateStore().orElse(null));
}
 

@Override
@SuppressWarnings("unchecked")
public <T extends StreamOperator<E>> T createStreamOperator(
    StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<E>> output) {
  final TypeSerializer<E> serializer =
      config.getTypeSerializerIn1(containingTask.getUserCodeClassLoader());

  final long totalMemoryUsedForFeedbackCheckpointing =
      config
          .getConfiguration()
          .getInteger(FeedbackConfiguration.TOTAL_MEMORY_USED_FOR_FEEDBACK_CHECKPOINTING);

  FeedbackUnionOperator<E> op =
      new FeedbackUnionOperator<>(
          feedbackKey,
          isBarrierMessage,
          keySelector,
          totalMemoryUsedForFeedbackCheckpointing,
          serializer,
          mailboxExecutor);

  op.setup(containingTask, config, output);

  return (T) op;
}
 

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::setup");
	super.setup(containingTask, config, output);
	if (simulateCheckpointing) {
		testCheckpointer = new Thread() {
			@Override
			public void run() {
				try {
					runStarted.await();
					if (getContainingTask().isCanceled() || getContainingTask().triggerCheckpoint(
							new CheckpointMetaData(0, System.currentTimeMillis()),
							CheckpointOptions.forCheckpointWithDefaultLocation(),
							false)) {
						LifecycleTrackingStreamSource.runFinish.trigger();
					}
				} catch (Exception e) {
					e.printStackTrace();
					Assert.fail();
				}
			}
		};
		testCheckpointer.start();
	}
}
 

@Override
@SuppressWarnings("unchecked")
public <T extends StreamOperator<E>> T createStreamOperator(
    StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<E>> output) {
  final TypeSerializer<E> serializer =
      config.getTypeSerializerIn1(containingTask.getUserCodeClassLoader());

  FeedbackUnionOperator<E> op =
      new FeedbackUnionOperator<>(
          feedbackKey,
          isBarrierMessage,
          keySelector,
          configuration.getFeedbackBufferSize().getBytes(),
          serializer,
          mailboxExecutor);

  op.setup(containingTask, config, output);

  return (T) op;
}
 

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::setup");
	super.setup(containingTask, config, output);
	if (simulateCheckpointing) {
		testCheckpointer = new Thread() {
			@Override
			public void run() {
				try {
					runStarted.await();
					if (getContainingTask().isCanceled() || getContainingTask().triggerCheckpoint(
							new CheckpointMetaData(0, System.currentTimeMillis()),
							CheckpointOptions.forCheckpointWithDefaultLocation())) {
						LifecycleTrackingStreamSource.runFinish.trigger();
					}
				} catch (Exception e) {
					e.printStackTrace();
					Assert.fail();
				}
			}
		};
		testCheckpointer.start();
	}
}
 

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
	super.setup(containingTask, config, output);

	this.inStreamElementSerializer = new StreamElementSerializer<>(
		getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader()));

	switch (outputMode) {
		case ORDERED:
			queue = new OrderedStreamElementQueue<>(capacity);
			break;
		case UNORDERED:
			queue = new UnorderedStreamElementQueue<>(capacity);
			break;
		default:
			throw new IllegalStateException("Unknown async mode: " + outputMode + '.');
	}

	this.timestampedCollector = new TimestampedCollector<>(output);
}
 

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::setup");
	super.setup(containingTask, config, output);
	if (simulateCheckpointing) {
		testCheckpointer = new Thread() {
			@Override
			public void run() {
				try {
					runStarted.await();
					if (getContainingTask().isCanceled() || getContainingTask().triggerCheckpointAsync(
							new CheckpointMetaData(0, System.currentTimeMillis()),
							CheckpointOptions.forCheckpointWithDefaultLocation(),
							false).get()) {
						LifecycleTrackingStreamSource.runFinish.trigger();
					}
				} catch (Exception e) {
					e.printStackTrace();
					Assert.fail();
				}
			}
		};
		testCheckpointer.start();
	}
}
 

/**
 * Checks that the state snapshot context is closed after a successful snapshot operation.
 */
@Test
public void testSnapshotMethod() throws Exception {
	final long checkpointId = 42L;
	final long timestamp = 1L;

	final CloseableRegistry closeableRegistry = new CloseableRegistry();

	StateSnapshotContextSynchronousImpl context = spy(new StateSnapshotContextSynchronousImpl(0L, 0L));

	whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context);

	StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class);
	when(containingTask.getCancelables()).thenReturn(closeableRegistry);

	AbstractStreamOperator<Void> operator = mock(AbstractStreamOperator.class);
	when(operator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenCallRealMethod();
	doReturn(containingTask).when(operator).getContainingTask();

	operator.snapshotState(
			checkpointId,
			timestamp,
			CheckpointOptions.forCheckpointWithDefaultLocation(),
			new MemCheckpointStreamFactory(Integer.MAX_VALUE));

}
 

/**
 * Checks that the state snapshot context is closed after a successful snapshot operation.
 */
@Test
public void testSnapshotMethod() throws Exception {
	final long checkpointId = 42L;
	final long timestamp = 1L;

	final CloseableRegistry closeableRegistry = new CloseableRegistry();

	StateSnapshotContextSynchronousImpl context = spy(new StateSnapshotContextSynchronousImpl(0L, 0L));

	whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context);

	StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class);
	when(containingTask.getCancelables()).thenReturn(closeableRegistry);

	AbstractStreamOperator<Void> operator = mock(AbstractStreamOperator.class);
	when(operator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenCallRealMethod();
	doReturn(containingTask).when(operator).getContainingTask();

	operator.snapshotState(
			checkpointId,
			timestamp,
			CheckpointOptions.forCheckpointWithDefaultLocation(),
			new MemCheckpointStreamFactory(Integer.MAX_VALUE));

	verify(context).close();
}
 

public StreamOperatorParameters(
		StreamTask<?, ?> containingTask,
		StreamConfig config,
		Output<StreamRecord<OUT>> output,
		Supplier<ProcessingTimeService> processingTimeServiceFactory,
		OperatorEventDispatcher operatorEventDispatcher) {
	this.containingTask = containingTask;
	this.config = config;
	this.output = output;
	this.processingTimeServiceFactory = processingTimeServiceFactory;
	this.operatorEventDispatcher = operatorEventDispatcher;
}
 

/**
 * Tests that the created StateSnapshotContextSynchronousImpl is closed in case of a failing
 * Operator#snapshotState(StateSnapshotContextSynchronousImpl) call.
 */
@Test
public void testFailingSnapshotMethod() throws Exception {
	final long checkpointId = 42L;
	final long timestamp = 1L;

	final Exception failingException = new Exception("Test exception");

	final CloseableRegistry closeableRegistry = new CloseableRegistry();

	StateSnapshotContextSynchronousImpl context = mock(StateSnapshotContextSynchronousImpl.class);

	whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context);

	StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class);
	when(containingTask.getCancelables()).thenReturn(closeableRegistry);

	AbstractStreamOperator<Void> operator = mock(AbstractStreamOperator.class);
	when(operator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenCallRealMethod();
	doReturn(containingTask).when(operator).getContainingTask();

	// lets fail when calling the actual snapshotState method
	doThrow(failingException).when(operator).snapshotState(eq(context));

	try {
		operator.snapshotState(
				checkpointId,
				timestamp,
				CheckpointOptions.forCheckpointWithDefaultLocation(),
				new MemCheckpointStreamFactory(Integer.MAX_VALUE));
		fail("Exception expected.");
	} catch (Exception e) {
		assertEquals(failingException, e.getCause());
	}

	verify(context).close();
}
 

@Test
public void testOpenCloseAndTimestamps() {
	// first one spawns thread
	timeService.registerTimer(System.currentTimeMillis(), timestamp -> {});

	assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
 

@Before
public void setup() throws Exception {
	testHarness = startTestHarness();

	StreamTask<?, ?> task = testHarness.getTask();
	timeService = task.getProcessingTimeServiceFactory().createProcessingTimeService(
		task.getMailboxExecutorFactory().createExecutor(testHarness.getStreamConfig().getChainIndex()));
}
 

@SuppressWarnings("unchecked")
@Override
public <T extends StreamOperator<OUT>> T createStreamOperator(StreamTask<?, ?> containingTask,
		StreamConfig config, Output<StreamRecord<OUT>> output) {
	return (T) generatedClass.newInstance(containingTask.getUserCodeClassLoader(),
			generatedClass.getReferences(), containingTask, config, output);
}
 

public static CheckpointBarrierHandler createCheckpointBarrierHandler(
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		IOManager ioManager,
		InputGate inputGate,
		Configuration taskManagerConfig) throws IOException {

	CheckpointBarrierHandler barrierHandler;
	if (checkpointMode == CheckpointingMode.EXACTLY_ONCE) {
		long maxAlign = taskManagerConfig.getLong(TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT);
		if (!(maxAlign == -1 || maxAlign > 0)) {
			throw new IllegalConfigurationException(
				TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT.key()
				+ " must be positive or -1 (infinite)");
		}

		if (taskManagerConfig.getBoolean(TaskManagerOptions.NETWORK_CREDIT_MODEL)) {
			barrierHandler = new BarrierBuffer(inputGate, new CachedBufferBlocker(inputGate.getPageSize()), maxAlign);
		} else {
			barrierHandler = new BarrierBuffer(inputGate, new BufferSpiller(ioManager, inputGate.getPageSize()), maxAlign);
		}
	} else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
		barrierHandler = new BarrierTracker(inputGate);
	} else {
		throw new IllegalArgumentException("Unrecognized Checkpointing Mode: " + checkpointMode);
	}

	if (checkpointedTask != null) {
		barrierHandler.registerCheckpointEventHandler(checkpointedTask);
	}

	return barrierHandler;
}
 

private void stopTestHarness(StreamTaskTestHarness<?> testHarness, long timeout) throws Exception {
	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// thread needs to die in time
	long deadline = System.currentTimeMillis() + timeout;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
 

private <IN, OT extends StreamOperator<IN>> StreamTask<IN, OT> createMockTask(
		StreamConfig streamConfig,
		Environment environment) throws Exception {

	//noinspection unchecked
	return new MockStreamTaskBuilder(environment)
		.setConfig(streamConfig)
		.setExecutionConfig(new ExecutionConfig().enableObjectReuse())
		.build();
}
 

private <IN, OT extends StreamOperator<IN>> StreamTask<IN, OT> createMockTask(
		StreamConfig streamConfig,
		Environment environment) throws Exception {

	//noinspection unchecked
	return new MockStreamTaskBuilder(environment)
		.setConfig(streamConfig)
		.setExecutionConfig(new ExecutionConfig().enableObjectReuse())
		.build();
}
 

public static void waitTaskIsRunning(StreamTask<?, ?> task, CompletableFuture<Void> taskInvocation) throws InterruptedException {
	while (!task.isRunning()) {
		if (taskInvocation.isDone()) {
			fail("Task has stopped");
		}
		Thread.sleep(10L);
	}
}
 

/**
 * Creates a new operator using a factory and makes sure that all special factory traits are properly handled.
 *
 * @param operatorFactory the operator factory.
 * @param containingTask the containing task.
 * @param configuration the configuration of the operator.
 * @param output the output of the operator.
 * @param operatorEventDispatcher the operator event dispatcher for communication between operator and coordinators.
 * @return a newly created and configured operator, and the {@link ProcessingTimeService} instance it can access.
 */
public static <OUT, OP extends StreamOperator<OUT>> Tuple2<OP, Optional<ProcessingTimeService>> createOperator(
		StreamOperatorFactory<OUT> operatorFactory,
		StreamTask<OUT, ?> containingTask,
		StreamConfig configuration,
		Output<StreamRecord<OUT>> output,
		OperatorEventDispatcher operatorEventDispatcher) {

	MailboxExecutor mailboxExecutor = containingTask.getMailboxExecutorFactory().createExecutor(configuration.getChainIndex());

	if (operatorFactory instanceof YieldingOperatorFactory) {
		((YieldingOperatorFactory<?>) operatorFactory).setMailboxExecutor(mailboxExecutor);
	}

	final Supplier<ProcessingTimeService> processingTimeServiceFactory =
			() -> containingTask.getProcessingTimeServiceFactory().createProcessingTimeService(mailboxExecutor);

	final ProcessingTimeService processingTimeService;
	if (operatorFactory instanceof ProcessingTimeServiceAware) {
		processingTimeService = processingTimeServiceFactory.get();
		((ProcessingTimeServiceAware) operatorFactory).setProcessingTimeService(processingTimeService);
	} else {
		processingTimeService = null;
	}

	// TODO: what to do with ProcessingTimeServiceAware?
	OP op = operatorFactory.createStreamOperator(
		new StreamOperatorParameters<>(
			containingTask,
			configuration,
			output,
			processingTimeService != null ? () -> processingTimeService : processingTimeServiceFactory,
			operatorEventDispatcher));
	return new Tuple2<>(op, Optional.ofNullable(processingTimeService));
}
 

@SuppressWarnings("unchecked")
@Override
public <T extends StreamOperator<OUT>> T createStreamOperator(StreamTask<?, ?> containingTask,
		StreamConfig config, Output<StreamRecord<OUT>> output) {
	if (operator instanceof SetupableStreamOperator) {
		((SetupableStreamOperator) operator).setup(containingTask, config, output);
	}
	return (T) operator;
}
 

@Override
public void setup(
    StreamTask<?, ?> containingTask,
    StreamConfig config,
    Output<StreamRecord<WindowedValue<OutputT>>> output) {

  // make sure that FileSystems is initialized correctly
  FileSystems.setDefaultPipelineOptions(serializedOptions.get());

  super.setup(containingTask, config, output);
}
 

/**
 * Tests that the created StateSnapshotContextSynchronousImpl is closed in case of a failing
 * Operator#snapshotState(StateSnapshotContextSynchronousImpl) call.
 */
@Test
public void testFailingSnapshotMethod() throws Exception {
	final long checkpointId = 42L;
	final long timestamp = 1L;

	final Exception failingException = new Exception("Test exception");

	final CloseableRegistry closeableRegistry = new CloseableRegistry();

	StateSnapshotContextSynchronousImpl context = mock(StateSnapshotContextSynchronousImpl.class);

	whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context);

	StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class);
	when(containingTask.getCancelables()).thenReturn(closeableRegistry);

	AbstractStreamOperator<Void> operator = mock(AbstractStreamOperator.class);
	when(operator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenCallRealMethod();
	doReturn(containingTask).when(operator).getContainingTask();

	// lets fail when calling the actual snapshotState method
	doThrow(failingException).when(operator).snapshotState(eq(context));

	try {
		operator.snapshotState(
				checkpointId,
				timestamp,
				CheckpointOptions.forCheckpointWithDefaultLocation(),
				new MemCheckpointStreamFactory(Integer.MAX_VALUE));
		fail("Exception expected.");
	} catch (Exception e) {
		assertEquals(failingException, e.getCause());
	}
}
 

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
    super.setup(containingTask, config, output);
    if (priorityQueue == null) {
        priorityQueue = new PriorityQueue<>(INITIAL_PRIORITY_QUEUE_CAPACITY, new StreamRecordComparator<IN>());
    }
    startSiddhiRuntime();
}
 

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
    super.setup(containingTask, config, output);
    if (priorityQueue == null) {
        priorityQueue = new PriorityQueue<>(INITIAL_PRIORITY_QUEUE_CAPACITY, new StreamRecordComparator<IN>());
    }
    if (siddhiRuntimeHandlers == null) {
        siddhiRuntimeHandlers = new ConcurrentHashMap<>();
    }
}
 

@Override
@SuppressWarnings("unchecked")
public <T extends StreamOperator<Message>> T createStreamOperator(
    StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<Message>> output) {

  FunctionGroupOperator fn =
      new FunctionGroupOperator(
          sideOutputs, configuration, mailboxExecutor, ChainingStrategy.ALWAYS);
  fn.setup(containingTask, config, output);

  return (T) fn;
}
 

@Override
@SuppressWarnings("unchecked")
public <T extends StreamOperator<Message>> T createStreamOperator(
    StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<Message>> output) {

  FunctionGroupOperator fn =
      new FunctionGroupOperator(sideOutputs, mailboxExecutor, ChainingStrategy.ALWAYS);
  fn.setup(containingTask, config, output);

  return (T) fn;
}
 

private void registerFeedbackConsumer(Executor mailboxExecutor) {
  final SubtaskFeedbackKey<T> key =
      feedbackKey.withSubTaskIndex(getRuntimeContext().getIndexOfThisSubtask());
  final StreamTask<?, ?> containingTask = getContainingTask();

  FeedbackChannelBroker broker = FeedbackChannelBroker.get();
  FeedbackChannel<T> channel = broker.getChannel(key);
  channel.registerConsumer(this, containingTask.getCheckpointLock(), mailboxExecutor);
}
 

public static void waitTaskIsRunning(StreamTask<?, ?> task, CompletableFuture<Void> taskInvocation) throws InterruptedException {
	while (!task.isRunning()) {
		if (taskInvocation.isDone()) {
			fail("Task has stopped");
		}
		Thread.sleep(10L);
	}
}
 

private <IN, OT extends StreamOperator<IN>> StreamTask<IN, OT> createMockTask(
		StreamConfig streamConfig,
		Environment environment) throws Exception {

	//noinspection unchecked
	return new MockStreamTaskBuilder(environment)
		.setConfig(streamConfig)
		.setExecutionConfig(new ExecutionConfig().enableObjectReuse())
		.build();
}