org.apache.flink.runtime.state.internal.InternalListState Java Examples

/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend = RocksDBTestUtils
		.builderForTestDefaults(temporaryFolder.getRoot(), LongSerializer.INSTANCE)
		.build();

	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 

/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 

/**
 * Verifies that the serialization of a list using the given list state
 * matches the deserialization with {@link KvStateSerializer#deserializeList}.
 *
 * @param key
 * 		key of the list state
 * @param listState
 * 		list state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
 *
 * @throws Exception
 */
public static void testListSerialization(
		final long key,
		final InternalListState<Long, VoidNamespace, Long> listState) throws Exception {

	TypeSerializer<Long> valueSerializer = LongSerializer.INSTANCE;
	listState.setCurrentNamespace(VoidNamespace.INSTANCE);

	// List
	final int numElements = 10;

	final List<Long> expectedValues = new ArrayList<>();
	for (int i = 0; i < numElements; i++) {
		final long value = ThreadLocalRandom.current().nextLong();
		expectedValues.add(value);
		listState.add(value);
	}

	final byte[] serializedKey =
		KvStateSerializer.serializeKeyAndNamespace(
			key, LongSerializer.INSTANCE,
			VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);

	final byte[] serializedValues = listState.getSerializedValue(
			serializedKey,
			listState.getKeySerializer(),
			listState.getNamespaceSerializer(),
			listState.getValueSerializer());

	List<Long> actualValues = KvStateSerializer.deserializeList(serializedValues, valueSerializer);
	assertEquals(expectedValues, actualValues);

	// Single value
	long expectedValue = ThreadLocalRandom.current().nextLong();
	byte[] serializedValue = KvStateSerializer.serializeValue(expectedValue, valueSerializer);
	List<Long> actualValue = KvStateSerializer.deserializeList(serializedValue, valueSerializer);
	assertEquals(1, actualValue.size());
	assertEquals(expectedValue, actualValue.get(0).longValue());
}
 

/**
 * Tests list serialization utils.
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(
		VoidNamespaceSerializer.INSTANCE,
		new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	testListSerialization(key, listState);
}
 

@Override
public void open() throws Exception {
	super.open();

	evictorContext = new EvictorContext(null, null);
	evictingWindowState = (InternalListState<K, W, StreamRecord<IN>>)
			getOrCreateKeyedState(windowSerializer, evictingWindowStateDescriptor);
}
 

@Override
public <N, V> StateListView<N, V> getStateListView(String stateName, ListViewTypeInfo<V> listViewTypeInfo) throws Exception {
	ListStateDescriptor<V> listStateDesc = new ListStateDescriptor<>(
		stateName,
		listViewTypeInfo.getElementType());

	ListState<V> listState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, listStateDesc);
	// explict cast to internal state
	InternalListState<?, N, V> internalListState = (InternalListState<?, N, V>) listState;

	return new StateListView.NamespacedStateListView<>(internalListState);
}
 

private InternalListState<String, Long, Message> delayedMessagesBufferState(
    ListStateDescriptor<Message> delayedMessageStateDescriptor) {
  try {
    KeyedStateBackend<String> keyedStateBackend = getKeyedStateBackend();
    return (InternalListState<String, Long, Message>)
        keyedStateBackend.getOrCreateKeyedState(
            LongSerializer.INSTANCE, delayedMessageStateDescriptor);
  } catch (Exception e) {
    throw new RuntimeException("Error registered Flink state for delayed messages buffer.", e);
  }
}
 

private InternalListState<String, Long, Message> delayedMessagesBufferState(
    ListStateDescriptor<Message> delayedMessageStateDescriptor) {
  try {
    KeyedStateBackend<String> keyedStateBackend = getKeyedStateBackend();
    return (InternalListState<String, Long, Message>)
        keyedStateBackend.getOrCreateKeyedState(
            LongSerializer.INSTANCE, delayedMessageStateDescriptor);
  } catch (Exception e) {
    throw new RuntimeException("Error registered Flink state for delayed messages buffer.", e);
  }
}
 

/**
 * Verifies that the serialization of a list using the given list state
 * matches the deserialization with {@link KvStateSerializer#deserializeList}.
 *
 * @param key
 * 		key of the list state
 * @param listState
 * 		list state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
 *
 * @throws Exception
 */
public static void testListSerialization(
		final long key,
		final InternalListState<Long, VoidNamespace, Long> listState) throws Exception {

	TypeSerializer<Long> valueSerializer = LongSerializer.INSTANCE;
	listState.setCurrentNamespace(VoidNamespace.INSTANCE);

	// List
	final int numElements = 10;

	final List<Long> expectedValues = new ArrayList<>();
	for (int i = 0; i < numElements; i++) {
		final long value = ThreadLocalRandom.current().nextLong();
		expectedValues.add(value);
		listState.add(value);
	}

	final byte[] serializedKey =
		KvStateSerializer.serializeKeyAndNamespace(
			key, LongSerializer.INSTANCE,
			VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);

	final byte[] serializedValues = listState.getSerializedValue(
			serializedKey,
			listState.getKeySerializer(),
			listState.getNamespaceSerializer(),
			listState.getValueSerializer());

	List<Long> actualValues = KvStateSerializer.deserializeList(serializedValues, valueSerializer);
	assertEquals(expectedValues, actualValues);

	// Single value
	long expectedValue = ThreadLocalRandom.current().nextLong();
	byte[] serializedValue = KvStateSerializer.serializeValue(expectedValue, valueSerializer);
	List<Long> actualValue = KvStateSerializer.deserializeList(serializedValue, valueSerializer);
	assertEquals(1, actualValue.size());
	assertEquals(expectedValue, actualValue.get(0).longValue());
}
 

/**
 * Tests list serialization utils.
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(
		VoidNamespaceSerializer.INSTANCE,
		new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	testListSerialization(key, listState);
}
 

@Override
public void open() throws Exception {
	super.open();

	evictorContext = new EvictorContext(null, null);
	evictingWindowState = (InternalListState<K, W, StreamRecord<IN>>)
			getOrCreateKeyedState(windowSerializer, evictingWindowStateDescriptor);
}
 

@Override
public void open() throws Exception {
	super.open();

	evictorContext = new EvictorContext(null, null);
	evictingWindowState = (InternalListState<K, W, StreamRecord<IN>>)
			getOrCreateKeyedState(windowSerializer, evictingWindowStateDescriptor);
}
 

@Override
public <N, V> StateListView<N, V> getStateListView(String stateName, ListViewTypeInfo<V> listViewTypeInfo) throws Exception {
	ListStateDescriptor<V> listStateDesc = new ListStateDescriptor<>(
		stateName,
		listViewTypeInfo.getElementType());

	ListState<V> listState = keyedStateBackend.getOrCreateKeyedState(windowSerializer, listStateDesc);
	// explict cast to internal state
	InternalListState<?, N, V> internalListState = (InternalListState<?, N, V>) listState;

	return new StateListView.NamespacedStateListView<>(internalListState);
}
 

/**
 * Tests list serialization utils.
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = getLongHeapKeyedStateBackend(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(
		VoidNamespaceSerializer.INSTANCE,
		new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	testListSerialization(key, listState);
}
 

/**
 * Verifies that the serialization of a list using the given list state
 * matches the deserialization with {@link KvStateSerializer#deserializeList}.
 *
 * @param key
 * 		key of the list state
 * @param listState
 * 		list state using the {@link VoidNamespace}, must also be a {@link InternalKvState} instance
 *
 * @throws Exception
 */
public static void testListSerialization(
		final long key,
		final InternalListState<Long, VoidNamespace, Long> listState) throws Exception {

	TypeSerializer<Long> valueSerializer = LongSerializer.INSTANCE;
	listState.setCurrentNamespace(VoidNamespace.INSTANCE);

	// List
	final int numElements = 10;

	final List<Long> expectedValues = new ArrayList<>();
	for (int i = 0; i < numElements; i++) {
		final long value = ThreadLocalRandom.current().nextLong();
		expectedValues.add(value);
		listState.add(value);
	}

	final byte[] serializedKey =
		KvStateSerializer.serializeKeyAndNamespace(
			key, LongSerializer.INSTANCE,
			VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);

	final byte[] serializedValues = listState.getSerializedValue(
			serializedKey,
			listState.getKeySerializer(),
			listState.getNamespaceSerializer(),
			listState.getValueSerializer());

	List<Long> actualValues = KvStateSerializer.deserializeList(serializedValues, valueSerializer);
	assertEquals(expectedValues, actualValues);

	// Single value
	long expectedValue = ThreadLocalRandom.current().nextLong();
	byte[] serializedValue = KvStateSerializer.serializeValue(expectedValue, valueSerializer);
	List<Long> actualValue = KvStateSerializer.deserializeList(serializedValue, valueSerializer);
	assertEquals(1, actualValue.size());
	assertEquals(expectedValue, actualValue.get(0).longValue());
}
 

/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 

@Inject
FlinkStateDelayedMessagesBuffer(
    @Label("delayed-messages-buffer-state")
        InternalListState<String, Long, Message> bufferState) {
  this.bufferState = Objects.requireNonNull(bufferState);
}
 

/**
 * [FLINK-5979]
 *
 * <p>This test takes a snapshot that was created with Flink 1.2 and tries to restore it in master to check
 * the backwards compatibility of the serialization format of {@link StateTable}s.
 */
@Test
public void testRestore1_2ToMaster() throws Exception {

	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_2.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final Integer namespace1 = 1;
	final Integer namespace2 = 2;
	final Integer namespace3 = 3;

	final KeyGroupsStateHandle stateHandle;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandle = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalListState<String, Integer, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		assertEquals(7, keyedBackend.numKeyValueStateEntries());

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(asList(33L, 55L), state.get());
		state.setCurrentNamespace(namespace2);
		assertEquals(asList(22L, 11L), state.get());
		state.setCurrentNamespace(namespace3);
		assertEquals(Collections.singletonList(44L), state.get());

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(asList(11L, 44L), state.get());

		state.setCurrentNamespace(namespace3);
		assertEquals(asList(22L, 55L, 33L), state.get());

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(asList(11L, 22L, 33L, 44L, 55L), state.get());

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(asList(11L, 22L, 33L, 44L, 55L), state.get());
	}
}
 

@Override
	public void open() throws Exception {
		super.open();

		this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
		timestampedCollector = new TimestampedCollector<>(output);

		internalTimerService =
				getInternalTimerService("window-timers", windowSerializer, this);

		triggerContext = new Context(null, null);
		processContext = new WindowContext(null);

		windowAssignerContext = new WindowAssigner.WindowAssignerContext() {
			@Override
			public long getCurrentProcessingTime() {
				return internalTimerService.currentProcessingTime();
			}
		};

		// create (or restore) the state that hold the actual window contents
		// NOTE - the state may be null in the case of the overriding evicting window operator
		if (windowStateDescriptor != null) {
			windowState = (InternalAppendingState<K, W, IN, ACC, ACC>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
		}

		// create the typed and helper states for merging windows
		if (windowAssigner instanceof MergingWindowAssigner) {

			// store a typed reference for the state of merging windows - sanity check
			if (windowState instanceof InternalMergingState) {
				windowMergingState = (InternalMergingState<K, W, IN, ACC, ACC>) windowState;
			}
			// TODO this sanity check should be here, but is prevented by an incorrect test (pending validation)
			// TODO see WindowOperatorTest.testCleanupTimerWithEmptyFoldingStateForSessionWindows()
			// TODO activate the sanity check once resolved
//			else if (windowState != null) {
//				throw new IllegalStateException(
//						"The window uses a merging assigner, but the window state is not mergeable.");
//			}

			@SuppressWarnings("unchecked")
			final Class<Tuple2<W, W>> typedTuple = (Class<Tuple2<W, W>>) (Class<?>) Tuple2.class;

			final TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>(
					typedTuple,
					new TypeSerializer[] {windowSerializer, windowSerializer});

			final ListStateDescriptor<Tuple2<W, W>> mergingSetsStateDescriptor =
					new ListStateDescriptor<>("merging-window-set", tupleSerializer);

			// get the state that stores the merging sets
			mergingSetsState = (InternalListState<K, VoidNamespace, Tuple2<W, W>>)
					getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, mergingSetsStateDescriptor);
			mergingSetsState.setCurrentNamespace(VoidNamespace.INSTANCE);
		}
	}
 

@Override
	public void open() throws Exception {
		super.open();

		this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
		timestampedCollector = new TimestampedCollector<>(output);

		internalTimerService =
				getInternalTimerService("window-timers", windowSerializer, this);

		triggerContext = new Context(null, null);
		processContext = new WindowContext(null);

		windowAssignerContext = new WindowAssigner.WindowAssignerContext() {
			@Override
			public long getCurrentProcessingTime() {
				return internalTimerService.currentProcessingTime();
			}
		};

		// create (or restore) the state that hold the actual window contents
		// NOTE - the state may be null in the case of the overriding evicting window operator
		if (windowStateDescriptor != null) {
			windowState = (InternalAppendingState<K, W, IN, ACC, ACC>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
		}

		// create the typed and helper states for merging windows
		if (windowAssigner instanceof MergingWindowAssigner) {

			// store a typed reference for the state of merging windows - sanity check
			if (windowState instanceof InternalMergingState) {
				windowMergingState = (InternalMergingState<K, W, IN, ACC, ACC>) windowState;
			}
			// TODO this sanity check should be here, but is prevented by an incorrect test (pending validation)
			// TODO see WindowOperatorTest.testCleanupTimerWithEmptyFoldingStateForSessionWindows()
			// TODO activate the sanity check once resolved
//			else if (windowState != null) {
//				throw new IllegalStateException(
//						"The window uses a merging assigner, but the window state is not mergeable.");
//			}

			@SuppressWarnings("unchecked")
			final Class<Tuple2<W, W>> typedTuple = (Class<Tuple2<W, W>>) (Class<?>) Tuple2.class;

			final TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>(
					typedTuple,
					new TypeSerializer[] {windowSerializer, windowSerializer});

			final ListStateDescriptor<Tuple2<W, W>> mergingSetsStateDescriptor =
					new ListStateDescriptor<>("merging-window-set", tupleSerializer);

			// get the state that stores the merging sets
			mergingSetsState = (InternalListState<K, VoidNamespace, Tuple2<W, W>>)
					getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, mergingSetsStateDescriptor);
			mergingSetsState.setCurrentNamespace(VoidNamespace.INSTANCE);
		}
	}
 

TtlListState(TtlStateContext<InternalListState<K, N, TtlValue<T>>, List<T>> ttlStateContext) {
	super(ttlStateContext);
}
 

public NamespacedStateListView(InternalListState<?, N, T> listState) {
	this.listState = listState;
}
 

/**
 * [FLINK-5979]
 *
 * <p>This test takes a snapshot that was created with Flink 1.2 and tries to restore it in master to check
 * the backwards compatibility of the serialization format of {@link StateTable}s.
 */
@Test
public void testRestore1_2ToMaster() throws Exception {

	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_2.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final Integer namespace1 = 1;
	final Integer namespace2 = 2;
	final Integer namespace3 = 3;

	final KeyGroupsStateHandle stateHandle;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandle = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalListState<String, Integer, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		assertEquals(7, keyedBackend.numKeyValueStateEntries());

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(asList(33L, 55L), state.get());
		state.setCurrentNamespace(namespace2);
		assertEquals(asList(22L, 11L), state.get());
		state.setCurrentNamespace(namespace3);
		assertEquals(Collections.singletonList(44L), state.get());

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(asList(11L, 44L), state.get());

		state.setCurrentNamespace(namespace3);
		assertEquals(asList(22L, 55L, 33L), state.get());

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(asList(11L, 22L, 33L, 44L, 55L), state.get());

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(asList(11L, 22L, 33L, 44L, 55L), state.get());
	}
}
 

static Reductions create(
    BackPressureValve valve,
    StatefulFunctionsUniverse statefulFunctionsUniverse,
    RuntimeContext context,
    KeyedStateBackend<Object> keyedStateBackend,
    TimerServiceFactory timerServiceFactory,
    InternalListState<String, Long, Message> delayedMessagesBufferState,
    Map<EgressIdentifier<?>, OutputTag<Object>> sideOutputs,
    Output<StreamRecord<Message>> output,
    MessageFactory messageFactory,
    Executor mailboxExecutor,
    MetricGroup metricGroup,
    MapState<Long, Message> asyncOperations) {

  ObjectContainer container = new ObjectContainer();

  container.add("function-providers", Map.class, statefulFunctionsUniverse.functions());
  container.add(
      "function-repository", FunctionRepository.class, StatefulFunctionRepository.class);

  // for FlinkState
  container.add("runtime-context", RuntimeContext.class, context);
  container.add("keyed-state-backend", KeyedStateBackend.class, keyedStateBackend);
  container.add(new DynamicallyRegisteredTypes(statefulFunctionsUniverse.types()));
  container.add("state", State.class, FlinkState.class);

  // For reductions
  container.add(messageFactory);

  container.add(
      new Partition(
          context.getMaxNumberOfParallelSubtasks(),
          context.getNumberOfParallelSubtasks(),
          context.getIndexOfThisSubtask()));

  container.add(new RemoteSink(output));
  container.add(new SideOutputSink(sideOutputs, output));

  container.add("applying-context", ApplyingContext.class, ReusableContext.class);
  container.add(LocalSink.class);
  container.add("function-loader", FunctionLoader.class, PredefinedFunctionLoader.class);
  container.add(StateBinder.class);
  container.add(Reductions.class);
  container.add(LocalFunctionGroup.class);
  container.add("metrics-factory", MetricsFactory.class, new FlinkMetricsFactory(metricGroup));

  // for delayed messages
  container.add(
      "delayed-messages-buffer-state", InternalListState.class, delayedMessagesBufferState);
  container.add(
      "delayed-messages-buffer",
      DelayedMessagesBuffer.class,
      FlinkStateDelayedMessagesBuffer.class);
  container.add(
      "delayed-messages-timer-service-factory", TimerServiceFactory.class, timerServiceFactory);
  container.add(DelaySink.class);

  // lazy providers for the sinks
  container.add("function-group", new Lazy<>(LocalFunctionGroup.class));
  container.add("reductions", new Lazy<>(Reductions.class));

  container.add("mailbox-executor", Executor.class, mailboxExecutor);

  // for the async operations
  container.add("async-operations", MapState.class, asyncOperations);
  container.add(AsyncSink.class);
  container.add(PendingAsyncOperations.class);

  container.add("backpressure-valve", BackPressureValve.class, valve);

  return container.get(Reductions.class);
}
 

public NamespacedStateListView(InternalListState<?, N, T> listState) {
	this.listState = listState;
}
 

@Inject
FlinkStateDelayedMessagesBuffer(
    @Label("delayed-messages-buffer-state")
        InternalListState<String, Long, Message> bufferState) {
  this.bufferState = Objects.requireNonNull(bufferState);
}
 

static Reductions create(
    Configuration configuration,
    StatefulFunctionsUniverse statefulFunctionsUniverse,
    RuntimeContext context,
    KeyedStateBackend<Object> keyedStateBackend,
    TimerServiceFactory timerServiceFactory,
    InternalListState<String, Long, Message> delayedMessagesBufferState,
    Map<EgressIdentifier<?>, OutputTag<Object>> sideOutputs,
    Output<StreamRecord<Message>> output,
    MessageFactory messageFactory,
    Executor mailboxExecutor,
    MetricGroup metricGroup,
    MapState<Long, Message> asyncOperations,
    Executor checkpointLockExecutor) {

  ObjectContainer container = new ObjectContainer();

  container.add("function-providers", Map.class, statefulFunctionsUniverse.functions());
  container.add(
      "function-repository", FunctionRepository.class, StatefulFunctionRepository.class);

  // for FlinkState
  container.add("runtime-context", RuntimeContext.class, context);
  container.add("keyed-state-backend", KeyedStateBackend.class, keyedStateBackend);
  container.add(new DynamicallyRegisteredTypes(statefulFunctionsUniverse.types()));

  if (configuration.getBoolean(StatefulFunctionsJobConstants.MULTIPLEX_FLINK_STATE)) {
    container.add("state", State.class, MultiplexedState.class);
  } else {
    container.add("state", State.class, FlinkState.class);
  }

  // For reductions
  container.add(messageFactory);

  container.add(
      new Partition(
          context.getMaxNumberOfParallelSubtasks(),
          context.getNumberOfParallelSubtasks(),
          context.getIndexOfThisSubtask()));

  container.add(new RemoteSink(output));
  container.add(new SideOutputSink(sideOutputs, output));

  container.add("applying-context", ApplyingContext.class, ReusableContext.class);
  container.add(LocalSink.class);
  container.add("function-loader", FunctionLoader.class, PredefinedFunctionLoader.class);
  container.add(StateBinder.class);
  container.add(Reductions.class);
  container.add(LocalFunctionGroup.class);
  container.add("metrics-factory", MetricsFactory.class, new FlinkMetricsFactory(metricGroup));

  // for delayed messages
  container.add(
      "delayed-messages-buffer-state", InternalListState.class, delayedMessagesBufferState);
  container.add(
      "delayed-messages-buffer",
      DelayedMessagesBuffer.class,
      FlinkStateDelayedMessagesBuffer.class);
  container.add(
      "delayed-messages-timer-service-factory", TimerServiceFactory.class, timerServiceFactory);
  container.add(DelaySink.class);

  // lazy providers for the sinks
  container.add("function-group", new Lazy<>(LocalFunctionGroup.class));
  container.add("reductions", new Lazy<>(Reductions.class));

  container.add("mailbox-executor", Executor.class, mailboxExecutor);

  // for the async operations
  container.add("async-operations", MapState.class, asyncOperations);
  container.add("checkpoint-lock-executor", Executor.class, checkpointLockExecutor);
  container.add(AsyncSink.class);

  return container.get(Reductions.class);
}
 

TtlListState(TtlStateContext<InternalListState<K, N, TtlValue<T>>, List<T>> ttlStateContext) {
	super(ttlStateContext);
}
 

TtlListState(TtlStateContext<InternalListState<K, N, TtlValue<T>>, List<T>> ttlStateContext) {
	super(ttlStateContext);
}
 

@Override
	public void open() throws Exception {
		super.open();

		this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
		timestampedCollector = new TimestampedCollector<>(output);

		internalTimerService =
				getInternalTimerService("window-timers", windowSerializer, this);

		triggerContext = new Context(null, null);
		processContext = new WindowContext(null);

		windowAssignerContext = new WindowAssigner.WindowAssignerContext() {
			@Override
			public long getCurrentProcessingTime() {
				return internalTimerService.currentProcessingTime();
			}
		};

		// create (or restore) the state that hold the actual window contents
		// NOTE - the state may be null in the case of the overriding evicting window operator
		if (windowStateDescriptor != null) {
			windowState = (InternalAppendingState<K, W, IN, ACC, ACC>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
		}

		// create the typed and helper states for merging windows
		if (windowAssigner instanceof MergingWindowAssigner) {

			// store a typed reference for the state of merging windows - sanity check
			if (windowState instanceof InternalMergingState) {
				windowMergingState = (InternalMergingState<K, W, IN, ACC, ACC>) windowState;
			}
			// TODO this sanity check should be here, but is prevented by an incorrect test (pending validation)
			// TODO see WindowOperatorTest.testCleanupTimerWithEmptyFoldingStateForSessionWindows()
			// TODO activate the sanity check once resolved
//			else if (windowState != null) {
//				throw new IllegalStateException(
//						"The window uses a merging assigner, but the window state is not mergeable.");
//			}

			@SuppressWarnings("unchecked")
			final Class<Tuple2<W, W>> typedTuple = (Class<Tuple2<W, W>>) (Class<?>) Tuple2.class;

			final TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>(
					typedTuple,
					new TypeSerializer[] {windowSerializer, windowSerializer});

			final ListStateDescriptor<Tuple2<W, W>> mergingSetsStateDescriptor =
					new ListStateDescriptor<>("merging-window-set", tupleSerializer);

			// get the state that stores the merging sets
			mergingSetsState = (InternalListState<K, VoidNamespace, Tuple2<W, W>>)
					getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, mergingSetsStateDescriptor);
			mergingSetsState.setCurrentNamespace(VoidNamespace.INSTANCE);
		}
	}