org.apache.flink.runtime.state.VoidNamespaceSerializer Java Examples

@Override
public void open() throws Exception {
	super.open();

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	collector = new TimestampedCollector<>(output);

	this.broadcastStates = new HashMap<>(broadcastStateDescriptors.size());
	for (MapStateDescriptor<?, ?> descriptor: broadcastStateDescriptors) {
		broadcastStates.put(descriptor, getOperatorStateBackend().getBroadcastState(descriptor));
	}

	rwContext = new ReadWriteContextImpl(getExecutionConfig(), getKeyedStateBackend(), userFunction, broadcastStates, timerService);
	rContext = new ReadOnlyContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
	onTimerContext = new OnTimerContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Override
public void open() throws Exception {
	joinCondition = generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader());
	joinCondition.setRuntimeContext(getRuntimeContext());
	joinCondition.open(new Configuration());

	nextLeftIndex = getRuntimeContext().getState(
		new ValueStateDescriptor<>(NEXT_LEFT_INDEX_STATE_NAME, Types.LONG));
	leftState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(LEFT_STATE_NAME, Types.LONG, leftType));
	rightState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(RIGHT_STATE_NAME, Types.LONG, rightType));
	registeredTimer = getRuntimeContext().getState(
		new ValueStateDescriptor<>(REGISTERED_TIMER_STATE_NAME, Types.LONG));

	timerService = getInternalTimerService(
		TIMERS_STATE_NAME, VoidNamespaceSerializer.INSTANCE, this);
	collector = new TimestampedCollector<>(output);
	outRow = new JoinedRowData();
	// all the output records should be INSERT only,
	// because current temporal join only supports INSERT only left stream
	outRow.setRowKind(RowKind.INSERT);
}
 

@Override
public void open() throws Exception {
	joinCondition = generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader());
	joinCondition.setRuntimeContext(getRuntimeContext());
	joinCondition.open(new Configuration());

	nextLeftIndex = getRuntimeContext().getState(
		new ValueStateDescriptor<>(NEXT_LEFT_INDEX_STATE_NAME, Types.LONG));
	leftState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(LEFT_STATE_NAME, Types.LONG, leftType));
	rightState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(RIGHT_STATE_NAME, Types.LONG, rightType));
	registeredTimer = getRuntimeContext().getState(
		new ValueStateDescriptor<>(REGISTERED_TIMER_STATE_NAME, Types.LONG));

	timerService = getInternalTimerService(
		TIMERS_STATE_NAME, VoidNamespaceSerializer.INSTANCE, this);
	collector = new TimestampedCollector<>(output);
	outRow = new JoinedRow();
	outRow.setHeader(BaseRowUtil.ACCUMULATE_MSG);
}
 

/** Removes the current key from <b>ALL</b> known states in the state backend. */
@Override
public void remove() {
	if (currentKey == null) {
		return;
	}

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		try {
			State state = backend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				descriptor);

			state.clear();
		} catch (Exception e) {
			throw new RuntimeException("Failed to drop partitioned state from state backend", e);
		}
	}
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Override
public void open() throws Exception {
	super.open();
	timerService = getInternalTimerService(
			"watermark-callbacks",
			VoidNamespaceSerializer.INSTANCE,
			this);

	nfa = nfaFactory.createNFA();
	nfa.open(cepRuntimeContext, new Configuration());

	context = new ContextFunctionImpl();
	collector = new TimestampedCollector<>(output);
	cepTimerService = new TimerServiceImpl();

	// metrics
	this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
}
 

/** Removes the current key from <b>ALL</b> known states in the state backend. */
@Override
public void remove() {
	if (currentKey == null) {
		return;
	}

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		try {
			State state = backend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				descriptor);

			state.clear();
		} catch (Exception e) {
			throw new RuntimeException("Failed to drop partitioned state from state backend", e);
		}
	}
}
 

public FlinkWatermarkHoldState(
    KeyedStateBackend<ByteBuffer> flinkStateBackend,
    MapStateDescriptor<String, Instant> watermarkHoldStateDescriptor,
    String stateId,
    StateNamespace namespace,
    TimestampCombiner timestampCombiner) {
  this.timestampCombiner = timestampCombiner;
  // Combines StateNamespace and stateId to generate a unique namespace for
  // watermarkHoldsState. We do not want to use Flink's namespacing to be
  // able to recover watermark holds efficiently during recovery.
  this.namespaceString = namespace.stringKey() + stateId;
  try {
    this.watermarkHoldsState =
        flinkStateBackend.getPartitionedState(
            VoidNamespace.INSTANCE,
            VoidNamespaceSerializer.INSTANCE,
            watermarkHoldStateDescriptor);
  } catch (Exception e) {
    throw new RuntimeException("Could not access state for watermark partition view");
  }
}
 

static void fireExpiredAsyncOperations(
    MapStateDescriptor<Long, Message> asyncOperationStateDescriptor,
    Reductions reductions,
    KeyedStateBackend<String> keyedStateBackend)
    throws Exception {

  AsyncOperationFailureNotifier asyncOperationFailureNotifier =
      new AsyncOperationFailureNotifier(reductions);

  keyedStateBackend.applyToAllKeys(
      VoidNamespace.get(),
      VoidNamespaceSerializer.INSTANCE,
      asyncOperationStateDescriptor,
      asyncOperationFailureNotifier);

  if (asyncOperationFailureNotifier.enqueued()) {
    reductions.processEnvelopes();
  }
}
 

static void fireExpiredAsyncOperations(
    MapStateDescriptor<Long, Message> asyncOperationStateDescriptor,
    Reductions reductions,
    MapState<Long, Message> asyncOperationState,
    KeyedStateBackend<String> keyedStateBackend)
    throws Exception {

  AsyncOperationFailureNotifier asyncOperationFailureNotifier =
      new AsyncOperationFailureNotifier(reductions, asyncOperationState);

  keyedStateBackend.applyToAllKeys(
      VoidNamespace.get(),
      VoidNamespaceSerializer.INSTANCE,
      asyncOperationStateDescriptor,
      asyncOperationFailureNotifier);

  if (asyncOperationFailureNotifier.enqueued()) {
    reductions.processEnvelopes();
  }
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Override
public void open() throws Exception {
	super.open();

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	collector = new TimestampedCollector<>(output);

	this.broadcastStates = new HashMap<>(broadcastStateDescriptors.size());
	for (MapStateDescriptor<?, ?> descriptor: broadcastStateDescriptors) {
		broadcastStates.put(descriptor, getOperatorStateBackend().getBroadcastState(descriptor));
	}

	rwContext = new ReadWriteContextImpl(getExecutionConfig(), getKeyedStateBackend(), userFunction, broadcastStates, timerService);
	rContext = new ReadOnlyContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
	onTimerContext = new OnTimerContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
}
 

@Override
public void open() throws Exception {
	super.open();

	// also get the state in open, this way we are sure that it was created before
	// we trigger the test checkpoint
	ValueState<String> state = getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("count", StringSerializer.INSTANCE));

}
 

@Override
public <VS, S extends State> void applyToKeyedState(
		final StateDescriptor<S, VS> stateDescriptor,
		final KeyedStateFunction<KS, S> function) throws Exception {

	keyedStateBackend.applyToAllKeys(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE,
			Preconditions.checkNotNull(stateDescriptor),
			Preconditions.checkNotNull(function));
}
 

@Test
public void testDisposeDeletesAllDirectories() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	Collection<File> allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
	try {
		ValueStateDescriptor<String> kvId =
			new ValueStateDescriptor<>("id", String.class, null);

		kvId.initializeSerializerUnlessSet(new ExecutionConfig());

		ValueState<String> state =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("Hello");

		// more than just the root directory
		assertTrue(allFilesInDbDir.size() > 1);
	} finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
	allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());

	// just the root directory left
	assertEquals(1, allFilesInDbDir.size());
}
 

@Override
public void processElement(StreamRecord<String> element) throws Exception {
	// we also don't care

	ValueState<String> state = getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("count", StringSerializer.INSTANCE));

	state.update(element.getValue());
}
 

@Override
public void open() throws Exception {
	super.open();
	collector = new TimestampedCollector<>(output);

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	context = new ContextImpl(userFunction, timerService);
	onTimerContext = new OnTimerContextImpl(userFunction, timerService);
}
 

/**
 * Tests to cover case when user misuse optimizeForPointLookup with iterator interfaces on map state.
 *
 * <p>The option {@link ColumnFamilyOptions#optimizeForPointLookup(long)} would lead to iterator.seek with prefix bytes invalid.
 */
@Test
public void testMisuseOptimizePointLookupWithMapState() throws Exception {
	RocksDBStateBackend rocksDBStateBackend = createStateBackendWithOptimizePointLookup();
	RocksDBKeyedStateBackend<Integer> keyedStateBackend =
		createKeyedStateBackend(rocksDBStateBackend, new MockEnvironmentBuilder().build(), IntSerializer.INSTANCE);
	try {
		MapStateDescriptor<Integer, Long> stateDescriptor = new MapStateDescriptor<>("map", IntSerializer.INSTANCE, LongSerializer.INSTANCE);
		MapState<Integer, Long> mapState = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);

		keyedStateBackend.setCurrentKey(1);
		Map<Integer, Long> expectedResult = new HashMap<>();
		for (int i = 0; i < 100; i++) {
			long uv = ThreadLocalRandom.current().nextLong();
			mapState.put(i, uv);
			expectedResult.put(i, uv);
		}

		Iterator<Map.Entry<Integer, Long>> iterator = mapState.entries().iterator();
		while (iterator.hasNext()) {
			Map.Entry<Integer, Long> entry = iterator.next();
			assertEquals(entry.getValue(), expectedResult.remove(entry.getKey()));
			iterator.remove();
		}
		assertTrue(expectedResult.isEmpty());
		assertTrue(mapState.isEmpty());
	} finally {
		keyedStateBackend.dispose();
	}
}
 

@Override
public void open() throws Exception {
	super.open();
	collector = new TimestampedCollector<>(output);

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	context = new ContextImpl<>(userFunction, timerService);
	onTimerContext = new OnTimerContextImpl<>(userFunction, timerService);
}
 

@Override
public void open() throws Exception {
	super.open();

	this.timerService = getInternalTimerService(
			"test-timers",
			VoidNamespaceSerializer.INSTANCE,
			this);
}
 

private KeyedBufferingElementsHandler(
    KeyedStateBackend<Object> backend, ListStateDescriptor<BufferedElement> stateDescriptor)
    throws Exception {
  this.backend = backend;
  this.stateName = stateDescriptor.getName();
  // Eagerly retrieve the state to work around https://jira.apache.org/jira/browse/FLINK-12653
  this.state =
      backend.getPartitionedState(
          VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);
}
 

@Override
public WatermarkHoldState bindWatermark(
    String id, StateSpec<WatermarkHoldState> spec, TimestampCombiner timestampCombiner) {
  try {
    keyedStateBackend.getOrCreateKeyedState(
        VoidNamespaceSerializer.INSTANCE,
        new MapStateDescriptor<>(
            "watermark-holds",
            StringSerializer.INSTANCE,
            new CoderTypeSerializer<>(InstantCoder.of())));
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
  return null;
}
 

@Test
public void testSetTtlTimeProvider() throws Exception {
	AbstractStreamOperator<Integer> operator = new AbstractStreamOperator<Integer>() {};
	try (AbstractStreamOperatorTestHarness<Integer> result = new AbstractStreamOperatorTestHarness<>(
			operator,
			1,
			1,
			0)) {

		result.config.setStateKeySerializer(IntSerializer.INSTANCE);

		Time timeToLive = Time.hours(1);
		result.initializeState(new OperatorSubtaskState());
		result.open();

		ValueStateDescriptor<Integer> stateDescriptor = new ValueStateDescriptor<>("test", IntSerializer.INSTANCE);
		stateDescriptor.enableTimeToLive(StateTtlConfig.newBuilder(timeToLive).build());
		KeyedStateBackend<Integer> keyedStateBackend = operator.getKeyedStateBackend();
		ValueState<Integer> state = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);

		int expectedValue = 42;
		keyedStateBackend.setCurrentKey(1);
		result.setStateTtlProcessingTime(0L);
		state.update(expectedValue);
		Assert.assertEquals(expectedValue, (int) state.value());
		result.setStateTtlProcessingTime(timeToLive.toMilliseconds() + 1);
		Assert.assertNull(state.value());
	}
}
 

@Override
public KeyedStateRow map(Tuple2<K, V> t) throws Exception {
	int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(t.f0, maxParallelism);
	ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos();
	DataOutputViewStreamWrapper ov = new DataOutputViewStreamWrapper(os);

	RocksDBUtils.writeKeyGroup(keyGroup, keygroupPrefixBytes, ov);
	RocksDBUtils.writeKey(t.f0, keySerializer, os, ov, false);
	RocksDBUtils.writeNameSpace(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, os,
			ov, false);

	os.close();
	return new KeyedStateRow(stateName, os.toByteArray(),
			InstantiationUtil.serializeToByteArray(valueSerializer, t.f1));
}
 

/**
 * Defines/redefines a value state with the given name and type. This can be
 * used to create new states of an operator or change the type of an already
 * existing state.
 * <p>
 * When redefining a pre-existing state make sure you haven't added that as
 * keyed state rows before.
 * 
 * @param stateName
 * @param newState
 * @param valueSerializer
 */
public <K, V> void createNewValueState(String stateName, DataSet<Tuple2<K, V>> newState,
		TypeSerializer<V> valueSerializer) {

	metaSnapshots.put(stateName, new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, stateName,
			VoidNamespaceSerializer.INSTANCE, valueSerializer).snapshot());

	updateProxy();
	addKeyedStateRows(newState
			.map(new ValueStateToKeyedStateRow<K, V>(stateName,
					getKeySerializer(),
					valueSerializer,
					baseOpState.getMaxParallelism())));
}
 

@Override
public InternalTimerService<VoidNamespace> createTimerService(
    Triggerable<String, VoidNamespace> triggerable) {
  final TimerSerializer<String, VoidNamespace> timerSerializer =
      new TimerSerializer<>(StringSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE);

  return timeServiceManager.getInternalTimerService(
      DELAYED_MSG_TIMER_SERVICE_NAME, timerSerializer, triggerable);
}
 

@Override
public InternalTimerService<VoidNamespace> createTimerService(
    Triggerable<String, VoidNamespace> triggerable) {
  final TimerSerializer<String, VoidNamespace> timerSerializer =
      new TimerSerializer<>(StringSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE);

  return timeServiceManager.getInternalTimerService(
      DELAYED_MSG_TIMER_SERVICE_NAME, timerSerializer, triggerable);
}
 

@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createMapPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<MapState<Integer, String>>() {

		@Override
		public MapState<Integer, String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			MapStateDescriptor<Integer, String> descr =
					(MapStateDescriptor<Integer, String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(MapStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}
 

@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createListPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<ListState<String>>() {

		@Override
		public ListState<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			ListStateDescriptor<String> descr =
					(ListStateDescriptor<String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(ListStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}