org.apache.flink.api.common.state.BroadcastState Java Examples

private Path validateStateAndTransform(Savepoint savepoint) throws IOException, Exception {
	ExecutionEnvironment environment = ExecutionEnvironment.createLocalEnvironment();

	// Validate the contents of the broadcast state
	OperatorStateReader reader = new OperatorStateReader(environment, savepoint, "stateful");
	OperatorStateBackend backend = reader.createOperatorStateBackendFromSnapshot(1);
	BroadcastState<Boolean, List<Integer>> state = backend.getBroadcastState(bcstate);
	assertEquals(Lists.newArrayList(1), state.get(true));

	Path newCheckpointBasePath = new Path(getCheckpointDir(), "new");
	OperatorStateWriter writer = new OperatorStateWriter(savepoint, "stateful", newCheckpointBasePath);
	writer.transformNonKeyedState((i, b) -> {
		try {
			b.getBroadcastState(bcstate).put(true, Lists.newArrayList(2, 3));
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	});

	StateMetadataUtils.writeSavepointMetadata(newCheckpointBasePath,
			StateMetadataUtils.createNewSavepoint(savepoint, writer.writeAll()));
	return newCheckpointBasePath;
}
 

ReadWriteContextImpl (
		final ExecutionConfig executionConfig,
		final KeyedStateBackend<KS> keyedStateBackend,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.keyedStateBackend = Preconditions.checkNotNull(keyedStateBackend);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

ReadWriteContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals(new TestType("foo", 13), state.get(3));
		assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
 

private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
 

ReadWriteContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadWriteContextImpl (
		final ExecutionConfig executionConfig,
		final KeyedStateBackend<KS> keyedStateBackend,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.keyedStateBackend = Preconditions.checkNotNull(keyedStateBackend);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

OnTimerContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
 

private void validateSecondSavepoint(Savepoint savepoint) throws Exception {
	// Validate the contents of the broadcast state
	OperatorStateReader reader = new OperatorStateReader(ExecutionEnvironment.createLocalEnvironment(), savepoint,
			"stateful");
	OperatorStateBackend backend = reader.createOperatorStateBackendFromSnapshot(1);
	BroadcastState<Boolean, List<Integer>> state = backend.getBroadcastState(bcstate);
	assertEquals(Lists.newArrayList(2, 3), state.get(true));
}
 

@Override
public void processBroadcastElement(Map<String, String> value, Context context, Collector<MetricEvent> collector) throws Exception {
    if (value != null) {
        BroadcastState<String, String> broadcastState = context.getBroadcastState(alarmRulesMapStateDescriptor);
        for (Map.Entry<String, String> entry : value.entrySet()) {
            broadcastState.put(entry.getKey(), entry.getValue());
        }
    }
}
 

private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals(new TestType("foo", 13), state.get(3));
		assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> descriptor) {
	try {
		return getOperatorStateBackend().getBroadcastState(descriptor);
	} catch (Exception e) {
		throw new FlinkRuntimeException(e);
	}
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> descriptor) {
	try {
		return getOperatorStateBackend().getBroadcastState(descriptor);
	} catch (Exception e) {
		throw new FlinkRuntimeException(e);
	}
}
 

private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals(new TestType("foo", 13), state.get(3));
		Assert.assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
 

private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		Assert.assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
 

ReadWriteContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final BroadcastProcessFunction<IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final ProcessingTimeService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

ReadWriteContextImpl (
		final ExecutionConfig executionConfig,
		final KeyedStateBackend<KS> keyedStateBackend,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.keyedStateBackend = Preconditions.checkNotNull(keyedStateBackend);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

@Override
public <K, V> BroadcastState<K, V> getBroadcastState(MapStateDescriptor<K, V> stateDescriptor) {
	Preconditions.checkNotNull(stateDescriptor);

	stateDescriptor.initializeSerializerUnlessSet(config);
	BroadcastState<K, V> state = (BroadcastState<K, V>) states.get(stateDescriptor);
	if (state == null) {
		throw new IllegalArgumentException("The requested state does not exist. " +
				"Check for typos in your state descriptor, or specify the state descriptor " +
				"in the datastream.broadcast(...) call if you forgot to register it.");
	}
	return state;
}
 

ReadOnlyContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

OnTimerContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}
 

OnTimerContextImpl(
		final ExecutionConfig executionConfig,
		final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function,
		final Map<MapStateDescriptor<?, ?>, BroadcastState<?, ?>> broadcastStates,
		final TimerService timerService) {

	function.super();
	this.config = Preconditions.checkNotNull(executionConfig);
	this.states = Preconditions.checkNotNull(broadcastStates);
	this.timerService = Preconditions.checkNotNull(timerService);
}