Java Code Examples for org.apache.flink.api.common.state.BroadcastState#put()

The following examples show how to use org.apache.flink.api.common.state.BroadcastState#put() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals(new TestType("foo", 13), state.get(3));
		Assert.assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
Example 2
Source File: StateBackendMigrationTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		Assert.assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
Example 3
Source File: MyBroadcastProcessFunction.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
@Override
public void processBroadcastElement(Map<String, String> value, Context context, Collector<MetricEvent> collector) throws Exception {
    if (value != null) {
        BroadcastState<String, String> broadcastState = context.getBroadcastState(alarmRulesMapStateDescriptor);
        for (Map.Entry<String, String> entry : value.entrySet()) {
            broadcastState.put(entry.getKey(), entry.getValue());
        }
    }
}
Example 4
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals(new TestType("foo", 13), state.get(3));
		assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
Example 5
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
Example 6
Source File: MyBroadcastProcessFunction.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
@Override
public void processBroadcastElement(Map<String, String> value, Context context, Collector<MetricEvent> collector) throws Exception {
    if (value != null) {
        BroadcastState<String, String> broadcastState = context.getBroadcastState(alarmRulesMapStateDescriptor);
        for (Map.Entry<String, String> entry : value.entrySet()) {
            broadcastState.put(entry.getKey(), entry.getValue());
        }
    }
}
Example 7
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals(new TestType("foo", 13), state.get(3));
		assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
Example 8
Source File: StateBackendMigrationTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
Example 9
Source File: OperatorStateBackendTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Test
public void testCorrectClassLoaderUsedOnSnapshot() throws Exception {

	AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);

	final Environment env = createMockEnvironment();
	CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
	OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(env, "test-op-name", emptyStateHandles, cancelStreamRegistry);

	AtomicInteger copyCounter = new AtomicInteger(0);
	TypeSerializer<Integer> serializer = new VerifyingIntSerializer(env.getUserClassLoader(), copyCounter);

	// write some state
	ListStateDescriptor<Integer> stateDescriptor = new ListStateDescriptor<>("test", serializer);
	ListState<Integer> listState = operatorStateBackend.getListState(stateDescriptor);

	listState.add(42);

	AtomicInteger keyCopyCounter = new AtomicInteger(0);
	AtomicInteger valueCopyCounter = new AtomicInteger(0);

	TypeSerializer<Integer> keySerializer = new VerifyingIntSerializer(env.getUserClassLoader(), keyCopyCounter);
	TypeSerializer<Integer> valueSerializer = new VerifyingIntSerializer(env.getUserClassLoader(), valueCopyCounter);

	MapStateDescriptor<Integer, Integer> broadcastStateDesc = new MapStateDescriptor<>(
			"test-broadcast", keySerializer, valueSerializer);

	BroadcastState<Integer, Integer> broadcastState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
	broadcastState.put(1, 2);
	broadcastState.put(3, 4);
	broadcastState.put(5, 6);

	CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
	RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture =
		operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
	FutureUtils.runIfNotDoneAndGet(runnableFuture);

	// make sure that the copy method has been called
	assertTrue(copyCounter.get() > 0);
	assertTrue(keyCopyCounter.get() > 0);
	assertTrue(valueCopyCounter.get() > 0);
}
Example 10
Source File: OperatorStateBackendTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSnapshotAsyncClose() throws Exception {
	DefaultOperatorStateBackend operatorStateBackend =
		new DefaultOperatorStateBackendBuilder(
			OperatorStateBackendTest.class.getClassLoader(),
			new ExecutionConfig(),
			true,
			emptyStateHandles,
			new CloseableRegistry()).build();

	ListStateDescriptor<MutableType> stateDescriptor1 =
			new ListStateDescriptor<>("test1", new JavaSerializer<MutableType>());

	ListState<MutableType> listState1 = operatorStateBackend.getOperatorState(stateDescriptor1);

	listState1.add(MutableType.of(42));
	listState1.add(MutableType.of(4711));

	MapStateDescriptor<MutableType, MutableType> broadcastStateDescriptor1 =
			new MapStateDescriptor<>("test4", new JavaSerializer<MutableType>(), new JavaSerializer<MutableType>());

	BroadcastState<MutableType, MutableType> broadcastState1 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor1);
	broadcastState1.put(MutableType.of(1), MutableType.of(2));
	broadcastState1.put(MutableType.of(2), MutableType.of(5));

	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);

	OneShotLatch waiterLatch = new OneShotLatch();
	OneShotLatch blockerLatch = new OneShotLatch();

	streamFactory.setWaiterLatch(waiterLatch);
	streamFactory.setBlockerLatch(blockerLatch);

	RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture =
			operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

	ExecutorService executorService = Executors.newFixedThreadPool(1);

	executorService.submit(runnableFuture);

	// wait until the async checkpoint is in the write code, then continue
	waiterLatch.await();

	operatorStateBackend.close();

	blockerLatch.trigger();

	try {
		runnableFuture.get(60, TimeUnit.SECONDS);
		Assert.fail();
	} catch (CancellationException expected) {
	}
}
Example 11
Source File: OperatorStateBackendTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Test
public void testCorrectClassLoaderUsedOnSnapshot() throws Exception {

	AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);

	final Environment env = createMockEnvironment();
	CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
	OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(env, "test-op-name", emptyStateHandles, cancelStreamRegistry);

	AtomicInteger copyCounter = new AtomicInteger(0);
	TypeSerializer<Integer> serializer = new VerifyingIntSerializer(env.getUserClassLoader(), copyCounter);

	// write some state
	ListStateDescriptor<Integer> stateDescriptor = new ListStateDescriptor<>("test", serializer);
	ListState<Integer> listState = operatorStateBackend.getListState(stateDescriptor);

	listState.add(42);

	AtomicInteger keyCopyCounter = new AtomicInteger(0);
	AtomicInteger valueCopyCounter = new AtomicInteger(0);

	TypeSerializer<Integer> keySerializer = new VerifyingIntSerializer(env.getUserClassLoader(), keyCopyCounter);
	TypeSerializer<Integer> valueSerializer = new VerifyingIntSerializer(env.getUserClassLoader(), valueCopyCounter);

	MapStateDescriptor<Integer, Integer> broadcastStateDesc = new MapStateDescriptor<>(
			"test-broadcast", keySerializer, valueSerializer);

	BroadcastState<Integer, Integer> broadcastState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
	broadcastState.put(1, 2);
	broadcastState.put(3, 4);
	broadcastState.put(5, 6);

	CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
	RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture =
		operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
	FutureUtils.runIfNotDoneAndGet(runnableFuture);

	// make sure that the copy method has been called
	assertTrue(copyCounter.get() > 0);
	assertTrue(keyCopyCounter.get() > 0);
	assertTrue(valueCopyCounter.get() > 0);
}
Example 12
Source File: OperatorStateBackendTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSnapshotAsyncClose() throws Exception {
	DefaultOperatorStateBackend operatorStateBackend =
		new DefaultOperatorStateBackendBuilder(
			OperatorStateBackendTest.class.getClassLoader(),
			new ExecutionConfig(),
			true,
			emptyStateHandles,
			new CloseableRegistry()).build();

	ListStateDescriptor<MutableType> stateDescriptor1 =
			new ListStateDescriptor<>("test1", new JavaSerializer<MutableType>());

	ListState<MutableType> listState1 = operatorStateBackend.getOperatorState(stateDescriptor1);

	listState1.add(MutableType.of(42));
	listState1.add(MutableType.of(4711));

	MapStateDescriptor<MutableType, MutableType> broadcastStateDescriptor1 =
			new MapStateDescriptor<>("test4", new JavaSerializer<MutableType>(), new JavaSerializer<MutableType>());

	BroadcastState<MutableType, MutableType> broadcastState1 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor1);
	broadcastState1.put(MutableType.of(1), MutableType.of(2));
	broadcastState1.put(MutableType.of(2), MutableType.of(5));

	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);

	OneShotLatch waiterLatch = new OneShotLatch();
	OneShotLatch blockerLatch = new OneShotLatch();

	streamFactory.setWaiterLatch(waiterLatch);
	streamFactory.setBlockerLatch(blockerLatch);

	RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture =
			operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

	ExecutorService executorService = Executors.newFixedThreadPool(1);

	executorService.submit(runnableFuture);

	// wait until the async checkpoint is in the write code, then continue
	waiterLatch.await();

	operatorStateBackend.close();

	blockerLatch.trigger();

	try {
		runnableFuture.get(60, TimeUnit.SECONDS);
		Assert.fail();
	} catch (CancellationException expected) {
	}
}
Example 13
Source File: OperatorStateBackendTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Test
public void testCorrectClassLoaderUsedOnSnapshot() throws Exception {

	AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);

	final Environment env = createMockEnvironment();
	CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
	OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(env, "test-op-name", emptyStateHandles, cancelStreamRegistry);

	AtomicInteger copyCounter = new AtomicInteger(0);
	TypeSerializer<Integer> serializer = new VerifyingIntSerializer(env.getUserClassLoader(), copyCounter);

	// write some state
	ListStateDescriptor<Integer> stateDescriptor = new ListStateDescriptor<>("test", serializer);
	ListState<Integer> listState = operatorStateBackend.getListState(stateDescriptor);

	listState.add(42);

	AtomicInteger keyCopyCounter = new AtomicInteger(0);
	AtomicInteger valueCopyCounter = new AtomicInteger(0);

	TypeSerializer<Integer> keySerializer = new VerifyingIntSerializer(env.getUserClassLoader(), keyCopyCounter);
	TypeSerializer<Integer> valueSerializer = new VerifyingIntSerializer(env.getUserClassLoader(), valueCopyCounter);

	MapStateDescriptor<Integer, Integer> broadcastStateDesc = new MapStateDescriptor<>(
			"test-broadcast", keySerializer, valueSerializer);

	BroadcastState<Integer, Integer> broadcastState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
	broadcastState.put(1, 2);
	broadcastState.put(3, 4);
	broadcastState.put(5, 6);

	CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
	RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture =
		operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
	FutureUtils.runIfNotDoneAndGet(runnableFuture);

	// make sure that the copy method has been called
	assertTrue(copyCounter.get() > 0);
	assertTrue(keyCopyCounter.get() > 0);
	assertTrue(valueCopyCounter.get() > 0);
}
Example 14
Source File: OperatorStateBackendTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSnapshotAsyncClose() throws Exception {
	DefaultOperatorStateBackend operatorStateBackend =
		new DefaultOperatorStateBackendBuilder(
			OperatorStateBackendTest.class.getClassLoader(),
			new ExecutionConfig(),
			true,
			emptyStateHandles,
			new CloseableRegistry()).build();

	ListStateDescriptor<MutableType> stateDescriptor1 =
			new ListStateDescriptor<>("test1", new JavaSerializer<MutableType>());

	ListState<MutableType> listState1 = operatorStateBackend.getListState(stateDescriptor1);

	listState1.add(MutableType.of(42));
	listState1.add(MutableType.of(4711));

	MapStateDescriptor<MutableType, MutableType> broadcastStateDescriptor1 =
			new MapStateDescriptor<>("test4", new JavaSerializer<MutableType>(), new JavaSerializer<MutableType>());

	BroadcastState<MutableType, MutableType> broadcastState1 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor1);
	broadcastState1.put(MutableType.of(1), MutableType.of(2));
	broadcastState1.put(MutableType.of(2), MutableType.of(5));

	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);

	OneShotLatch waiterLatch = new OneShotLatch();
	OneShotLatch blockerLatch = new OneShotLatch();

	streamFactory.setWaiterLatch(waiterLatch);
	streamFactory.setBlockerLatch(blockerLatch);

	RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture =
			operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

	ExecutorService executorService = Executors.newFixedThreadPool(1);

	executorService.submit(runnableFuture);

	// wait until the async checkpoint is in the write code, then continue
	waiterLatch.await();

	operatorStateBackend.close();

	blockerLatch.trigger();

	try {
		runnableFuture.get(60, TimeUnit.SECONDS);
		Assert.fail();
	} catch (CancellationException expected) {
	}
}