Java Code Examples for org.apache.flink.api.common.state.ReducingState#add()
The following examples show how to use
org.apache.flink.api.common.state.ReducingState#add() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ContinuousEventTimeTrigger.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public TriggerResult onEventTime(long time, W window, TriggerContext ctx) throws Exception {
if (time == window.maxTimestamp()){
return TriggerResult.FIRE;
}
ReducingState<Long> fireTimestampState = ctx.getPartitionedState(stateDesc);
Long fireTimestamp = fireTimestampState.get();
if (fireTimestamp != null && fireTimestamp == time) {
fireTimestampState.clear();
fireTimestampState.add(time + interval);
ctx.registerEventTimeTimer(time + interval);
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
Example 2
| Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Verify that an empty {@code ReduceState} yields {@code null}.
*/
@Test
public void testReducingStateDefaultValue() throws Exception {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), String.class);
ReducingState<String> state = backend.getPartitionedState(
VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
backend.setCurrentKey(1);
assertNull(state.get());
state.add("Ciao");
assertEquals("Ciao", state.get());
state.clear();
assertNull(state.get());
backend.dispose();
}
Example 3
| Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Verify that an empty {@code ReduceState} yields {@code null}.
*/
@Test
public void testReducingStateDefaultValue() throws Exception {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), String.class);
ReducingState<String> state = backend.getPartitionedState(
VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
backend.setCurrentKey(1);
assertNull(state.get());
state.add("Ciao");
assertEquals("Ciao", state.get());
state.clear();
assertNull(state.get());
backend.dispose();
}
Example 4
| Source File: ContinuousEventTimeTrigger.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception {
if (window.maxTimestamp() <= ctx.getCurrentWatermark()) {
// if the watermark is already past the window fire immediately
return TriggerResult.FIRE;
} else {
ctx.registerEventTimeTimer(window.maxTimestamp());
}
ReducingState<Long> fireTimestamp = ctx.getPartitionedState(stateDesc);
if (fireTimestamp.get() == null) {
long start = timestamp - (timestamp % interval);
long nextFireTimestamp = start + interval;
ctx.registerEventTimeTimer(nextFireTimestamp);
fireTimestamp.add(nextFireTimestamp);
}
return TriggerResult.CONTINUE;
}
Example 5
| Source File: ContinuousEventTimeTrigger.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
public TriggerResult onEventTime(long time, W window, TriggerContext ctx) throws Exception {
if (time == window.maxTimestamp()){
return TriggerResult.FIRE;
}
ReducingState<Long> fireTimestampState = ctx.getPartitionedState(stateDesc);
Long fireTimestamp = fireTimestampState.get();
if (fireTimestamp != null && fireTimestamp == time) {
fireTimestampState.clear();
fireTimestampState.add(time + interval);
ctx.registerEventTimeTimer(time + interval);
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
Example 6
| Source File: ContinuousEventTimeTrigger.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public TriggerResult onEventTime(long time, W window, TriggerContext ctx) throws Exception {
if (time == window.maxTimestamp()){
return TriggerResult.FIRE;
}
ReducingState<Long> fireTimestampState = ctx.getPartitionedState(stateDesc);
Long fireTimestamp = fireTimestampState.get();
if (fireTimestamp != null && fireTimestamp == time) {
fireTimestampState.clear();
fireTimestampState.add(time + interval);
ctx.registerEventTimeTimer(time + interval);
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
Example 7
| Source File: ContinuousProcessingTimeTrigger.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception {
ReducingState<Long> fireTimestamp = ctx.getPartitionedState(stateDesc);
timestamp = ctx.getCurrentProcessingTime();
if (fireTimestamp.get() == null) {
long start = timestamp - (timestamp % interval);
long nextFireTimestamp = start + interval;
ctx.registerProcessingTimeTimer(nextFireTimestamp);
fireTimestamp.add(nextFireTimestamp);
return TriggerResult.CONTINUE;
}
return TriggerResult.CONTINUE;
}
Example 8
| Source File: ElementTriggers.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public boolean onElement(Object element, long timestamp, W window) throws Exception {
ReducingState<Long> count = ctx.getPartitionedState(countStateDesc);
count.add(1L);
if (count.get() >= countElems) {
count.clear();
return true;
} else {
return false;
}
}
Example 9
| Source File: ContinuousProcessingTimeTrigger.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public TriggerResult onProcessingTime(long time, W window, TriggerContext ctx) throws Exception {
ReducingState<Long> fireTimestamp = ctx.getPartitionedState(stateDesc);
if (fireTimestamp.get().equals(time)) {
fireTimestamp.clear();
fireTimestamp.add(time + interval);
ctx.registerProcessingTimeTimer(time + interval);
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
Example 10
| Source File: ProcessingTimeTriggers.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public boolean onProcessingTime(long time, W window) throws Exception {
ReducingState<Long> nextFiring = ctx.getPartitionedState(nextFiringStateDesc);
Long timer = nextFiring.get();
if (timer != null && timer == time) {
long newTimer = time + interval;
ctx.registerProcessingTimeTimer(newTimer);
nextFiring.clear();
nextFiring.add(newTimer);
return true;
} else {
return false;
}
}
Example 11
| Source File: CountTrigger.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception {
ReducingState<Long> count = ctx.getPartitionedState(stateDesc);
count.add(1L);
if (count.get() >= maxCount) {
count.clear();
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
Example 12
| Source File: ContinuousProcessingTimeTrigger.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
public TriggerResult onProcessingTime(long time, W window, TriggerContext ctx) throws Exception {
ReducingState<Long> fireTimestamp = ctx.getPartitionedState(stateDesc);
if (fireTimestamp.get().equals(time)) {
fireTimestamp.clear();
fireTimestamp.add(time + interval);
ctx.registerProcessingTimeTimer(time + interval);
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
Example 13
| Source File: CountTrigger.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception {
ReducingState<Long> count = ctx.getPartitionedState(stateDesc);
count.add(1L);
if (count.get() >= maxCount) {
count.clear();
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
Example 14
| Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testReducingState() throws Exception {
CheckpointStreamFactory streamFactory = createStreamFactory();
SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), String.class);
TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
ReducingState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked")
InternalKvState<Integer, VoidNamespace, String> kvState = (InternalKvState<Integer, VoidNamespace, String>) state;
// this is only available after the backend initialized the serializer
TypeSerializer<String> valueSerializer = kvId.getSerializer();
// some modifications to the state
backend.setCurrentKey(1);
assertNull(state.get());
assertNull(getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.add("1");
backend.setCurrentKey(2);
assertNull(state.get());
assertNull(getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.add("2");
backend.setCurrentKey(1);
assertEquals("1", state.get());
assertEquals("1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
// draw a snapshot
KeyedStateHandle snapshot1 = runSnapshot(
backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
sharedStateRegistry);
// make some more modifications
backend.setCurrentKey(1);
state.add("u1");
backend.setCurrentKey(2);
state.add("u2");
backend.setCurrentKey(3);
state.add("u3");
// draw another snapshot
KeyedStateHandle snapshot2 = runSnapshot(
backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
sharedStateRegistry);
// validate the original state
backend.setCurrentKey(1);
assertEquals("1,u1", state.get());
assertEquals("1,u1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2,u2", state.get());
assertEquals("2,u2", getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", state.get());
assertEquals("u3", getSerializedValue(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
// restore the first snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
snapshot1.discardState();
ReducingState<String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked")
InternalKvState<Integer, VoidNamespace, String> restoredKvState1 = (InternalKvState<Integer, VoidNamespace, String>) restored1;
backend.setCurrentKey(1);
assertEquals("1", restored1.get());
assertEquals("1", getSerializedValue(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2", restored1.get());
assertEquals("2", getSerializedValue(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
// restore the second snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
snapshot2.discardState();
ReducingState<String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked")
InternalKvState<Integer, VoidNamespace, String> restoredKvState2 = (InternalKvState<Integer, VoidNamespace, String>) restored2;
backend.setCurrentKey(1);
assertEquals("1,u1", restored2.get());
assertEquals("1,u1", getSerializedValue(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2,u2", restored2.get());
assertEquals("2,u2", getSerializedValue(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", restored2.get());
assertEquals("u3", getSerializedValue(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
}
Example 15
| Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testReducingStateAddAndGet() throws Exception {
final ReducingStateDescriptor<Long> stateDescr =
new ReducingStateDescriptor<>("my-state", (a, b) -> a + b, Long.class);
AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);
try {
ReducingState<Long> state =
keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
assertNull(state.get());
keyedBackend.setCurrentKey("def");
assertNull(state.get());
state.add(17L);
state.add(11L);
assertEquals(28L, state.get().longValue());
keyedBackend.setCurrentKey("abc");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertNull(state.get());
state.add(1L);
state.add(2L);
keyedBackend.setCurrentKey("def");
assertEquals(28L, state.get().longValue());
state.clear();
assertNull(state.get());
keyedBackend.setCurrentKey("g");
state.add(3L);
state.add(2L);
state.add(1L);
keyedBackend.setCurrentKey("def");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertEquals(9L, state.get().longValue());
state.clear();
// make sure all lists / maps are cleared
assertThat("State backend is not empty.", keyedBackend.numKeyValueStateEntries(), is(0));
}
finally {
keyedBackend.close();
keyedBackend.dispose();
}
}
Example 16
| Source File: TtlReducingStateVerifier.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
void updateInternal(@Nonnull ReducingState<Integer> state, Integer update) throws Exception {
state.add(update);
}
Example 17
| Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testReducingStateRestoreWithWrongSerializers() throws Exception {
CheckpointStreamFactory streamFactory = createStreamFactory();
SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
try {
ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id",
new AppendingReduce(),
StringSerializer.INSTANCE);
ReducingState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
backend.setCurrentKey(1);
state.add("1");
backend.setCurrentKey(2);
state.add("2");
// draw a snapshot
KeyedStateHandle snapshot1 = runSnapshot(
backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
sharedStateRegistry);
backend.dispose();
// restore the first snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
snapshot1.discardState();
@SuppressWarnings("unchecked")
TypeSerializer<String> fakeStringSerializer =
(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;
try {
kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), fakeStringSerializer);
state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
state.get();
fail("should recognize wrong serializers");
} catch (StateMigrationException ignored) {
// expected
}
} finally {
backend.dispose();
}
}
Example 18
| Source File: TtlReducingStateVerifier.java From flink with Apache License 2.0 | 4 votes |
|
@Override
void updateInternal(@Nonnull ReducingState<Integer> state, Integer update) throws Exception {
state.add(update);
}
Example 19
| Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testReducingState() throws Exception {
CheckpointStreamFactory streamFactory = createStreamFactory();
SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), String.class);
TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
ReducingState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked")
InternalKvState<Integer, VoidNamespace, String> kvState = (InternalKvState<Integer, VoidNamespace, String>) state;
// this is only available after the backend initialized the serializer
TypeSerializer<String> valueSerializer = kvId.getSerializer();
// some modifications to the state
backend.setCurrentKey(1);
assertNull(state.get());
assertNull(getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.add("1");
backend.setCurrentKey(2);
assertNull(state.get());
assertNull(getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.add("2");
backend.setCurrentKey(1);
assertEquals("1", state.get());
assertEquals("1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
// draw a snapshot
KeyedStateHandle snapshot1 = runSnapshot(
backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
sharedStateRegistry);
// make some more modifications
backend.setCurrentKey(1);
state.add("u1");
backend.setCurrentKey(2);
state.add("u2");
backend.setCurrentKey(3);
state.add("u3");
// draw another snapshot
KeyedStateHandle snapshot2 = runSnapshot(
backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
sharedStateRegistry);
// validate the original state
backend.setCurrentKey(1);
assertEquals("1,u1", state.get());
assertEquals("1,u1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2,u2", state.get());
assertEquals("2,u2", getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", state.get());
assertEquals("u3", getSerializedValue(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
// restore the first snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
snapshot1.discardState();
ReducingState<String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked")
InternalKvState<Integer, VoidNamespace, String> restoredKvState1 = (InternalKvState<Integer, VoidNamespace, String>) restored1;
backend.setCurrentKey(1);
assertEquals("1", restored1.get());
assertEquals("1", getSerializedValue(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2", restored1.get());
assertEquals("2", getSerializedValue(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
// restore the second snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
snapshot2.discardState();
ReducingState<String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked")
InternalKvState<Integer, VoidNamespace, String> restoredKvState2 = (InternalKvState<Integer, VoidNamespace, String>) restored2;
backend.setCurrentKey(1);
assertEquals("1,u1", restored2.get());
assertEquals("1,u1", getSerializedValue(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2,u2", restored2.get());
assertEquals("2,u2", getSerializedValue(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", restored2.get());
assertEquals("u3", getSerializedValue(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
}
Example 20
| Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testReducingStateAddAndGet() throws Exception {
final ReducingStateDescriptor<Long> stateDescr =
new ReducingStateDescriptor<>("my-state", (a, b) -> a + b, Long.class);
AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);
try {
ReducingState<Long> state =
keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("abc");
assertNull(state.get());
keyedBackend.setCurrentKey("def");
assertNull(state.get());
state.add(17L);
state.add(11L);
assertEquals(28L, state.get().longValue());
keyedBackend.setCurrentKey("abc");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertNull(state.get());
state.add(1L);
state.add(2L);
keyedBackend.setCurrentKey("def");
assertEquals(28L, state.get().longValue());
state.clear();
assertNull(state.get());
keyedBackend.setCurrentKey("g");
state.add(3L);
state.add(2L);
state.add(1L);
keyedBackend.setCurrentKey("def");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertEquals(9L, state.get().longValue());
state.clear();
// make sure all lists / maps are cleared
assertThat("State backend is not empty.", keyedBackend.numKeyValueStateEntries(), is(0));
}
finally {
keyedBackend.close();
keyedBackend.dispose();
}
}
