Java Code Examples for org.apache.flink.api.common.state.ListState#clear()
The following examples show how to use
org.apache.flink.api.common.state.ListState#clear() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
/**
* Verify that an empty {@code ListState} yields {@code null}.
*/
@Test
public void testListStateDefaultValue() throws Exception {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);
ListState<String> state = backend.getPartitionedState(
VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
backend.setCurrentKey(1);
assertNull(state.get());
state.update(Arrays.asList("Ciao", "Bello"));
assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));
state.clear();
assertNull(state.get());
backend.dispose();
}
Example 2
| Source File: FlinkBroadcastStateInternals.java From beam with Apache License 2.0 | 6 votes |
|
/** Update map(namespce->T) from index 0. */
void updateMap(Map<String, T> map) throws Exception {
if (indexInSubtaskGroup == 0) {
ListState<Map<String, T>> state = flinkStateBackend.getUnionListState(flinkStateDescriptor);
state.clear();
if (map.size() > 0) {
state.add(map);
}
} else {
if (map.isEmpty()) {
stateForNonZeroOperator.remove(name);
// updateMap is always behind getMap,
// getMap will clear map in BroadcastOperatorState,
// we don't need clear here.
} else {
stateForNonZeroOperator.put(name, map);
}
}
}
Example 3
| Source File: Buckets.java From flink with Apache License 2.0 | 6 votes |
|
void snapshotState(
final long checkpointId,
final ListState<byte[]> bucketStatesContainer,
final ListState<Long> partCounterStateContainer) throws Exception {
Preconditions.checkState(
fsWriter != null && bucketStateSerializer != null,
"sink has not been initialized");
LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
subtaskIndex, checkpointId, maxPartCounter);
bucketStatesContainer.clear();
partCounterStateContainer.clear();
snapshotActiveBuckets(checkpointId, bucketStatesContainer);
partCounterStateContainer.add(maxPartCounter);
}
Example 4
| Source File: AsyncWaitOperator.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
super.snapshotState(context);
ListState<StreamElement> partitionableState =
getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
partitionableState.clear();
Collection<StreamElementQueueEntry<?>> values = queue.values();
try {
for (StreamElementQueueEntry<?> value : values) {
partitionableState.add(value.getStreamElement());
}
// add the pending stream element queue entry if the stream element queue is currently full
if (pendingStreamElementQueueEntry != null) {
partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
}
} catch (Exception e) {
partitionableState.clear();
throw new Exception("Could not add stream element queue entries to operator state " +
"backend of operator " + getOperatorName() + '.', e);
}
}
Example 5
| Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Verify that an empty {@code ListState} yields {@code null}.
*/
@Test
public void testListStateDefaultValue() throws Exception {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);
ListState<String> state = backend.getPartitionedState(
VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
backend.setCurrentKey(1);
assertNull(state.get());
state.update(Arrays.asList("Ciao", "Bello"));
assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));
state.clear();
assertNull(state.get());
backend.dispose();
}
Example 6
| Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Verify that an empty {@code ListState} yields {@code null}.
*/
@Test
public void testListStateDefaultValue() throws Exception {
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);
ListState<String> state = backend.getPartitionedState(
VoidNamespace.INSTANCE,
VoidNamespaceSerializer.INSTANCE, kvId);
backend.setCurrentKey(1);
assertNull(state.get());
state.update(Arrays.asList("Ciao", "Bello"));
assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));
state.clear();
assertNull(state.get());
backend.dispose();
}
Example 7
| Source File: Buckets.java From flink with Apache License 2.0 | 6 votes |
|
public void snapshotState(
final long checkpointId,
final ListState<byte[]> bucketStatesContainer,
final ListState<Long> partCounterStateContainer) throws Exception {
Preconditions.checkState(
bucketWriter != null && bucketStateSerializer != null,
"sink has not been initialized");
LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
subtaskIndex, checkpointId, maxPartCounter);
bucketStatesContainer.clear();
partCounterStateContainer.clear();
snapshotActiveBuckets(checkpointId, bucketStatesContainer);
partCounterStateContainer.add(maxPartCounter);
}
Example 8
| Source File: Buckets.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
void snapshotState(
final long checkpointId,
final ListState<byte[]> bucketStatesContainer,
final ListState<Long> partCounterStateContainer) throws Exception {
Preconditions.checkState(
fsWriter != null && bucketStateSerializer != null,
"sink has not been initialized");
LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
subtaskIndex, checkpointId, maxPartCounter);
bucketStatesContainer.clear();
partCounterStateContainer.clear();
snapshotActiveBuckets(checkpointId, bucketStatesContainer);
partCounterStateContainer.add(maxPartCounter);
}
Example 9
| Source File: AsyncWaitOperator.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
super.snapshotState(context);
ListState<StreamElement> partitionableState =
getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
partitionableState.clear();
try {
partitionableState.addAll(queue.values());
} catch (Exception e) {
partitionableState.clear();
throw new Exception("Could not add stream element queue entries to operator state " +
"backend of operator " + getOperatorName() + '.', e);
}
}
Example 10
| Source File: FlinkKafkaProducer.java From flink with Apache License 2.0 | 5 votes |
|
private void migrateNextTransactionalIdHindState(FunctionInitializationContext context) throws Exception {
ListState<NextTransactionalIdHint> oldNextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(
NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR);
nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2);
ArrayList<NextTransactionalIdHint> oldTransactionalIdHints = Lists.newArrayList(oldNextTransactionalIdHintState.get());
if (!oldTransactionalIdHints.isEmpty()) {
nextTransactionalIdHintState.addAll(oldTransactionalIdHints);
//clear old state
oldNextTransactionalIdHintState.clear();
}
}
Example 11
| Source File: StreamingFunctionUtils.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private static boolean trySnapshotFunctionState(
StateSnapshotContext context,
OperatorStateBackend backend,
Function userFunction) throws Exception {
if (userFunction instanceof CheckpointedFunction) {
((CheckpointedFunction) userFunction).snapshotState(context);
return true;
}
if (userFunction instanceof ListCheckpointed) {
@SuppressWarnings("unchecked")
List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());
ListState<Serializable> listState = backend.
getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);
listState.clear();
if (null != partitionableState) {
try {
for (Serializable statePartition : partitionableState) {
listState.add(statePartition);
}
} catch (Exception e) {
listState.clear();
throw new Exception("Could not write partitionable state to operator " +
"state backend.", e);
}
}
return true;
}
return false;
}
Example 12
| Source File: EvictingWindowOperator.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private void clearAllState(
W window,
ListState<StreamRecord<IN>> windowState,
MergingWindowSet<W> mergingWindows) throws Exception {
windowState.clear();
triggerContext.clear();
processContext.window = window;
processContext.clear();
if (mergingWindows != null) {
mergingWindows.retireWindow(window);
mergingWindows.persist();
}
}
Example 13
| Source File: EvictingWindowOperator.java From flink with Apache License 2.0 | 5 votes |
|
private void clearAllState(
W window,
ListState<StreamRecord<IN>> windowState,
MergingWindowSet<W> mergingWindows) throws Exception {
windowState.clear();
triggerContext.clear();
processContext.window = window;
processContext.clear();
if (mergingWindows != null) {
mergingWindows.retireWindow(window);
mergingWindows.persist();
}
}
Example 14
| Source File: OneInputStreamTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
ListState<Integer> partitionableState =
getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
partitionableState.clear();
partitionableState.add(42);
partitionableState.add(4711);
++numberSnapshotCalls;
}
Example 15
| Source File: OneInputStreamTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
ListState<Integer> partitionableState =
getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
partitionableState.clear();
partitionableState.add(42);
partitionableState.add(4711);
++numberSnapshotCalls;
}
Example 16
| Source File: FlinkKafkaProducer.java From flink with Apache License 2.0 | 5 votes |
|
private void migrateNextTransactionalIdHindState(FunctionInitializationContext context) throws Exception {
ListState<NextTransactionalIdHint> oldNextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(
NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR);
nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2);
ArrayList<NextTransactionalIdHint> oldTransactionalIdHints = Lists.newArrayList(oldNextTransactionalIdHintState.get());
if (!oldTransactionalIdHints.isEmpty()) {
nextTransactionalIdHintState.addAll(oldTransactionalIdHints);
//clear old state
oldNextTransactionalIdHintState.clear();
}
}
Example 17
| Source File: FlinkKafkaProducer011.java From flink with Apache License 2.0 | 5 votes |
|
private void migrateNextTransactionalIdHindState(FunctionInitializationContext context) throws Exception {
ListState<NextTransactionalIdHint> oldNextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(
NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR);
nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2);
ArrayList<NextTransactionalIdHint> oldTransactionalIdHints = Lists.newArrayList(oldNextTransactionalIdHintState.get());
if (!oldTransactionalIdHints.isEmpty()) {
nextTransactionalIdHintState.addAll(oldTransactionalIdHints);
//clear old state
oldNextTransactionalIdHintState.clear();
}
}
Example 18
| Source File: StreamingFunctionUtils.java From flink with Apache License 2.0 | 4 votes |
|
private static boolean trySnapshotFunctionState(
StateSnapshotContext context,
OperatorStateBackend backend,
Function userFunction) throws Exception {
if (userFunction instanceof CheckpointedFunction) {
((CheckpointedFunction) userFunction).snapshotState(context);
return true;
}
if (userFunction instanceof ListCheckpointed) {
@SuppressWarnings("unchecked")
List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());
// We are using JavaSerializer from the flink-runtime module here. This is very naughty and
// we shouldn't be doing it because ideally nothing in the API modules/connector depends
// directly on flink-runtime. We are doing it here because we need to maintain backwards
// compatibility with old state and because we will have to rework/remove this code soon.
ListStateDescriptor<Serializable> listStateDescriptor =
new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new JavaSerializer<>());
ListState<Serializable> listState = backend.getListState(listStateDescriptor);
listState.clear();
if (null != partitionableState) {
try {
for (Serializable statePartition : partitionableState) {
listState.add(statePartition);
}
} catch (Exception e) {
listState.clear();
throw new Exception("Could not write partitionable state to operator " +
"state backend.", e);
}
}
return true;
}
return false;
}
Example 19
| Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testListStateAPIs() throws Exception {
AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);
final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
try {
ListState<Long> state =
keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("def");
assertNull(state.get());
state.add(17L);
state.add(11L);
assertThat(state.get(), containsInAnyOrder(17L, 11L));
// update(emptyList) should remain the value null
state.update(Collections.emptyList());
assertNull(state.get());
state.update(Arrays.asList(10L, 16L));
assertThat(state.get(), containsInAnyOrder(16L, 10L));
assertThat(state.get(), containsInAnyOrder(16L, 10L));
keyedBackend.setCurrentKey("abc");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertNull(state.get());
assertNull(state.get());
state.addAll(Collections.emptyList());
assertNull(state.get());
state.addAll(Arrays.asList(3L, 4L));
assertThat(state.get(), containsInAnyOrder(3L, 4L));
assertThat(state.get(), containsInAnyOrder(3L, 4L));
state.addAll(new ArrayList<>());
assertThat(state.get(), containsInAnyOrder(3L, 4L));
state.addAll(Arrays.asList(5L, 6L));
assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
state.addAll(new ArrayList<>());
assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
state.update(Arrays.asList(1L, 2L));
assertThat(state.get(), containsInAnyOrder(1L, 2L));
keyedBackend.setCurrentKey("def");
assertThat(state.get(), containsInAnyOrder(10L, 16L));
state.clear();
assertNull(state.get());
keyedBackend.setCurrentKey("g");
state.add(3L);
state.add(2L);
state.add(1L);
keyedBackend.setCurrentKey("def");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertThat(state.get(), containsInAnyOrder(1L, 2L, 3L, 2L, 1L));
state.update(Arrays.asList(5L, 6L));
assertThat(state.get(), containsInAnyOrder(5L, 6L));
state.clear();
// make sure all lists / maps are cleared
assertThat("State backend is not empty.", keyedBackend.numKeyValueStateEntries(), is(0));
} finally {
keyedBackend.close();
keyedBackend.dispose();
}
}
Example 20
| Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testListStateAPIs() throws Exception {
AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);
final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
try {
ListState<Long> state =
keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr);
keyedBackend.setCurrentKey("def");
assertNull(state.get());
state.add(17L);
state.add(11L);
assertThat(state.get(), containsInAnyOrder(17L, 11L));
// update(emptyList) should remain the value null
state.update(Collections.emptyList());
assertNull(state.get());
state.update(Arrays.asList(10L, 16L));
assertThat(state.get(), containsInAnyOrder(16L, 10L));
assertThat(state.get(), containsInAnyOrder(16L, 10L));
keyedBackend.setCurrentKey("abc");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertNull(state.get());
assertNull(state.get());
state.addAll(Collections.emptyList());
assertNull(state.get());
state.addAll(Arrays.asList(3L, 4L));
assertThat(state.get(), containsInAnyOrder(3L, 4L));
assertThat(state.get(), containsInAnyOrder(3L, 4L));
state.addAll(new ArrayList<>());
assertThat(state.get(), containsInAnyOrder(3L, 4L));
state.addAll(Arrays.asList(5L, 6L));
assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
state.addAll(new ArrayList<>());
assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
state.update(Arrays.asList(1L, 2L));
assertThat(state.get(), containsInAnyOrder(1L, 2L));
keyedBackend.setCurrentKey("def");
assertThat(state.get(), containsInAnyOrder(10L, 16L));
state.clear();
assertNull(state.get());
keyedBackend.setCurrentKey("g");
state.add(3L);
state.add(2L);
state.add(1L);
keyedBackend.setCurrentKey("def");
assertNull(state.get());
keyedBackend.setCurrentKey("g");
assertThat(state.get(), containsInAnyOrder(1L, 2L, 3L, 2L, 1L));
state.update(Arrays.asList(5L, 6L));
assertThat(state.get(), containsInAnyOrder(5L, 6L));
state.clear();
// make sure all lists / maps are cleared
assertThat("State backend is not empty.", keyedBackend.numKeyValueStateEntries(), is(0));
} finally {
keyedBackend.close();
keyedBackend.dispose();
}
}
