org.apache.flink.runtime.state.AbstractStateBackend Java Examples

The following examples show how to use org.apache.flink.runtime.state.AbstractStateBackend. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HeapStateBackendTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
Example #2
Source File: QueryableWindowOperator.java    From yahoo-streaming-benchmark with Apache License 2.0 6 votes vote down vote up 
@Override
public StateHandle<DataInputView> materialize() throws Exception {
  AbstractStateBackend.CheckpointStateOutputView out = backend.createCheckpointStateOutputView(
    checkpointId,
    timestamp);

  int numWindows = stateSnapshot.size();
  out.writeInt(numWindows);


  for (Map.Entry<String, Map<Long, CountAndAccessTime>> window : stateSnapshot.entrySet()) {
    out.writeUTF(window.getKey());
    int numKeys = window.getValue().size();
    out.writeInt(numKeys);

    for (Map.Entry<Long, CountAndAccessTime> value : window.getValue().entrySet()) {
      out.writeLong(value.getKey());
      out.writeLong(value.getValue().count);
      out.writeLong(value.getValue().lastAccessTime);
      out.writeLong(value.getValue().lastEventTime);
    }
  }

  this.size = out.size();
  return out.closeAndGetHandle();
}
Example #3
Source File: QueryableWindowOperatorEvicting.java    From yahoo-streaming-benchmark with Apache License 2.0 6 votes vote down vote up 
@Override
public StateHandle<DataInputView> materialize() throws Exception {

	AbstractStateBackend.CheckpointStateOutputView out =
			backend.createCheckpointStateOutputView(checkpointId, timestamp);
	
	out.writeInt(stateSnapshot.size());
	
	for (Map.Entry<Long, Map<UUID, CountAndAccessTime>> window: stateSnapshot.entrySet()) {
		out.writeLong(window.getKey());
		out.writeInt(window.getValue().size());

		for (Map.Entry<UUID, CountAndAccessTime> value : window.getValue().entrySet()) {
			out.writeLong(value.getKey().getMostSignificantBits());
			out.writeLong(value.getKey().getLeastSignificantBits());
			out.writeLong(value.getValue().count);
		}
	}

	this.size = out.size();
	return out.closeAndGetHandle();
}
Example #4
Source File: FlinkGroupAlsoByWindowWrapper.java    From flink-dataflow with Apache License 2.0 6 votes vote down vote up 
@Override
public StreamTaskState snapshotOperatorState(long checkpointId, long timestamp) throws Exception {
	StreamTaskState taskState = super.snapshotOperatorState(checkpointId, timestamp);
	AbstractStateBackend.CheckpointStateOutputView out = getStateBackend().createCheckpointStateOutputView(checkpointId, timestamp);
	StateCheckpointWriter writer = StateCheckpointWriter.create(out);
	Coder<K> keyCoder = inputKvCoder.getKeyCoder();

	// checkpoint the timers
	StateCheckpointUtils.encodeTimers(activeTimers, writer, keyCoder);

	// checkpoint the state
	StateCheckpointUtils.encodeState(perKeyStateInternals, writer, keyCoder);

	// checkpoint the timerInternals
	context.timerInternals.encodeTimerInternals(context, writer,
			inputKvCoder, windowingStrategy.getWindowFn().windowCoder());

	taskState.setOperatorState(out.closeAndGetHandle());
	return taskState;
}
Example #5
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #6
Source File: KvStateRequestSerializerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #7
Source File: MockStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return new MockKeyedStateBackendBuilder<>(
		new KvStateRegistry().createTaskRegistry(jobID, new JobVertexID()),
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		cancelStreamRegistry).build();
}
Example #8
Source File: StateBackendBenchmarkUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static HeapKeyedStateBackend<Long> createHeapKeyedStateBackend(File rootDir) throws IOException {
	File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
	KeyGroupRange keyGroupRange = new KeyGroupRange(0, 1);
	int numberOfKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	HeapKeyedStateBackendBuilder<Long> backendBuilder = new HeapKeyedStateBackendBuilder<>(
		null,
		new LongSerializer(),
		Thread.currentThread().getContextClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		Collections.emptyList(),
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		new LocalRecoveryConfig(false, new LocalRecoveryDirectoryProviderImpl(recoveryBaseDir, new JobID(), new JobVertexID(), 0)),
		priorityQueueSetFactory,
		false,
		new CloseableRegistry()
	);
	return backendBuilder.build();
}
Example #9
Source File: HeapStateBackendTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
Example #10
Source File: MockStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return new MockKeyedStateBackendBuilder<>(
		new KvStateRegistry().createTaskRegistry(jobID, new JobVertexID()),
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		cancelStreamRegistry).build();
}
Example #11
Source File: HeapStateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
Example #12
Source File: MockStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return new MockKeyedStateBackendBuilder<>(
		new KvStateRegistry().createTaskRegistry(jobID, new JobVertexID()),
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		cancelStreamRegistry).build();
}
Example #13
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
Example #14
Source File: MemoryStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		taskStateManager.createLocalRecoveryConfig(),
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #15
Source File: StateSerializationTest.java    From flink-dataflow with Apache License 2.0 5 votes vote down vote up 
private void storeState(AbstractStateBackend.CheckpointStateOutputView out) throws Exception {
	StateCheckpointWriter checkpointBuilder = StateCheckpointWriter.create(out);
	Coder<String> keyCoder = StringUtf8Coder.of();

	// checkpoint the timers
	StateCheckpointUtils.encodeTimers(activeTimers, checkpointBuilder, keyCoder);

	// checkpoint the state
	StateCheckpointUtils.encodeState(statePerKey, checkpointBuilder, keyCoder);
}
Example #16
Source File: StreamExecutionEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * @deprecated Use {@link #setStateBackend(StateBackend)} instead.
 */
@Deprecated
@PublicEvolving
public StreamExecutionEnvironment setStateBackend(AbstractStateBackend backend) {
	this.defaultStateBackend = Preconditions.checkNotNull(backend);
	return this;
}
Example #17
Source File: KvStateServerHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private AbstractKeyedStateBackend<Integer> createKeyedStateBackend(KvStateRegistry registry, int numKeyGroups, AbstractStateBackend abstractBackend, DummyEnvironment dummyEnv) throws java.io.IOException {
	return abstractBackend.createKeyedStateBackend(
		dummyEnv,
		dummyEnv.getJobID(),
		"test_op",
		IntSerializer.INSTANCE,
		numKeyGroups,
		new KeyGroupRange(0, 0),
		registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}
Example #18
Source File: FsStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #19
Source File: KvStateServerHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private AbstractKeyedStateBackend<Integer> createKeyedStateBackend(KvStateRegistry registry, int numKeyGroups, AbstractStateBackend abstractBackend, DummyEnvironment dummyEnv) throws java.io.IOException {
	return abstractBackend.createKeyedStateBackend(
		dummyEnv,
		dummyEnv.getJobID(),
		"test_op",
		IntSerializer.INSTANCE,
		numKeyGroups,
		new KeyGroupRange(0, 0),
		registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}
Example #20
Source File: StreamExecutionEnvironment.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * @deprecated Use {@link #setStateBackend(StateBackend)} instead.
 */
@Deprecated
@PublicEvolving
public StreamExecutionEnvironment setStateBackend(AbstractStateBackend backend) {
	this.defaultStateBackend = Preconditions.checkNotNull(backend);
	return this;
}
Example #21
Source File: QueryableWindowOperator.java    From yahoo-streaming-benchmark with Apache License 2.0 5 votes vote down vote up 
public DataInputViewAsynchronousStateHandle(long checkpointId,
                                            long timestamp,
                                            Map<String, Map<Long, CountAndAccessTime>> stateSnapshot,
                                            AbstractStateBackend backend) {
  this.checkpointId = checkpointId;
  this.timestamp = timestamp;
  this.stateSnapshot = stateSnapshot;
  this.backend = backend;
}
Example #22
Source File: QueryableWindowOperatorEvicting.java    From yahoo-streaming-benchmark with Apache License 2.0 5 votes vote down vote up 
public DataInputViewAsynchronousStateHandle(long checkpointId,
		long timestamp,
		Map<Long, Map<UUID, CountAndAccessTime>> stateSnapshot,
											AbstractStateBackend backend) {
	this.checkpointId = checkpointId;
	this.timestamp = timestamp;
	this.stateSnapshot = stateSnapshot;
	this.backend = backend;
}
Example #23
Source File: KVStateRequestSerializerRocksDBTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
Example #24
Source File: KeyedStateCheckpointingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected void testProgramWithBackend(AbstractStateBackend stateBackend) throws Exception {
	assertEquals("Broken test setup", 0, (NUM_STRINGS / 2) % NUM_KEYS);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(PARALLELISM);
	env.enableCheckpointing(500);
			env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));

	env.setStateBackend(stateBackend);

	// compute when (randomly) the failure should happen
	final int failurePosMin = (int) (0.6 * NUM_STRINGS / PARALLELISM);
	final int failurePosMax = (int) (0.8 * NUM_STRINGS / PARALLELISM);
	final int failurePos = (new Random().nextInt(failurePosMax - failurePosMin) + failurePosMin);

	final DataStream<Integer> stream1 = env.addSource(
			new IntGeneratingSourceFunction(NUM_STRINGS / 2, NUM_STRINGS / 4));

	final DataStream<Integer> stream2 = env.addSource(
			new IntGeneratingSourceFunction(NUM_STRINGS / 2, NUM_STRINGS / 4));

	stream1.union(stream2)
			.keyBy(new IdentityKeySelector<Integer>())
			.map(new OnceFailingPartitionedSum(failurePos))
			.keyBy(0)
			.addSink(new CounterSink());

	env.execute();

	// verify that we counted exactly right
	assertEquals(NUM_KEYS, CounterSink.ALL_COUNTS.size());
	assertEquals(NUM_KEYS, OnceFailingPartitionedSum.ALL_SUMS.size());

	for (Entry<Integer, Long> sum : OnceFailingPartitionedSum.ALL_SUMS.entrySet()) {
		assertEquals((long) sum.getKey() * NUM_STRINGS / NUM_KEYS, sum.getValue().longValue());
	}
	for (long count : CounterSink.ALL_COUNTS.values()) {
		assertEquals(NUM_STRINGS / NUM_KEYS, count);
	}
}
Example #25
Source File: StateBackendBenchmarkUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static RocksDBKeyedStateBackend<Long> createRocksDBKeyedStateBackend(File rootDir) throws IOException {
	File recoveryBaseDir = prepareDirectory(recoveryDirName, rootDir);
	File dbPathFile = prepareDirectory(dbDirName, rootDir);
	ExecutionConfig executionConfig = new ExecutionConfig();
	RocksDBResourceContainer resourceContainer = new RocksDBResourceContainer();
	RocksDBKeyedStateBackendBuilder<Long> builder = new RocksDBKeyedStateBackendBuilder<>(
		"Test",
		Thread.currentThread().getContextClassLoader(),
		dbPathFile,
		resourceContainer,
		stateName -> resourceContainer.getColumnOptions(),
		null,
		LongSerializer.INSTANCE,
		2,
		new KeyGroupRange(0, 1),
		executionConfig,
		new LocalRecoveryConfig(false, new LocalRecoveryDirectoryProviderImpl(recoveryBaseDir, new JobID(), new JobVertexID(), 0)),
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		new CloseableRegistry());
	try {
		return builder.build();
	} catch (Exception e) {
		IOUtils.closeQuietly(resourceContainer);
		throw e;
	}
}
Example #26
Source File: FsStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);

	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		localRecoveryConfig,
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #27
Source File: MemoryStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws BackendBuildingException {

	TaskStateManager taskStateManager = env.getTaskStateManager();
	HeapPriorityQueueSetFactory priorityQueueSetFactory =
		new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128);
	return new HeapKeyedStateBackendBuilder<>(
		kvStateRegistry,
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		taskStateManager.createLocalRecoveryConfig(),
		priorityQueueSetFactory,
		isUsingAsynchronousSnapshots(),
		cancelStreamRegistry).build();
}
Example #28
Source File: StreamExecutionEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * @deprecated Use {@link #setStateBackend(StateBackend)} instead.
 */
@Deprecated
@PublicEvolving
public StreamExecutionEnvironment setStateBackend(AbstractStateBackend backend) {
	this.defaultStateBackend = Preconditions.checkNotNull(backend);
	return this;
}
Example #29
Source File: SourceOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private OperatorStateStore createOperatorStateStore() throws Exception {
	MockEnvironment env = new MockEnvironmentBuilder().build();
	final AbstractStateBackend abstractStateBackend = new MemoryStateBackend();
	CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
	return abstractStateBackend.createOperatorStateBackend(
			env, "test-operator", Collections.emptyList(), cancelStreamRegistry);
}
Example #30
Source File: KvStateServerHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private AbstractKeyedStateBackend<Integer> createKeyedStateBackend(KvStateRegistry registry, int numKeyGroups, AbstractStateBackend abstractBackend, DummyEnvironment dummyEnv) throws java.io.IOException {
	return abstractBackend.createKeyedStateBackend(
		dummyEnv,
		dummyEnv.getJobID(),
		"test_op",
		IntSerializer.INSTANCE,
		numKeyGroups,
		new KeyGroupRange(0, 0),
		registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}