Java Code Examples for org.apache.flink.runtime.checkpoint.OperatorState#putState()

The following examples show how to use org.apache.flink.runtime.checkpoint.OperatorState#putState() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobMasterTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private Collection<OperatorState> createOperatorState(OperatorID... operatorIds) {
	Collection<OperatorState> operatorStates = new ArrayList<>(operatorIds.length);

	for (OperatorID operatorId : operatorIds) {
		final OperatorState operatorState = new OperatorState(operatorId, 1, 42);
		final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
			new OperatorStreamStateHandle(
				Collections.emptyMap(),
				new ByteStreamStateHandle("foobar", new byte[0])),
			null,
			null,
			null);
		operatorState.putState(0, subtaskState);
		operatorStates.add(operatorState);
	}

	return operatorStates;
}
Example 2
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadState() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new ReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 2, 3), data);
}
Example 3
Source File: JobMasterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private Collection<OperatorState> createOperatorState(OperatorID... operatorIds) {
	Collection<OperatorState> operatorStates = new ArrayList<>(operatorIds.length);

	for (OperatorID operatorId : operatorIds) {
		final OperatorState operatorState = new OperatorState(operatorId, 1, 42);
		final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
			new OperatorStreamStateHandle(
				Collections.emptyMap(),
				new ByteStreamStateHandle("foobar", new byte[0])),
			null,
			null,
			null);
		operatorState.putState(0, subtaskState);
		operatorStates.add(operatorState);
	}

	return operatorStates;
}
Example 4
Source File: MetadataV3Serializer.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected OperatorState deserializeOperatorState(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
	final OperatorID jobVertexId = new OperatorID(dis.readLong(), dis.readLong());
	final int parallelism = dis.readInt();
	final int maxParallelism = dis.readInt();

	final OperatorState operatorState = new OperatorState(jobVertexId, parallelism, maxParallelism);

	// Coordinator state
	operatorState.setCoordinatorState(deserializeAndCheckByteStreamStateHandle(dis, context));

	// Sub task states
	final int numSubTaskStates = dis.readInt();

	for (int j = 0; j < numSubTaskStates; j++) {
		final int subtaskIndex = dis.readInt();
		final OperatorSubtaskState subtaskState = deserializeSubtaskState(dis, context);
		operatorState.putState(subtaskIndex, subtaskState);
	}

	return operatorState;
}
Example 5
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadTime() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new KeyedProcessOperator<>(new StatefulFunctionWithTime()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new TimeReaderFunction());
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new TimeReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}
Example 6
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IOException.class)
public void testInvalidProcessReaderFunctionFails() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new InvalidReaderFunction();

	readInputSplit(split, userFunction);

	Assert.fail("KeyedStateReaderFunction did not fail on invalid RuntimeContext use");
}
Example 7
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadMultipleOutputPerKey() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new DoubleReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}
Example 8
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadState() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new ReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 2, 3), data);
}
Example 9
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadTime() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new KeyedProcessOperator<>(new StatefulFunctionWithTime()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new TimerReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new TimerReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}
Example 10
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IOException.class)
public void testInvalidProcessReaderFunctionFails() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new InvalidReaderFunction();

	readInputSplit(split, userFunction);

	Assert.fail("KeyedStateReaderFunction did not fail on invalid RuntimeContext use");
}
Example 11
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadMultipleOutputPerKey() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new DoubleReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}
Example 12
Source File: UnionStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReadUnionOperatorState() throws Exception {
	try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) {
		testHarness.open();

		testHarness.processElement(1, 0);
		testHarness.processElement(2, 0);
		testHarness.processElement(3, 0);

		OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0);
		OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4);
		state.putState(0, subtaskState);

		OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0);

		UnionStateInputFormat<Integer> format = new UnionStateInputFormat<>(state, descriptor);

		format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
		format.open(split);

		List<Integer> results = new ArrayList<>();

		while (!format.reachedEnd()) {
			results.add(format.nextRecord(0));
		}

		results.sort(Comparator.naturalOrder());

		Assert.assertEquals("Failed to read correct list state from state backend", Arrays.asList(1, 2, 3), results);
	}
}
Example 13
Source File: ListStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReadListOperatorState() throws Exception {
	try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) {
		testHarness.open();

		testHarness.processElement(1, 0);
		testHarness.processElement(2, 0);
		testHarness.processElement(3, 0);

		OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0);
		OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4);
		state.putState(0, subtaskState);

		OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0);

		ListStateInputFormat<Integer> format = new ListStateInputFormat<>(state, descriptor);

		format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
		format.open(split);

		List<Integer> results = new ArrayList<>();

		while (!format.reachedEnd()) {
			results.add(format.nextRecord(0));
		}

		results.sort(Comparator.naturalOrder());

		Assert.assertEquals(
			"Failed to read correct list state from state backend",
			Arrays.asList(1, 2, 3),
			results);
	}
}
Example 14
Source File: BroadcastStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReadBroadcastState() throws Exception {
	try (TwoInputStreamOperatorTestHarness<Void, Integer, Void> testHarness = getTestHarness()) {
		testHarness.open();

		testHarness.processElement2(new StreamRecord<>(1));
		testHarness.processElement2(new StreamRecord<>(2));
		testHarness.processElement2(new StreamRecord<>(3));

		OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0);
		OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4);
		state.putState(0, subtaskState);

		OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0);

		BroadcastStateInputFormat<Integer, Integer> format = new BroadcastStateInputFormat<>(state, descriptor);

		format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
		format.open(split);

		Map<Integer, Integer> results = new HashMap<>(3);

		while (!format.reachedEnd()) {
			Tuple2<Integer, Integer> entry = format.nextRecord(null);
			results.put(entry.f0, entry.f1);
		}

		Map<Integer, Integer> expected = new HashMap<>(3);
		expected.put(1, 1);
		expected.put(2, 2);
		expected.put(3, 3);

		Assert.assertEquals("Failed to read correct list state from state backend", expected, results);
	}
}
Example 15
Source File: OperatorSubtaskStateReducer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void reduce(Iterable<TaggedOperatorSubtaskState> values, Collector<OperatorState> out) {
	List<TaggedOperatorSubtaskState> subtasks = StreamSupport
		.stream(values.spliterator(), false)
		.collect(Collectors.toList());

	OperatorState operatorState = new OperatorState(operatorID, subtasks.size(), maxParallelism);

	for (TaggedOperatorSubtaskState value : subtasks) {
		operatorState.putState(value.index, value.state);
	}

	out.collect(operatorState);
}
Example 16
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCreatePartitionedInputSplits() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit[] splits = format.createInputSplits(4);
	Assert.assertEquals("Failed to properly partition operator state into input splits", 4, splits.length);
}
Example 17
Source File: BroadcastStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReadBroadcastState() throws Exception {
	try (TwoInputStreamOperatorTestHarness<Void, Integer, Void> testHarness = getTestHarness()) {
		testHarness.open();

		testHarness.processElement2(new StreamRecord<>(1));
		testHarness.processElement2(new StreamRecord<>(2));
		testHarness.processElement2(new StreamRecord<>(3));

		OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0);
		OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4);
		state.putState(0, subtaskState);

		OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0);

		BroadcastStateInputFormat<Integer, Integer> format = new BroadcastStateInputFormat<>(state, descriptor);

		format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
		format.open(split);

		Map<Integer, Integer> results = new HashMap<>(3);

		while (!format.reachedEnd()) {
			Tuple2<Integer, Integer> entry = format.nextRecord(null);
			results.put(entry.f0, entry.f1);
		}

		Map<Integer, Integer> expected = new HashMap<>(3);
		expected.put(1, 1);
		expected.put(2, 2);
		expected.put(3, 3);

		Assert.assertEquals("Failed to read correct list state from state backend", expected, results);
	}
}
Example 18
Source File: SavepointOutputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private SavepointV2 createSavepoint() {
	OperatorState operatorState = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 128);

	operatorState.putState(0, new OperatorSubtaskState());
	return new SavepointV2(0, Collections.singleton(operatorState), Collections.emptyList());
}
Example 19
Source File: SavepointV2Serializer.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public SavepointV2 deserialize(DataInputStream dis, ClassLoader cl) throws IOException {
	// first: checkpoint ID
	final long checkpointId = dis.readLong();
	if (checkpointId < 0) {
		throw new IOException("invalid checkpoint ID: " + checkpointId);
	}

	// second: master state
	final List<MasterState> masterStates;
	final int numMasterStates = dis.readInt();

	if (numMasterStates == 0) {
		masterStates = Collections.emptyList();
	}
	else if (numMasterStates > 0) {
		masterStates = new ArrayList<>(numMasterStates);
		for (int i = 0; i < numMasterStates; i++) {
			masterStates.add(deserializeMasterState(dis));
		}
	}
	else {
		throw new IOException("invalid number of master states: " + numMasterStates);
	}

	// third: operator states
	int numTaskStates = dis.readInt();
	List<OperatorState> operatorStates = new ArrayList<>(numTaskStates);

	for (int i = 0; i < numTaskStates; i++) {
		OperatorID jobVertexId = new OperatorID(dis.readLong(), dis.readLong());
		int parallelism = dis.readInt();
		int maxParallelism = dis.readInt();
		int chainLength = dis.readInt();

		// Add task state
		OperatorState taskState = new OperatorState(jobVertexId, parallelism, maxParallelism);
		operatorStates.add(taskState);

		// Sub task states
		int numSubTaskStates = dis.readInt();

		for (int j = 0; j < numSubTaskStates; j++) {
			int subtaskIndex = dis.readInt();

			OperatorSubtaskState subtaskState = deserializeSubtaskState(dis);
			taskState.putState(subtaskIndex, subtaskState);
		}
	}

	return new SavepointV2(checkpointId, operatorStates, masterStates);
}
Example 20
Source File: SavepointOutputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private CheckpointMetadata createSavepoint() {
	OperatorState operatorState = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 128);

	operatorState.putState(0, new OperatorSubtaskState());
	return new CheckpointMetadata(0, Collections.singleton(operatorState), Collections.emptyList());
}