org.apache.flink.runtime.testutils.statemigration.TestType Java Examples

@Test
public void testKeyedListStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testLazilyRegisterIncompatibleSerializer() {
	TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer();
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration());

	// register serializer that requires migration for state
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.registerNewSerializerForRestoredState(new TestType.IncompatibleTestTypeSerializer());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 

@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() throws Exception {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testEagerlyRegisterIncompatibleSerializer() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer());

	// set previous serializer snapshot for state, which should let the new serializer be considered incompatible
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 

@Test
public void testEagerlyRegisterIncompatibleSerializer() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer());

	// set previous serializer snapshot for state, which should let the new serializer be considered incompatible
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 

@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testEagerlyRegisterIncompatibleSerializer() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer());

	// set previous serializer snapshot for state, which should let the new serializer be considered incompatible
	TypeSerializerSchemaCompatibility<TestType> schemaCompatibility =
		testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration());
	assertTrue(schemaCompatibility.isIncompatible());

	try {
		// a serializer for the current schema will no longer be accessible
		testProvider.currentSchemaSerializer();

		fail();
	} catch (Exception excepted) {
		// success
	}
}
 

@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		Assert.fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testBroadcastStateRegistrationFailsIfNewValueSerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateValueUpgrade(
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
				stateName,
				IntSerializer.INSTANCE,
				// new value serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testBroadcastStateRegistrationFailsIfNewKeySerializerIsIncompatible() {
	final String stateName = "broadcast-state";

	try {
		testBroadcastStateKeyUpgrade(
			new MapStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer(),
				IntSerializer.INSTANCE),
			new MapStateDescriptor<>(
				stateName,
				// new key serializer is incompatible
				new TestType.IncompatibleTestTypeSerializer(),
				IntSerializer.INSTANCE));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testOperatorParitionableListStateRegistrationFailsIfNewSerializerIsIncompatible() throws Exception {
	final String stateName = "partitionable-list-state";

	try {
		testOperatorPartitionableListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 

@Test(expected = UnsupportedOperationException.class)
public void testSetSerializerSnapshotWithLazilyRegisteredSerializerProviderShouldFail() {
	TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer();
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration());

	testProvider.setPreviousSerializerSnapshotForRestoredState(serializer.snapshotConfiguration());
}
 

@Test
public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		InternalPriorityQueue<TestType> internalPriorityQueue = backend.create(
			"testPriorityQueue", new TestType.V1TestTypeSerializer());

		internalPriorityQueue.add(new TestType("key-1", 123));
		internalPriorityQueue.add(new TestType("key-2", 346));
		internalPriorityQueue.add(new TestType("key-1", 777));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
		backend.create(
			"testPriorityQueue", new TestType.IncompatibleTestTypeSerializer());

		fail("should have failed");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	} finally {
		backend.dispose();
	}
}
 

@Test
public void testKeyedMapStateAsIs() throws Exception {
	final String stateName = "test-name";

	testKeyedMapStateUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()));
}
 

@Test
public void testKeyedMapStateSerializerReconfiguration() throws Exception {
	final String stateName = "test-name";

	testKeyedMapStateUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// restore with a V2 serializer that has a different schema
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

@Test
public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerIsIncompatible() throws Exception {
	try {
		testNamespaceSerializerUpgrade(
			new TestType.V1TestTypeSerializer(),
			new TestType.IncompatibleTestTypeSerializer());

		fail("should have failed");
	} catch (Exception expected) {
		// the new namespace serializer is incompatible; this should fail the restore
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Test(expected = UnsupportedOperationException.class)
public void testPreviousSchemaSerializerForEagerlyRegisteredStateSerializerProvider() {
	StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.V1TestTypeSerializer());

	// this should fail with an exception
	testProvider.previousSchemaSerializer();
}
 

@Test
public void testOperatorUnionListStateSerializerReconfiguration() throws Exception {
	final String stateName = "union-list-state";

	testOperatorUnionListStateUpgrade(
		new ListStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ListStateDescriptor<>(
			stateName,
			// restore with a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

@Test
public void testKeyedValueStateSerializerReconfiguration() throws Exception {
	final String stateName = "test-name";

	testKeyedValueStateUpgrade(
		new ValueStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ValueStateDescriptor<>(
			stateName,
			// the test fails if this serializer is used instead of a reconfigured new serializer
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

@Test
public void testBroadcastStateValueSerializerReconfiguration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateValueUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// new value serializer is a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

private void testBroadcastStateKeyUpgrade(
		MapStateDescriptor<TestType, Integer> initialAccessDescriptor,
		MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(new TestType("foo", 13), 3);
		state.put(new TestType("bar", 278), 5);

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
		Assert.assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
		state.put(new TestType("new-entry", 777), 17);
	} finally {
		backend.dispose();
	}
}
 

private void testBroadcastStateValueUpgrade(
		MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
		MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();

	OperatorStateBackend backend = createOperatorStateBackend();

	try {
		BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);

		state.put(3, new TestType("foo", 13));
		state.put(5, new TestType("bar", 278));

		OperatorStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
		backend.dispose();

		backend = restoreOperatorStateBackend(snapshot);

		state = backend.getBroadcastState(newAccessDescriptorAfterRestore);

		// the state backend should have decided whether or not it needs to perform state migration;
		// make sure that reading and writing each broadcast entry works with the new serializer
		Assert.assertEquals(new TestType("foo", 13), state.get(3));
		Assert.assertEquals(new TestType("bar", 278), state.get(5));
		state.put(17, new TestType("new-entry", 777));
	} finally {
		backend.dispose();
	}
}
 

@Test
public void testOperatorParitionableListStateSerializerReconfiguration() throws Exception {
	final String stateName = "partitionable-list-state";

	testOperatorPartitionableListStateUpgrade(
		new ListStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ListStateDescriptor<>(
			stateName,
			// restore with a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

@Test
public void testKeyedListStateSerializerReconfiguration() throws Exception {
	final String stateName = "test-name";

	testKeyedListStateUpgrade(
		new ListStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer()),
		new ListStateDescriptor<>(
			stateName,
			// the test fails if this serializer is used instead of a reconfigured new serializer
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

@Test
public void testBroadcastStateKeySerializerReconfiguration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateKeyUpgrade(
		new MapStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer(),
			IntSerializer.INSTANCE),
		new MapStateDescriptor<>(
			stateName,
			// new key serializer is a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer(),
			IntSerializer.INSTANCE));
}
 

@Test
public void testBroadcastStateValueSerializerReconfiguration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateValueUpgrade(
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
		new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// new value serializer is a new serializer that requires reconfiguration
			new TestType.ReconfigurationRequiringTestTypeSerializer()));
}
 

@Test
public void testBroadcastStateKeyMigration() throws Exception {
	final String stateName = "broadcast-state";

	testBroadcastStateKeyUpgrade(
		new MapStateDescriptor<>(
			stateName,
			new TestType.V1TestTypeSerializer(),
			IntSerializer.INSTANCE),
		new MapStateDescriptor<>(
			stateName,
			// new key serializer is a V2 serializer with a different schema
			new TestType.V2TestTypeSerializer(),
			IntSerializer.INSTANCE));
}
 

@Test
public void testKeyedStateRegistrationFailsIfNewNamespaceSerializerRequiresMigration() throws Exception {
	try {
		testNamespaceSerializerUpgrade(
			new TestType.V1TestTypeSerializer(),
			new TestType.V2TestTypeSerializer());

		fail("should have failed");
	} catch (Exception expected) {
		// the new namespace serializer requires migration; this should fail the restore
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}