Java Code Examples for org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility#getReconfiguredSerializer()
The following examples show how to use
org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility#getReconfiguredSerializer() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StateSerializerProvider.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Nonnull @Override @SuppressWarnings("ConstantConditions") public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) { checkNotNull(newSerializer); if (registeredSerializer != null) { throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); } TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); if (result.isIncompatible()) { invalidateCurrentSchemaSerializerAccess(); } if (result.isCompatibleWithReconfiguredSerializer()) { this.registeredSerializer = result.getReconfiguredSerializer(); } else { this.registeredSerializer = newSerializer; } return result; }
Example 2
Source File: StateSerializerProvider.java From flink with Apache License 2.0 | 6 votes |
@Nonnull @Override @SuppressWarnings("ConstantConditions") public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) { checkNotNull(newSerializer); if (registeredSerializer != null) { throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); } TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); if (result.isIncompatible()) { invalidateCurrentSchemaSerializerAccess(); } if (result.isCompatibleWithReconfiguredSerializer()) { this.registeredSerializer = result.getReconfiguredSerializer(); } else { this.registeredSerializer = newSerializer; } return result; }
Example 3
Source File: StateSerializerProvider.java From flink with Apache License 2.0 | 6 votes |
@Nonnull @Override @SuppressWarnings("ConstantConditions") public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) { checkNotNull(newSerializer); if (registeredSerializer != null) { throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); } TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); if (result.isIncompatible()) { invalidateCurrentSchemaSerializerAccess(); } if (result.isCompatibleWithReconfiguredSerializer()) { this.registeredSerializer = result.getReconfiguredSerializer(); } else { this.registeredSerializer = newSerializer; } return result; }
Example 4
Source File: StateSerializerProvider.java From flink with Apache License 2.0 | 6 votes |
@Nonnull @Override public TypeSerializerSchemaCompatibility<T> setPreviousSerializerSnapshotForRestoredState(TypeSerializerSnapshot<T> previousSerializerSnapshot) { checkNotNull(previousSerializerSnapshot); if (this.previousSerializerSnapshot != null) { throw new UnsupportedOperationException("The snapshot of the state's previous serializer has already been set; cannot reset."); } this.previousSerializerSnapshot = previousSerializerSnapshot; TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(registeredSerializer); if (result.isIncompatible()) { invalidateCurrentSchemaSerializerAccess(); } if (result.isCompatibleWithReconfiguredSerializer()) { this.registeredSerializer = result.getReconfiguredSerializer(); } return result; }
Example 5
Source File: EnumSerializerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReconfiguration() { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // after reconfiguration, the order should be first the original BAR, PAULA, NATHANIEL, // followed by the "new enum constants" FOO, PETER, EMMA PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example 6
Source File: EnumSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testReconfiguration() { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // after reconfiguration, the order should be first the original BAR, PAULA, NATHANIEL, // followed by the "new enum constants" FOO, PETER, EMMA PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example 7
Source File: EnumSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testSerializeReconfiguredEnumSerializer() throws Exception { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // verify that after the serializer was read, the reconfigured constant ordering is untouched PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example 8
Source File: EnumSerializerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSerializeReconfiguredEnumSerializer() throws Exception { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // verify that after the serializer was read, the reconfigured constant ordering is untouched PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example 9
Source File: SerializerTestUtil.java From flink with Apache License 2.0 | 5 votes |
/** * Snapshot and restore the given serializer. Returns the restored serializer. */ public static <T> TypeSerializer<T> snapshotAndReconfigure( TypeSerializer<T> serializer, SerializerGetter<T> serializerGetter) throws IOException { TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot, serializer); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<T> restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), serializerGetter.getSerializer()); } TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(serializerGetter.getSerializer()); final TypeSerializer<T> restoredSerializer; if (strategy.isCompatibleAsIs()) { restoredSerializer = restoredConfig.restoreSerializer(); } else if (strategy.isCompatibleWithReconfiguredSerializer()) { restoredSerializer = strategy.getReconfiguredSerializer(); } else { throw new AssertionError("Unable to restore serializer with " + strategy); } assertEquals(serializer.getClass(), restoredSerializer.getClass()); return restoredSerializer; }
Example 10
Source File: SerializerTestUtil.java From flink with Apache License 2.0 | 5 votes |
/** * Snapshot and restore the given serializer. Returns the restored serializer. */ public static <T> TypeSerializer<T> snapshotAndReconfigure( TypeSerializer<T> serializer, SerializerGetter<T> serializerGetter) throws IOException { TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot, serializer); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<T> restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), serializerGetter.getSerializer()); } TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(serializerGetter.getSerializer()); final TypeSerializer<T> restoredSerializer; if (strategy.isCompatibleAsIs()) { restoredSerializer = restoredConfig.restoreSerializer(); } else if (strategy.isCompatibleWithReconfiguredSerializer()) { restoredSerializer = strategy.getReconfiguredSerializer(); } else { throw new AssertionError("Unable to restore serializer with " + strategy); } assertEquals(serializer.getClass(), restoredSerializer.getClass()); return restoredSerializer; }
Example 11
Source File: PojoSerializerSnapshotTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testResolveSchemaCompatibilityWithCompatibleWithReconfigurationFieldSerializers() { final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList( mockFieldSerializerSnapshot( ID_FIELD, SchemaCompatibilityTestingSnapshot.thatIsCompatibleWithNextSerializerAfterReconfiguration()), NAME_FIELD, HEIGHT_FIELD )); final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList( mockFieldSerializer(ID_FIELD, new SchemaCompatibilityTestingSerializer()), NAME_FIELD, HEIGHT_FIELD )); final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility = testSnapshot.resolveSchemaCompatibility(newPojoSerializer); assertTrue(resultCompatibility.isCompatibleWithReconfiguredSerializer()); final TypeSerializer<TestPojo> reconfiguredSerializer = resultCompatibility.getReconfiguredSerializer(); assertSame(reconfiguredSerializer.getClass(), PojoSerializer.class); final PojoSerializer<TestPojo> reconfiguredPojoSerializer = (PojoSerializer<TestPojo>) reconfiguredSerializer; final TypeSerializer<?>[] reconfiguredFieldSerializers = reconfiguredPojoSerializer.getFieldSerializers(); assertArrayEquals( new TypeSerializer[] { new SchemaCompatibilityTestingSerializer(), StringSerializer.INSTANCE, DoubleSerializer.INSTANCE }, reconfiguredFieldSerializers); }
Example 12
Source File: EnumSerializerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReconfiguration() { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // after reconfiguration, the order should be first the original BAR, PAULA, NATHANIEL, // followed by the "new enum constants" FOO, PETER, EMMA PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example 13
Source File: EnumSerializerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSerializeReconfiguredEnumSerializer() throws Exception { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // verify that after the serializer was read, the reconfigured constant ordering is untouched PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example 14
Source File: InternalTimerServiceImpl.java From flink with Apache License 2.0 | 4 votes |
/** * Starts the local {@link InternalTimerServiceImpl} by: * <ol> * <li>Setting the {@code keySerialized} and {@code namespaceSerializer} for the timers it will contain.</li> * <li>Setting the {@code triggerTarget} which contains the action to be performed when a timer fires.</li> * <li>Re-registering timers that were retrieved after recovering from a node failure, if any.</li> * </ol> * This method can be called multiple times, as long as it is called with the same serializers. */ public void startTimerService( TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, Triggerable<K, N> triggerTarget) { if (!isInitialized) { if (keySerializer == null || namespaceSerializer == null) { throw new IllegalArgumentException("The TimersService serializers cannot be null."); } if (this.keySerializer != null || this.namespaceSerializer != null || this.triggerTarget != null) { throw new IllegalStateException("The TimerService has already been initialized."); } // the following is the case where we restore if (restoredTimersSnapshot != null) { TypeSerializerSchemaCompatibility<K> keySerializerCompatibility = restoredTimersSnapshot.getKeySerializerSnapshot().resolveSchemaCompatibility(keySerializer); if (keySerializerCompatibility.isIncompatible() || keySerializerCompatibility.isCompatibleAfterMigration()) { throw new IllegalStateException( "Tried to initialize restored TimerService with new key serializer that requires migration or is incompatible."); } TypeSerializerSchemaCompatibility<N> namespaceSerializerCompatibility = restoredTimersSnapshot.getNamespaceSerializerSnapshot().resolveSchemaCompatibility(namespaceSerializer); restoredTimersSnapshot = null; if (namespaceSerializerCompatibility.isIncompatible() || namespaceSerializerCompatibility.isCompatibleAfterMigration()) { throw new IllegalStateException( "Tried to initialize restored TimerService with new namespace serializer that requires migration or is incompatible."); } this.keySerializer = keySerializerCompatibility.isCompatibleAsIs() ? keySerializer : keySerializerCompatibility.getReconfiguredSerializer(); this.namespaceSerializer = namespaceSerializerCompatibility.isCompatibleAsIs() ? namespaceSerializer : namespaceSerializerCompatibility.getReconfiguredSerializer(); } else { this.keySerializer = keySerializer; this.namespaceSerializer = namespaceSerializer; } this.keyDeserializer = null; this.namespaceDeserializer = null; this.triggerTarget = Preconditions.checkNotNull(triggerTarget); // re-register the restored timers (if any) final InternalTimer<K, N> headTimer = processingTimeTimersQueue.peek(); if (headTimer != null) { nextTimer = processingTimeService.registerTimer(headTimer.getTimestamp(), this::onProcessingTime); } this.isInitialized = true; } else { if (!(this.keySerializer.equals(keySerializer) && this.namespaceSerializer.equals(namespaceSerializer))) { throw new IllegalArgumentException("Already initialized Timer Service " + "tried to be initialized with different key and namespace serializers."); } } }
Example 15
Source File: PojoSerializerTest.java From flink with Apache License 2.0 | 4 votes |
/** * Tests that: * - Previous Pojo serializer did not have registrations, and created cached serializers for subclasses * - On restore, it had those subclasses registered * * In this case, after reconfiguration, the cache should be repopulated, and registrations should * also exist for the subclasses. * * Note: the cache still needs to be repopulated because previous data of those subclasses were * written with the cached serializers. In this case, the repopulated cache has reconfigured serializers * for the subclasses so that previous written data can be read, but the registered serializers * for the subclasses do not necessarily need to be reconfigured since they will only be used to * write new data. */ @Test public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception { // don't register any subclasses at first PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // create cached serializers for SubTestUserClassA and SubTestUserClassB pojoSerializer.getSubclassSerializer(SubTestUserClassA.class); pojoSerializer.getSubclassSerializer(SubTestUserClassB.class); // make sure serializers are in cache assertEquals(2, pojoSerializer.getSubclassSerializerCache().size()); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); // make sure that registrations are empty assertTrue(pojoSerializer.getRegisteredClasses().isEmpty()); assertEquals(0, pojoSerializer.getRegisteredSerializers().length); // snapshot configuration and serialize to bytes TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer); serializedConfig = out.toByteArray(); } // instantiate new PojoSerializer, with new execution config that has the subclass registrations ExecutionConfig newExecutionConfig = new ExecutionConfig(); newExecutionConfig.registerPojoType(SubTestUserClassA.class); newExecutionConfig.registerPojoType(SubTestUserClassB.class); pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig); // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer); } // reconfigure - check reconfiguration result and that // 1) subclass serializer cache is repopulated // 2) registrations also contain the now registered subclasses @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleWithReconfiguredSerializer()); assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer); PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer(); assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size()); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size()); assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class)); assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class)); }
Example 16
Source File: PojoSerializerTest.java From flink with Apache License 2.0 | 4 votes |
/** * Tests that reconfiguration repopulates previously cached subclass serializers. */ @Test public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() throws Exception { // don't register any subclasses PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // create cached serializers for SubTestUserClassA and SubTestUserClassB pojoSerializer.getSubclassSerializer(SubTestUserClassA.class); pojoSerializer.getSubclassSerializer(SubTestUserClassB.class); assertEquals(2, pojoSerializer.getSubclassSerializerCache().size()); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); // snapshot configuration and serialize to bytes TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer); serializedConfig = out.toByteArray(); } // instantiate new PojoSerializer pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer); } // reconfigure - check reconfiguration result and that subclass serializer cache is repopulated @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleWithReconfiguredSerializer()); assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer); PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer(); assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size()); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); }
Example 17
Source File: InternalTimerServiceImpl.java From flink with Apache License 2.0 | 4 votes |
/** * Starts the local {@link InternalTimerServiceImpl} by: * <ol> * <li>Setting the {@code keySerialized} and {@code namespaceSerializer} for the timers it will contain.</li> * <li>Setting the {@code triggerTarget} which contains the action to be performed when a timer fires.</li> * <li>Re-registering timers that were retrieved after recovering from a node failure, if any.</li> * </ol> * This method can be called multiple times, as long as it is called with the same serializers. */ public void startTimerService( TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, Triggerable<K, N> triggerTarget) { if (!isInitialized) { if (keySerializer == null || namespaceSerializer == null) { throw new IllegalArgumentException("The TimersService serializers cannot be null."); } if (this.keySerializer != null || this.namespaceSerializer != null || this.triggerTarget != null) { throw new IllegalStateException("The TimerService has already been initialized."); } // the following is the case where we restore if (restoredTimersSnapshot != null) { TypeSerializerSchemaCompatibility<K> keySerializerCompatibility = restoredTimersSnapshot.getKeySerializerSnapshot().resolveSchemaCompatibility(keySerializer); if (keySerializerCompatibility.isIncompatible() || keySerializerCompatibility.isCompatibleAfterMigration()) { throw new IllegalStateException( "Tried to initialize restored TimerService with new key serializer that requires migration or is incompatible."); } TypeSerializerSchemaCompatibility<N> namespaceSerializerCompatibility = restoredTimersSnapshot.getNamespaceSerializerSnapshot().resolveSchemaCompatibility(namespaceSerializer); if (namespaceSerializerCompatibility.isIncompatible() || namespaceSerializerCompatibility.isCompatibleAfterMigration()) { throw new IllegalStateException( "Tried to initialize restored TimerService with new namespace serializer that requires migration or is incompatible."); } this.keySerializer = keySerializerCompatibility.isCompatibleAsIs() ? keySerializer : keySerializerCompatibility.getReconfiguredSerializer(); this.namespaceSerializer = namespaceSerializerCompatibility.isCompatibleAsIs() ? namespaceSerializer : namespaceSerializerCompatibility.getReconfiguredSerializer(); } else { this.keySerializer = keySerializer; this.namespaceSerializer = namespaceSerializer; } this.keyDeserializer = null; this.namespaceDeserializer = null; this.triggerTarget = Preconditions.checkNotNull(triggerTarget); // re-register the restored timers (if any) final InternalTimer<K, N> headTimer = processingTimeTimersQueue.peek(); if (headTimer != null) { nextTimer = processingTimeService.registerTimer(headTimer.getTimestamp(), this); } this.isInitialized = true; } else { if (!(this.keySerializer.equals(keySerializer) && this.namespaceSerializer.equals(namespaceSerializer))) { throw new IllegalArgumentException("Already initialized Timer Service " + "tried to be initialized with different key and namespace serializers."); } } }
Example 18
Source File: PojoSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Tests that reconfiguration repopulates previously cached subclass serializers. */ @Test public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() throws Exception { // don't register any subclasses PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // create cached serializers for SubTestUserClassA and SubTestUserClassB pojoSerializer.getSubclassSerializer(SubTestUserClassA.class); pojoSerializer.getSubclassSerializer(SubTestUserClassB.class); assertEquals(2, pojoSerializer.getSubclassSerializerCache().size()); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); // snapshot configuration and serialize to bytes TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer); serializedConfig = out.toByteArray(); } // instantiate new PojoSerializer pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer); } // reconfigure - check reconfiguration result and that subclass serializer cache is repopulated @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleWithReconfiguredSerializer()); assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer); PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer(); assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size()); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); }
Example 19
Source File: PojoSerializerTest.java From flink with Apache License 2.0 | 4 votes |
/** * Tests that reconfiguration repopulates previously cached subclass serializers. */ @Test public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() throws Exception { // don't register any subclasses PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // create cached serializers for SubTestUserClassA and SubTestUserClassB pojoSerializer.getSubclassSerializer(SubTestUserClassA.class); pojoSerializer.getSubclassSerializer(SubTestUserClassB.class); assertEquals(2, pojoSerializer.getSubclassSerializerCache().size()); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); // snapshot configuration and serialize to bytes TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer); serializedConfig = out.toByteArray(); } // instantiate new PojoSerializer pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer); } // reconfigure - check reconfiguration result and that subclass serializer cache is repopulated @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleWithReconfiguredSerializer()); assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer); PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer(); assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size()); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); }
Example 20
Source File: PojoSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Tests that: * - Previous Pojo serializer did not have registrations, and created cached serializers for subclasses * - On restore, it had those subclasses registered * * In this case, after reconfiguration, the cache should be repopulated, and registrations should * also exist for the subclasses. * * Note: the cache still needs to be repopulated because previous data of those subclasses were * written with the cached serializers. In this case, the repopulated cache has reconfigured serializers * for the subclasses so that previous written data can be read, but the registered serializers * for the subclasses do not necessarily need to be reconfigured since they will only be used to * write new data. */ @Test public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception { // don't register any subclasses at first PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig()); // create cached serializers for SubTestUserClassA and SubTestUserClassB pojoSerializer.getSubclassSerializer(SubTestUserClassA.class); pojoSerializer.getSubclassSerializer(SubTestUserClassB.class); // make sure serializers are in cache assertEquals(2, pojoSerializer.getSubclassSerializerCache().size()); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); // make sure that registrations are empty assertTrue(pojoSerializer.getRegisteredClasses().isEmpty()); assertEquals(0, pojoSerializer.getRegisteredSerializers().length); // snapshot configuration and serialize to bytes TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer); serializedConfig = out.toByteArray(); } // instantiate new PojoSerializer, with new execution config that has the subclass registrations ExecutionConfig newExecutionConfig = new ExecutionConfig(); newExecutionConfig.registerPojoType(SubTestUserClassA.class); newExecutionConfig.registerPojoType(SubTestUserClassB.class); pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig); // read configuration from bytes try(ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer); } // reconfigure - check reconfiguration result and that // 1) subclass serializer cache is repopulated // 2) registrations also contain the now registered subclasses @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer); assertTrue(compatResult.isCompatibleWithReconfiguredSerializer()); assertTrue(compatResult.getReconfiguredSerializer() instanceof PojoSerializer); PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer(); assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size()); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class)); assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class)); assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size()); assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class)); assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class)); }