org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility Java Examples
The following examples show how to use
org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PojoSerializer.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * This legacy snapshot delegates compatibility checks to the {@link PojoSerializerSnapshot}. */ @Override public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) { LinkedHashMap<String, TypeSerializerSnapshot<?>> legacyFieldSerializerSnapshots = preprocessLegacySerializerSnapshotTuples(fieldToSerializerConfigSnapshot); int numFields = legacyFieldSerializerSnapshots.size(); ArrayList<Field> fields = new ArrayList<>(numFields); ArrayList<TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new ArrayList<>(numFields); legacyFieldSerializerSnapshots.forEach((fieldName, fieldSerializerSnapshot) -> { fields.add(PojoFieldUtils.getField(fieldName, getTypeClass())); fieldSerializerSnapshots.add(fieldSerializerSnapshot); }); PojoSerializerSnapshot<T> newSnapshot = new PojoSerializerSnapshot<>( getTypeClass(), fields.toArray(new Field[numFields]), fieldSerializerSnapshots.toArray(new TypeSerializerSnapshot[numFields]), preprocessLegacySerializerSnapshotTuples(registeredSubclassesToSerializerConfigSnapshots), preprocessLegacySerializerSnapshotTuples(nonRegisteredSubclassesToSerializerConfigSnapshots)); return newSnapshot.resolveSchemaCompatibility(newSerializer); }
Example #2
Source File: AvroSerializerSnapshot.java From flink with Apache License 2.0 | 6 votes |
private static <T> TypeSerializerSchemaCompatibility<T> avroCompatibilityToFlinkCompatibility(SchemaPairCompatibility compatibility) { switch (compatibility.getType()) { case COMPATIBLE: { // The new serializer would be able to read data persisted with *this* serializer, therefore no migration // is required. return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); } case INCOMPATIBLE: { return TypeSerializerSchemaCompatibility.incompatible(); } case RECURSION_IN_PROGRESS: default: return TypeSerializerSchemaCompatibility.incompatible(); } }
Example #3
Source File: StateSerializerProvider.java From flink with Apache License 2.0 | 6 votes |
@Nonnull @Override @SuppressWarnings("ConstantConditions") public TypeSerializerSchemaCompatibility<T> registerNewSerializerForRestoredState(TypeSerializer<T> newSerializer) { checkNotNull(newSerializer); if (registeredSerializer != null) { throw new UnsupportedOperationException("A serializer has already been registered for the state; re-registration is not allowed."); } TypeSerializerSchemaCompatibility<T> result = previousSerializerSnapshot.resolveSchemaCompatibility(newSerializer); if (result.isIncompatible()) { invalidateCurrentSchemaSerializerAccess(); } if (result.isCompatibleWithReconfiguredSerializer()) { this.registeredSerializer = result.getReconfiguredSerializer(); } else { this.registeredSerializer = newSerializer; } return result; }
Example #4
Source File: AvroSerializerSnapshot.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private static <T> TypeSerializerSchemaCompatibility<T> avroCompatibilityToFlinkCompatibility(SchemaPairCompatibility compatibility) { switch (compatibility.getType()) { case COMPATIBLE: { // The new serializer would be able to read data persisted with *this* serializer, therefore no migration // is required. return TypeSerializerSchemaCompatibility.compatibleAfterMigration(); } case INCOMPATIBLE: { return TypeSerializerSchemaCompatibility.incompatible(); } case RECURSION_IN_PROGRESS: default: return TypeSerializerSchemaCompatibility.incompatible(); } }
Example #5
Source File: MapSerializerConfigSnapshot.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public TypeSerializerSchemaCompatibility<Map<K, V>> resolveSchemaCompatibility(TypeSerializer<Map<K, V>> newSerializer) { if (newSerializer instanceof MapSerializer) { List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nestedSerializersAndConfigs = getNestedSerializersAndConfigs(); // redirect the compatibility check to the new MapSerializerSnapshot return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, new MapSerializerSnapshot<>(), nestedSerializersAndConfigs.get(0).f1, nestedSerializersAndConfigs.get(1).f1); } else { return super.resolveSchemaCompatibility(newSerializer); } }
Example #6
Source File: PojoSerializerSnapshotTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testResolveSchemaCompatibilityWithCompatibleAfterMigrationFieldSerializers() { final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList( ID_FIELD, NAME_FIELD, mockFieldSerializerSnapshot( HEIGHT_FIELD, SchemaCompatibilityTestingSnapshot.thatIsCompatibleWithNextSerializerAfterMigration()))); final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList( ID_FIELD, NAME_FIELD, mockFieldSerializer(HEIGHT_FIELD, new SchemaCompatibilityTestingSerializer()) )); final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility = testSnapshot.resolveSchemaCompatibility(newPojoSerializer); assertTrue(resultCompatibility.isCompatibleAfterMigration()); }
Example #7
Source File: PojoSerializer.java From flink with Apache License 2.0 | 6 votes |
/** * This legacy snapshot delegates compatibility checks to the {@link PojoSerializerSnapshot}. */ @Override public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) { LinkedHashMap<String, TypeSerializerSnapshot<?>> legacyFieldSerializerSnapshots = preprocessLegacySerializerSnapshotTuples(fieldToSerializerConfigSnapshot); int numFields = legacyFieldSerializerSnapshots.size(); ArrayList<Field> fields = new ArrayList<>(numFields); ArrayList<TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new ArrayList<>(numFields); legacyFieldSerializerSnapshots.forEach((fieldName, fieldSerializerSnapshot) -> { fields.add(PojoFieldUtils.getField(fieldName, getTypeClass())); fieldSerializerSnapshots.add(fieldSerializerSnapshot); }); PojoSerializerSnapshot<T> newSnapshot = new PojoSerializerSnapshot<>( getTypeClass(), fields.toArray(new Field[numFields]), fieldSerializerSnapshots.toArray(new TypeSerializerSnapshot[numFields]), preprocessLegacySerializerSnapshotTuples(registeredSubclassesToSerializerConfigSnapshots), preprocessLegacySerializerSnapshotTuples(nonRegisteredSubclassesToSerializerConfigSnapshots)); return newSnapshot.resolveSchemaCompatibility(newSerializer); }
Example #8
Source File: StateSerializerProviderTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testEagerlyRegisterIncompatibleSerializer() { StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.IncompatibleTestTypeSerializer()); // set previous serializer snapshot for state, which should let the new serializer be considered incompatible TypeSerializerSchemaCompatibility<TestType> schemaCompatibility = testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration()); assertTrue(schemaCompatibility.isIncompatible()); try { // a serializer for the current schema will no longer be accessible testProvider.currentSchemaSerializer(); fail(); } catch (Exception excepted) { // success } }
Example #9
Source File: EitherSerializerSnapshot.java From flink with Apache License 2.0 | 6 votes |
@Override public TypeSerializerSchemaCompatibility<Either<L, R>> resolveSchemaCompatibility( TypeSerializer<Either<L, R>> newSerializer) { checkState(nestedSnapshot != null); if (newSerializer instanceof EitherSerializer) { // delegate compatibility check to the new snapshot class return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, new JavaEitherSerializerSnapshot<>(), nestedSnapshot.getNestedSerializerSnapshots()); } else { return TypeSerializerSchemaCompatibility.incompatible(); } }
Example #10
Source File: RocksDBKeyedStateBackend.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (s.isCompatibleAfterMigration() || s.isIncompatible()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
Example #11
Source File: StateSerializerProviderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testLazilyRegisterNewCompatibleAsIsSerializer() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); // register compatible serializer for state TypeSerializerSchemaCompatibility<TestType> schemaCompatibility = testProvider.registerNewSerializerForRestoredState(new TestType.V1TestTypeSerializer()); assertTrue(schemaCompatibility.isCompatibleAsIs()); assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); }
Example #12
Source File: StateSerializerProviderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testLazilyRegisterNewSerializerRequiringReconfiguration() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); // register serializer that requires reconfiguration, and verify that // the resulting current schema serializer is the reconfigured one TypeSerializerSchemaCompatibility<TestType> schemaCompatibility = testProvider.registerNewSerializerForRestoredState(new TestType.ReconfigurationRequiringTestTypeSerializer()); assertTrue(schemaCompatibility.isCompatibleWithReconfiguredSerializer()); assertTrue(testProvider.currentSchemaSerializer().getClass() == TestType.V1TestTypeSerializer.class); }
Example #13
Source File: StateSerializerProviderTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testLazilyRegisterNewCompatibleAsIsSerializer() { TestType.V1TestTypeSerializer serializer = new TestType.V1TestTypeSerializer(); StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromPreviousSerializerSnapshot(serializer.snapshotConfiguration()); // register compatible serializer for state TypeSerializerSchemaCompatibility<TestType> schemaCompatibility = testProvider.registerNewSerializerForRestoredState(new TestType.V1TestTypeSerializer()); assertTrue(schemaCompatibility.isCompatibleAsIs()); assertTrue(testProvider.currentSchemaSerializer() instanceof TestType.V1TestTypeSerializer); assertTrue(testProvider.previousSchemaSerializer() instanceof TestType.V1TestTypeSerializer); }
Example #14
Source File: DecimalDataSerializer.java From flink with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<DecimalData> resolveSchemaCompatibility(TypeSerializer<DecimalData> newSerializer) { if (!(newSerializer instanceof DecimalDataSerializer)) { return TypeSerializerSchemaCompatibility.incompatible(); } DecimalDataSerializer newDecimalDataSerializer = (DecimalDataSerializer) newSerializer; if (previousPrecision != newDecimalDataSerializer.precision || previousScale != newDecimalDataSerializer.scale) { return TypeSerializerSchemaCompatibility.incompatible(); } else { return TypeSerializerSchemaCompatibility.compatibleAsIs(); } }
Example #15
Source File: TraversableSerializerConfigSnapshot.java From flink with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) { TraversableSerializer<T, E> previousSerializer = (TraversableSerializer<T, E>) restoreSerializer(); TraversableSerializerSnapshot<T, E> newCompositeSnapshot = new TraversableSerializerSnapshot<>(previousSerializer.cbfCode()); return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, newCompositeSnapshot, getSingleNestedSerializerAndConfig().f1 ); }
Example #16
Source File: EncodedValueSerializer.java From beam with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSnapshot<byte[]> snapshotConfiguration() { return new TypeSerializerSnapshot<byte[]>() { @Override public int getCurrentVersion() { return 1; } @Override public void writeSnapshot(DataOutputView out) throws IOException {} @Override public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {} @Override public TypeSerializer<byte[]> restoreSerializer() { return new EncodedValueSerializer(); } @Override public TypeSerializerSchemaCompatibility<byte[]> resolveSchemaCompatibility( TypeSerializer<byte[]> newSerializer) { return newSerializer instanceof EncodedValueSerializer ? TypeSerializerSchemaCompatibility.compatibleAsIs() : TypeSerializerSchemaCompatibility.compatibleAfterMigration(); } }; }
Example #17
Source File: EnumSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testSerializeReconfiguredEnumSerializer() throws Exception { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // verify that after the serializer was read, the reconfigured constant ordering is untouched PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example #18
Source File: PojoSerializerSnapshotTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testResolveSchemaCompatibilityWithCompatibleWithReconfigurationFieldSerializers() { final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList( mockFieldSerializerSnapshot( ID_FIELD, SchemaCompatibilityTestingSnapshot.thatIsCompatibleWithNextSerializerAfterReconfiguration()), NAME_FIELD, HEIGHT_FIELD )); final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Arrays.asList( mockFieldSerializer(ID_FIELD, new SchemaCompatibilityTestingSerializer()), NAME_FIELD, HEIGHT_FIELD )); final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility = testSnapshot.resolveSchemaCompatibility(newPojoSerializer); assertTrue(resultCompatibility.isCompatibleWithReconfiguredSerializer()); final TypeSerializer<TestPojo> reconfiguredSerializer = resultCompatibility.getReconfiguredSerializer(); assertSame(reconfiguredSerializer.getClass(), PojoSerializer.class); final PojoSerializer<TestPojo> reconfiguredPojoSerializer = (PojoSerializer<TestPojo>) reconfiguredSerializer; final TypeSerializer<?>[] reconfiguredFieldSerializers = reconfiguredPojoSerializer.getFieldSerializers(); assertArrayEquals( new TypeSerializer[] { new SchemaCompatibilityTestingSerializer(), StringSerializer.INSTANCE, DoubleSerializer.INSTANCE }, reconfiguredFieldSerializers); }
Example #19
Source File: NFA.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<MigratedNFA<T>> resolveSchemaCompatibility(TypeSerializer<MigratedNFA<T>> newSerializer) { return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, new MigratedNFASerializerSnapshot<>(), getNestedSerializerSnapshots()); }
Example #20
Source File: TimerSerializer.java From flink with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<TimerHeapInternalTimer<K, N>> resolveSchemaCompatibility( TypeSerializer<TimerHeapInternalTimer<K, N>> newSerializer) { final TypeSerializerSnapshot<?>[] nestedSnapshots = getNestedSerializersAndConfigs() .stream() .map(t -> t.f1) .toArray(TypeSerializerSnapshot[]::new); return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, new TimerSerializerSnapshot<>(), nestedSnapshots ); }
Example #21
Source File: StateSerializerProviderTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testEagerlyRegisterNewSerializerRequiringReconfiguration() { StateSerializerProvider<TestType> testProvider = StateSerializerProvider.fromNewRegisteredSerializer(new TestType.ReconfigurationRequiringTestTypeSerializer()); // set previous serializer snapshot, which should let the new serializer be considered to require reconfiguration, // and verify that the resulting current schema serializer is the reconfigured one TypeSerializerSchemaCompatibility<TestType> schemaCompatibility = testProvider.setPreviousSerializerSnapshotForRestoredState(new TestType.V1TestTypeSerializer().snapshotConfiguration()); assertTrue(schemaCompatibility.isCompatibleWithReconfiguredSerializer()); assertTrue(testProvider.currentSchemaSerializer().getClass() == TestType.V1TestTypeSerializer.class); }
Example #22
Source File: EnumSerializerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSerializeReconfiguredEnumSerializer() throws Exception { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // verify that after the serializer was read, the reconfigured constant ordering is untouched PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example #23
Source File: TwoPhaseCommitSinkFunction.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<State<TXN, CONTEXT>> resolveSchemaCompatibility( TypeSerializer<State<TXN, CONTEXT>> newSerializer) { final TypeSerializerSnapshot<?>[] nestedSnapshots = getNestedSerializersAndConfigs() .stream() .map(t -> t.f1) .toArray(TypeSerializerSnapshot[]::new); return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, new StateSerializerSnapshot<>(), nestedSnapshots ); }
Example #24
Source File: EnumSerializerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReconfiguration() { // mock the previous ordering of enum constants to be BAR, PAULA, NATHANIEL PublicEnum[] mockPreviousOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL}; // now, the actual order of FOO, BAR, PETER, NATHANIEL, EMMA, PAULA will be the "new wrong order" EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); // verify that the serializer is first using the "wrong order" (i.e., the initial new configuration) assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); // reconfigure and verify compatibility EnumSerializer.EnumSerializerSnapshot serializerSnapshot = new EnumSerializer.EnumSerializerSnapshot(PublicEnum.class, mockPreviousOrder); TypeSerializerSchemaCompatibility compatibility = serializerSnapshot.resolveSchemaCompatibility(serializer); assertTrue(compatibility.isCompatibleWithReconfiguredSerializer()); // after reconfiguration, the order should be first the original BAR, PAULA, NATHANIEL, // followed by the "new enum constants" FOO, PETER, EMMA PublicEnum[] expectedOrder = {PublicEnum.BAR, PublicEnum.PAULA, PublicEnum.NATHANIEL, PublicEnum.FOO, PublicEnum.PETER, PublicEnum.EMMA}; EnumSerializer<PublicEnum> configuredSerializer = (EnumSerializer<PublicEnum>) compatibility.getReconfiguredSerializer(); int i = 0; for (PublicEnum constant : expectedOrder) { assertEquals(i, configuredSerializer.getValueToOrdinal().get(constant).intValue()); i++; } assertTrue(Arrays.equals(expectedOrder, configuredSerializer.getValues())); }
Example #25
Source File: BinaryRowDataSerializer.java From flink with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<BinaryRowData> resolveSchemaCompatibility( TypeSerializer<BinaryRowData> newSerializer) { if (!(newSerializer instanceof BinaryRowDataSerializer)) { return TypeSerializerSchemaCompatibility.incompatible(); } BinaryRowDataSerializer newBinaryRowSerializer = (BinaryRowDataSerializer) newSerializer; if (previousNumFields != newBinaryRowSerializer.numFields) { return TypeSerializerSchemaCompatibility.incompatible(); } else { return TypeSerializerSchemaCompatibility.compatibleAsIs(); } }
Example #26
Source File: TwoPhaseCommitSinkFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<State<TXN, CONTEXT>> resolveSchemaCompatibility( TypeSerializer<State<TXN, CONTEXT>> newSerializer) { final TypeSerializerSnapshot<?>[] nestedSnapshots = getNestedSerializersAndConfigs() .stream() .map(t -> t.f1) .toArray(TypeSerializerSnapshot[]::new); return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, new StateSerializerSnapshot<>(), nestedSnapshots ); }
Example #27
Source File: KryoSerializerCompatibilityTest.java From flink with Apache License 2.0 | 5 votes |
/** * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed. */ @Test public void testMigrationStrategyWithDifferentKryoType() throws Exception { KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig()); // snapshot configuration and serialize to bytes TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializerForA); serializedConfig = out.toByteArray(); } KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig()); // read configuration again from bytes try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForB); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClassB> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForB); assertTrue(compatResult.isIncompatible()); }
Example #28
Source File: PojoSerializerSnapshotTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testResolveSchemaCompatibilityWithRemovedFields() { final PojoSerializerSnapshot<TestPojo> testSnapshot = buildTestSnapshot(Arrays.asList( mockRemovedField(ID_FIELD), NAME_FIELD, mockRemovedField(HEIGHT_FIELD) )); final PojoSerializer<TestPojo> newPojoSerializer = buildTestNewPojoSerializer(Collections.singletonList(NAME_FIELD)); final TypeSerializerSchemaCompatibility<TestPojo> resultCompatibility = testSnapshot.resolveSchemaCompatibility(newPojoSerializer); assertTrue(resultCompatibility.isCompatibleAfterMigration()); }
Example #29
Source File: CoGroupedStreams.java From flink with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<TaggedUnion<T1, T2>> resolveSchemaCompatibility(TypeSerializer<TaggedUnion<T1, T2>> newSerializer) { List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> nestedSerializersAndConfigs = getNestedSerializersAndConfigs(); return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, new UnionSerializerSnapshot<>(), nestedSerializersAndConfigs.get(0).f1, nestedSerializersAndConfigs.get(1).f1 ); }
Example #30
Source File: TraversableSerializerConfigSnapshot.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(TypeSerializer<T> newSerializer) { TraversableSerializer<T, E> previousSerializer = (TraversableSerializer<T, E>) restoreSerializer(); TraversableSerializerSnapshot<T, E> newCompositeSnapshot = new TraversableSerializerSnapshot<>(previousSerializer.cbfCode()); return CompositeTypeSerializerUtil.delegateCompatibilityCheckToNewSnapshot( newSerializer, newCompositeSnapshot, getSingleNestedSerializerAndConfig().f1 ); }