org.apache.flink.core.memory.DataInputViewStreamWrapper Java Examples
The following examples show how to use
org.apache.flink.core.memory.DataInputViewStreamWrapper.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: VersionedIOWriteableTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testReadCompatibleVersion() throws Exception { String payload = "test"; TestWriteable testWriteable = new TestWriteable(1, payload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { testWriteable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } testWriteable = new TestWriteable(2) { @Override public int[] getCompatibleVersions() { return new int[] {1, 2}; } }; try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { testWriteable.read(new DataInputViewStreamWrapper(in)); } Assert.assertEquals(payload, testWriteable.getData()); }
Example #2
Source File: StateInitializationContextImplTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void getOperatorStateStore() throws Exception { Set<Integer> readStatesCount = new HashSet<>(); for (StatePartitionStreamProvider statePartitionStreamProvider : initializationContext.getRawOperatorStateInputs()) { Assert.assertNotNull(statePartitionStreamProvider); try (InputStream is = statePartitionStreamProvider.getStream()) { DataInputView div = new DataInputViewStreamWrapper(is); Assert.assertTrue(readStatesCount.add(div.readInt())); } } Assert.assertEquals(writtenOperatorStates, readStatesCount); }
Example #3
Source File: OperatorStateOutputCheckpointStreamTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private static void verifyRead(OperatorStateHandle fullHandle, int numPartitions) throws IOException { int count = 0; try (FSDataInputStream in = fullHandle.openInputStream()) { OperatorStateHandle.StateMetaInfo metaInfo = fullHandle.getStateNameToPartitionOffsets(). get(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME); long[] offsets = metaInfo.getOffsets(); Assert.assertNotNull(offsets); DataInputView div = new DataInputViewStreamWrapper(in); for (int i = 0; i < numPartitions; ++i) { in.seek(offsets[i]); Assert.assertEquals(i, div.readInt()); ++count; } } Assert.assertEquals(numPartitions, count); }
Example #4
Source File: FoldApplyProcessAllWindowFunction.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public void open(Configuration configuration) throws Exception { FunctionUtils.openFunction(this.windowFunction, configuration); if (serializedInitialValue == null) { throw new RuntimeException("No initial value was serialized for the fold " + "window function. Probably the setOutputType method was not called."); } ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue); DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); initialValue = accSerializer.deserialize(in); ctx = new InternalProcessApplyAllWindowContext<>(windowFunction); }
Example #5
Source File: RocksDBKeySerializationUtilsTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testKeyGroupSerializationAndDeserialization() throws Exception { ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8); DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream); for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) { for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) { outputStream.reset(); RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView); int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup( keyGroupPrefixBytes, new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray()))); Assert.assertEquals(orgKeyGroup, deserializedKeyGroup); } } }
Example #6
Source File: StateInitializationContextImplTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void getKeyedStateStreams() throws Exception { int readKeyGroupCount = 0; for (KeyGroupStatePartitionStreamProvider stateStreamProvider : initializationContext.getRawKeyedStateInputs()) { Assert.assertNotNull(stateStreamProvider); try (InputStream is = stateStreamProvider.getStream()) { DataInputView div = new DataInputViewStreamWrapper(is); int val = div.readInt(); ++readKeyGroupCount; Assert.assertEquals(stateStreamProvider.getKeyGroupId(), val); } } Assert.assertEquals(writtenKeyGroups, readKeyGroupCount); }
Example #7
Source File: StringValueSerializationTest.java From flink with Apache License 2.0 | 6 votes |
public static void testSerialization(String[] values) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); DataOutputViewStreamWrapper serializer = new DataOutputViewStreamWrapper(baos); for (String value : values) { StringValue sv = new StringValue(value); sv.write(serializer); } serializer.close(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); DataInputViewStreamWrapper deserializer = new DataInputViewStreamWrapper(bais); int num = 0; while (bais.available() > 0) { StringValue deser = new StringValue(); deser.read(deserializer); assertEquals("DeserializedString differs from original string.", values[num], deser.getValue()); num++; } assertEquals("Wrong number of deserialized values", values.length, num); }
Example #8
Source File: CollectionInputFormat.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); int collectionLength = in.readInt(); List<T> list = new ArrayList<T>(collectionLength); if (collectionLength > 0) { try { DataInputViewStreamWrapper wrapper = new DataInputViewStreamWrapper(in); for (int i = 0; i < collectionLength; i++){ T element = serializer.deserialize(wrapper); list.add(element); } } catch (Throwable t) { throw new IOException("Error while deserializing element from collection", t); } } dataSet = list; }
Example #9
Source File: FoldApplyAllWindowFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public void open(Configuration configuration) throws Exception { super.open(configuration); if (accSerializer == null) { throw new RuntimeException("No serializer set for the fold accumulator type. " + "Probably the setOutputType method was not called."); } if (serializedInitialValue == null) { throw new RuntimeException("No initial value was serialized for the fold " + "window function. Probably the setOutputType method was not called."); } ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue); DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); initialValue = accSerializer.deserialize(in); }
Example #10
Source File: StringSerializationBenchmark.java From flink-benchmarks with Apache License 2.0 | 6 votes |
@Setup public void setup() throws IOException { length = Integer.parseInt(lengthStr); switch (type) { case "ascii": input = generate(asciiChars, length); break; case "russian": input = generate(russianChars, length); break; case "chinese": input = generate(chineseChars, length); break; default: throw new IllegalArgumentException(type + "charset is not supported"); } byte[] stringBytes = stringWrite(); serializedBuffer = new ByteArrayInputStream(stringBytes); serializedStream = new DataInputViewStreamWrapper(serializedBuffer); }
Example #11
Source File: EventWithAggregatorsTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); event.write(new DataOutputViewStreamWrapper(baos)); byte[] data = baos.toByteArray(); baos.close(); DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data)); IterationEventWithAggregators newEvent = event.getClass().newInstance(); newEvent.read(in); in.close(); return newEvent; } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail("Test threw an exception: " + e.getMessage()); return null; } }
Example #12
Source File: KryoSerializerCompatibilityTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testMigrationStrategyForRemovedAvroDependency() throws Exception { KryoSerializer<TestClass> kryoSerializerForA = new KryoSerializer<>(TestClass.class, new ExecutionConfig()); // read configuration again from bytes TypeSerializerSnapshot kryoSerializerConfigSnapshot; try (InputStream in = getClass().getResourceAsStream("/kryo-serializer-flink1.3-snapshot")) { kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForA); } @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClass> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForA); assertTrue(compatResult.isCompatibleAsIs()); }
Example #13
Source File: EventWithAggregatorsTest.java From flink with Apache License 2.0 | 6 votes |
private IterationEventWithAggregators pipeThroughSerialization(IterationEventWithAggregators event) { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); event.write(new DataOutputViewStreamWrapper(baos)); byte[] data = baos.toByteArray(); baos.close(); DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(new ByteArrayInputStream(data)); IterationEventWithAggregators newEvent = event.getClass().newInstance(); newEvent.read(in); in.close(); return newEvent; } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail("Test threw an exception: " + e.getMessage()); return null; } }
Example #14
Source File: TypeSerializerSerializationUtilTest.java From flink with Apache License 2.0 | 6 votes |
/** * Verifies that reading and writing serializers work correctly. */ @Test public void testSerializerSerialization() throws Exception { TypeSerializer<?> serializer = IntSerializer.INSTANCE; byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer); serialized = out.toByteArray(); } TypeSerializer<?> deserializedSerializer; try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } Assert.assertEquals(serializer, deserializedSerializer); }
Example #15
Source File: ValueStateValueReader.java From bravo with Apache License 2.0 | 6 votes |
@Override public void flatMap(KeyedStateRow row, Collector<V> out) throws Exception { if (!stateName.equals(row.getStateName())) { return; } byte[] valueBytes = row.getValueBytes(); V value = null; try (ByteArrayInputStreamWithPos valIs = new ByteArrayInputStreamWithPos(valueBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(valIs); skipTimestampIfTtlEnabled(iw); value = valueDeserializer.deserialize(iw); } if (value == null) { throw new RuntimeException("MapStates with null values are not supported at the moment."); } else { out.collect(value); } }
Example #16
Source File: StringValueSerializationTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public static void testSerialization(String[] values) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); DataOutputViewStreamWrapper serializer = new DataOutputViewStreamWrapper(baos); for (String value : values) { StringValue sv = new StringValue(value); sv.write(serializer); } serializer.close(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); DataInputViewStreamWrapper deserializer = new DataInputViewStreamWrapper(bais); int num = 0; while (bais.available() > 0) { StringValue deser = new StringValue(); deser.read(deserializer); assertEquals("DeserializedString differs from original string.", values[num], deser.getValue()); num++; } assertEquals("Wrong number of deserialized values", values.length, num); }
Example #17
Source File: PostVersionedIOReadableWritable.java From flink with Apache License 2.0 | 6 votes |
/** * This read attempts to first identify if the input view contains the special * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes. * If identified to be versioned, the usual version resolution read path * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked. * Otherwise, we "reset" the input stream by pushing back the read buffered bytes * into the stream. */ public final void read(InputStream inputStream) throws IOException { byte[] tmp = new byte[VERSIONED_IDENTIFIER.length]; inputStream.read(tmp); if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) { DataInputView inputView = new DataInputViewStreamWrapper(inputStream); super.read(inputView); read(inputView, true); } else { PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length); resetStream.unread(tmp); read(new DataInputViewStreamWrapper(resetStream), false); } }
Example #18
Source File: TypeSerializerSerializationUtilTest.java From flink with Apache License 2.0 | 6 votes |
/** * Verifies that reading and writing configuration snapshots work correctly. */ @Test public void testSerializeConfigurationSnapshots() throws Exception { TypeSerializerSerializationUtilTest.TestConfigSnapshot<String> configSnapshot1 = new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(1, "foo"); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot1, StringSerializer.INSTANCE); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<?> restoredConfigs; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfigs = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), null); } assertEquals(configSnapshot1, restoredConfigs); }
Example #19
Source File: TypeSerializerSerializationUtilTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Verifies that deserializing config snapshots fail if the config class could not be found. */ @Test(expected = IOException.class) public void testFailsWhenConfigurationSnapshotClassNotFound() throws Exception { byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(123, "foobar"), StringSerializer.INSTANCE); serializedConfig = out.toByteArray(); } try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { // read using a dummy classloader TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), new URLClassLoader(new URL[0], null), null); } fail("Expected a ClassNotFoundException wrapped in IOException"); }
Example #20
Source File: PostVersionedIOReadableWritable.java From flink with Apache License 2.0 | 6 votes |
/** * This read attempts to first identify if the input view contains the special * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes. * If identified to be versioned, the usual version resolution read path * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked. * Otherwise, we "reset" the input stream by pushing back the read buffered bytes * into the stream. */ public final void read(InputStream inputStream) throws IOException { byte[] tmp = new byte[VERSIONED_IDENTIFIER.length]; inputStream.read(tmp); if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) { DataInputView inputView = new DataInputViewStreamWrapper(inputStream); super.read(inputView); read(inputView, true); } else { PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length); resetStream.unread(tmp); read(new DataInputViewStreamWrapper(resetStream), false); } }
Example #21
Source File: TypeSerializerSerializationUtilTest.java From flink with Apache License 2.0 | 6 votes |
/** * Verifies that reading and writing configuration snapshots work correctly. */ @Test public void testSerializeConfigurationSnapshots() throws Exception { TypeSerializerSerializationUtilTest.TestConfigSnapshot<String> configSnapshot1 = new TypeSerializerSerializationUtilTest.TestConfigSnapshot<>(1, "foo"); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot1, StringSerializer.INSTANCE); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<?> restoredConfigs; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfigs = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), null); } assertEquals(configSnapshot1, restoredConfigs); }
Example #22
Source File: InstantiationUtil.java From flink with Apache License 2.0 | 6 votes |
/** * Clones the given writable using the {@link IOReadableWritable serialization}. * * @param original Object to clone * @param <T> Type of the object to clone * @return Cloned object * @throws IOException Thrown is the serialization fails. */ public static <T extends IOReadableWritable> T createCopyWritable(T original) throws IOException { if (original == null) { return null; } final ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) { original.write(out); } final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); try (DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) { @SuppressWarnings("unchecked") T copy = (T) instantiate(original.getClass()); copy.read(in); return copy; } }
Example #23
Source File: OperatorStateRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private <K, V> void deserializeBroadcastStateValues( final BackendWritableBroadcastState<K, V> broadcastStateForName, final FSDataInputStream in, final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception { if (metaInfo != null) { long[] offsets = metaInfo.getOffsets(); if (offsets != null) { TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer(); TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer(); in.seek(offsets[0]); DataInputView div = new DataInputViewStreamWrapper(in); int size = div.readInt(); for (int i = 0; i < size; i++) { broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div)); } } } }
Example #24
Source File: StateDescriptor.java From flink with Apache License 2.0 | 5 votes |
private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException { // read the non-transient fields in.defaultReadObject(); // read the default value field boolean hasDefaultValue = in.readBoolean(); if (hasDefaultValue) { TypeSerializer<T> serializer = serializerAtomicReference.get(); checkNotNull(serializer, "Serializer not initialized."); int size = in.readInt(); byte[] buffer = new byte[size]; in.readFully(buffer); try (ByteArrayInputStream bais = new ByteArrayInputStream(buffer); DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(bais)) { defaultValue = serializer.deserialize(inView); } catch (Exception e) { throw new IOException("Unable to deserialize default value.", e); } } else { defaultValue = null; } }
Example #25
Source File: HeapRestoreOperation.java From flink with Apache License 2.0 | 5 votes |
private void readStateHandleStateData( FSDataInputStream fsDataInputStream, DataInputViewStreamWrapper inView, KeyGroupRangeOffsets keyGroupOffsets, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int numStates, int readVersion, boolean isCompressed) throws IOException { final StreamCompressionDecorator streamCompressionDecorator = isCompressed ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1; // Check that restored key groups all belong to the backend. Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend."); fsDataInputStream.seek(offset); int writtenKeyGroupIndex = inView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex, "Unexpected key-group in restore."); try (InputStream kgCompressionInStream = streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) { readKeyGroupStateData( kgCompressionInStream, kvStatesById, keyGroupIndex, numStates, readVersion); } } }
Example #26
Source File: SerializerTestBase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSnapshotConfigurationAndReconfigure() throws Exception { final TypeSerializer<T> serializer = getSerializer(); final TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration(); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), configSnapshot, serializer); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<T> restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), getSerializer()); } TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(getSerializer()); final TypeSerializer<T> restoreSerializer; if (strategy.isCompatibleAsIs()) { restoreSerializer = restoredConfig.restoreSerializer(); } else if (strategy.isCompatibleWithReconfiguredSerializer()) { restoreSerializer = strategy.getReconfiguredSerializer(); } else { throw new AssertionError("Unable to restore serializer with " + strategy); } assertEquals(serializer.getClass(), restoreSerializer.getClass()); }
Example #27
Source File: RocksDBFullRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Restore one key groups state handle. */ private void restoreKeyGroupsInStateHandle() throws IOException, StateMigrationException, RocksDBException { try { currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(currentStateHandleInStream); currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream); restoreKVStateMetaData(); restoreKVStateData(); } finally { if (cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) { IOUtils.closeQuietly(currentStateHandleInStream); } } }
Example #28
Source File: Kafka09ITCase.java From flink with Apache License 2.0 | 5 votes |
@Override public String deserialize(byte[] messageKey, byte[] message, String topic, int partition, long offset) throws IOException { cnt++; DataInputView in = new DataInputViewStreamWrapper(new ByteArrayInputStream(message)); String e = ser.deserialize(in); return e; }
Example #29
Source File: EnumSerializerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testConfigurationSnapshotSerialization() throws Exception { EnumSerializer<PublicEnum> serializer = new EnumSerializer<>(PublicEnum.class); byte[] serializedConfig; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( new DataOutputViewStreamWrapper(out), serializer.snapshotConfiguration(), serializer); serializedConfig = out.toByteArray(); } TypeSerializerSnapshot<PublicEnum> restoredConfig; try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) { restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), serializer); } TypeSerializerSchemaCompatibility<PublicEnum> compatResult = restoredConfig.resolveSchemaCompatibility(serializer); assertTrue(compatResult.isCompatibleAsIs()); assertEquals(PublicEnum.FOO.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue()); assertEquals(PublicEnum.BAR.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.BAR).intValue()); assertEquals(PublicEnum.PETER.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PETER).intValue()); assertEquals(PublicEnum.NATHANIEL.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.NATHANIEL).intValue()); assertEquals(PublicEnum.EMMA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.EMMA).intValue()); assertEquals(PublicEnum.PAULA.ordinal(), serializer.getValueToOrdinal().get(PublicEnum.PAULA).intValue()); assertTrue(Arrays.equals(PublicEnum.values(), serializer.getValues())); }
Example #30
Source File: FoldApplyProcessWindowFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration configuration) throws Exception { FunctionUtils.openFunction(this.windowFunction, configuration); if (serializedInitialValue == null) { throw new RuntimeException("No initial value was serialized for the fold " + "window function. Probably the setOutputType method was not called."); } ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue); DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); initialValue = accSerializer.deserialize(in); ctx = new InternalProcessApplyWindowContext<>(windowFunction); }