Java Code Examples for org.apache.flink.api.common.typeutils.TypeSerializer#serialize()

/**
 * Converts a list of checkpoints into an array of SerializedCheckpointData.
 *
 * @param checkpoints The checkpoints to be converted into IdsCheckpointData.
 * @param serializer The serializer to serialize the IDs.
 * @param outputBuffer The reusable serialization buffer.
 * @param <T> The type of the ID.
 * @return An array of serializable SerializedCheckpointData, one per entry in the queue.
 *
 * @throws IOException Thrown, if the serialization fails.
 */
public static <T> SerializedCheckpointData[] fromDeque(
		ArrayDeque<Tuple2<Long, Set<T>>> checkpoints,
		TypeSerializer<T> serializer,
		DataOutputSerializer outputBuffer) throws IOException {

	SerializedCheckpointData[] serializedCheckpoints = new SerializedCheckpointData[checkpoints.size()];

	int pos = 0;
	for (Tuple2<Long, Set<T>> checkpoint : checkpoints) {
		outputBuffer.clear();
		Set<T> checkpointIds = checkpoint.f1;

		for (T id : checkpointIds) {
			serializer.serialize(id, outputBuffer);
		}

		serializedCheckpoints[pos++] = new SerializedCheckpointData(
				checkpoint.f0, outputBuffer.getCopyOfBuffer(), checkpointIds.size());
	}

	return serializedCheckpoints;
}
 

byte[] serializeValueList(
	List<T> valueList,
	TypeSerializer<T> elementSerializer,
	@SuppressWarnings("SameParameterValue") byte delimiter) throws IOException {

	out.clear();
	boolean first = true;

	for (T value : valueList) {
		Preconditions.checkNotNull(value, "You cannot add null to a value list.");

		if (first) {
			first = false;
		} else {
			out.write(delimiter);
		}
		elementSerializer.serialize(value, out);
	}

	return out.getCopyOfBuffer();
}
 

/**
 * Copied from HeapListState.getSerializedValue(Object, Object).
 */
private byte[] serializeInitValue(List<Long> toSerialize) throws IOException {
	TypeSerializer<Long> serializer = listStateDesc.getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < toSerialize.size(); i++) {
		serializer.serialize(toSerialize.get(i), view);
		if (i < toSerialize.size() - 1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 

<T> byte[] serializeValueList(
	List<T> valueList,
	TypeSerializer<T> elementSerializer,
	byte delimiter) throws IOException {

	dataOutputView.clear();
	boolean first = true;

	for (T value : valueList) {
		Preconditions.checkNotNull(value, "You cannot add null to a value list.");

		if (first) {
			first = false;
		} else {
			dataOutputView.write(delimiter);
		}
		elementSerializer.serialize(value, dataOutputView);
	}

	return dataOutputView.getCopyOfBuffer();
}
 

/**
 * Serializes the value with the given serializer.
 *
 * @param value      Value of type T to serialize
 * @param serializer Serializer for T
 * @param <T>        Type of the value
 * @return Serialized value or <code>null</code> if value <code>null</code>
 * @throws IOException On failure during serialization
 */
public static <T> byte[] serializeValue(T value, TypeSerializer<T> serializer) throws IOException {
	if (value != null) {
		// Serialize
		DataOutputSerializer dos = new DataOutputSerializer(32);
		serializer.serialize(value, dos);
		return dos.getCopyOfBuffer();
	} else {
		return null;
	}
}
 

/**
 * Writes a subset of the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @param start The logical start position of the subset.
 * @param num The number of elements to write.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output, final int start, int num) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int currentMemSeg = start / recordsPerSegment;
	int offset = (start % recordsPerSegment) * this.recordSize;
	
	while (num > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, offset);
		
		// check whether we have a full or partially full segment
		if (num >= recordsPerSegment && offset == 0) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			num -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; num > 0 && offset <= this.lastEntryOffset; num--, offset += this.recordSize) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}

		offset = 0;
	}
}
 

/**
 * Serializes the value with the given serializer.
 *
 * @param value      Value of type T to serialize
 * @param serializer Serializer for T
 * @param <T>        Type of the value
 * @return Serialized value or <code>null</code> if value <code>null</code>
 * @throws IOException On failure during serialization
 */
public static <T> byte[] serializeValue(T value, TypeSerializer<T> serializer) throws IOException {
	if (value != null) {
		// Serialize
		DataOutputSerializer dos = new DataOutputSerializer(32);
		serializer.serialize(value, dos);
		return dos.getCopyOfBuffer();
	} else {
		return null;
	}
}
 

public static <K> void writeKey(
	K key,
	TypeSerializer<K> keySerializer,
	DataOutputSerializer keySerializationDataOutputView,
	boolean ambiguousKeyPossible) throws IOException {
	//write key
	int beforeWrite = keySerializationDataOutputView.length();
	keySerializer.serialize(key, keySerializationDataOutputView);

	if (ambiguousKeyPossible) {
		//write size of key
		writeLengthFrom(beforeWrite, keySerializationDataOutputView);
	}
}
 

/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {

	DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);

	try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
		User rec = dataFileReader.next();

		// check if record has been read correctly
		assertNotNull(rec);
		assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());

		// now serialize it with our framework:
		ExecutionConfig ec = new ExecutionConfig();
		TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);

		assertEquals(AvroTypeInfo.class, te.getClass());
		TypeSerializer<User> tser = te.createSerializer(ec);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
			tser.serialize(rec, outView);
		}

		User newRec;
		try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
				new ByteArrayInputStream(out.toByteArray()))) {
			newRec = tser.deserialize(inView);
		}

		// check if it is still the same
		assertNotNull(newRec);
		assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
	}
}
 

/**
 * Writes the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int recordsLeft = this.numRecords;
	int currentMemSeg = 0;
	
	while (recordsLeft > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, 0);
		
		// check whether we have a full or partially full segment
		if (recordsLeft >= recordsPerSegment) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			recordsLeft -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; recordsLeft > 0; recordsLeft--) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}
	}
}
 

public static <T> List<byte[]> toBytesList(List<T> values, TypeSerializer<T> serializer) {
	List<byte[]> ret = new ArrayList<>();
	for (T value : values) {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);
		try {
			serializer.serialize(value, wrapper);
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
		ret.add(baos.toByteArray());
	}
	return ret;
}
 

private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 

/**
 * Writes a subset of the records in this buffer in their logical order to the given output.
 * 
 * @param output The output view to write the records to.
 * @param start The logical start position of the subset.
 * @param num The number of elements to write.
 * @throws IOException Thrown, if an I/O exception occurred writing to the output view.
 */
@Override
public void writeToOutput(final ChannelWriterOutputView output, final int start, int num) throws IOException {
	final TypeComparator<T> comparator = this.comparator;
	final TypeSerializer<T> serializer = this.serializer;
	T record = this.recordInstance;
	
	final SingleSegmentInputView inView = this.inView;
	
	final int recordsPerSegment = this.recordsPerSegment;
	int currentMemSeg = start / recordsPerSegment;
	int offset = (start % recordsPerSegment) * this.recordSize;
	
	while (num > 0) {
		final MemorySegment currentIndexSegment = this.sortBuffer.get(currentMemSeg++);
		inView.set(currentIndexSegment, offset);
		
		// check whether we have a full or partially full segment
		if (num >= recordsPerSegment && offset == 0) {
			// full segment
			for (int numInMemSeg = 0; numInMemSeg < recordsPerSegment; numInMemSeg++) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
			num -= recordsPerSegment;
		} else {
			// partially filled segment
			for (; num > 0 && offset <= this.lastEntryOffset; num--, offset += this.recordSize) {
				record = comparator.readWithKeyDenormalization(record, inView);
				serializer.serialize(record, output);
			}
		}

		offset = 0;
	}
}
 

private <T> byte[] write(TypeSerializer<T> serializer, T value) throws IOException {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    DataOutputView out = new DataOutputViewStreamWrapper(buffer);
    serializer.serialize(value, out);
    return buffer.toByteArray();
}
 

@Test
public void testReadTooMany() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);

	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());

	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();

	// read and re-generate all records and compare them
	try {
		final Tuple2<Integer, String> readRec = new Tuple2<>();
		for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
			generator.next(rec);
			serializer.deserialize(readRec, inView);
			final int k1 = rec.f0;
			final String v1 = rec.f1;
			final int k2 = readRec.f0;
			final String v2 = readRec.f1;
			Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
		}
		Assert.fail("Expected an EOFException which did not occur.");
	}
	catch (EOFException eofex) {
		// expected
	}
	catch (Throwable t) {
		// unexpected
		Assert.fail("Unexpected Exception: " + t.getMessage());
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testIterator() throws Exception {

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final long seed = new Random().nextLong();
	final int numElements = 1000;

	final SocketStreamIterator<Long> iterator = new SocketStreamIterator<>(LongSerializer.INSTANCE);

	Thread writer = new Thread() {

		@Override
		public void run() {
			try {
				try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort());
					DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) {

					final TypeSerializer<Long> serializer = LongSerializer.INSTANCE;
					final Random rnd = new Random(seed);

					for (int i = 0; i < numElements; i++) {
						serializer.serialize(rnd.nextLong(), out);
					}
				}
			}
			catch (Throwable t) {
				error.set(t);
			}
		}
	};

	writer.start();

	final Random validator = new Random(seed);
	for (int i = 0; i < numElements; i++) {
		assertTrue(iterator.hasNext());
		assertTrue(iterator.hasNext());
		assertEquals(validator.nextLong(), iterator.next().longValue());
	}

	assertFalse(iterator.hasNext());
	writer.join();
	assertFalse(iterator.hasNext());
}
 

private static <T> ByteBuffer serialize(TypeSerializer<T> serializer, T record) throws IOException {
	DataOutputSerializer out = new DataOutputSerializer(1024);
	serializer.serialize(record, out);
	return out.wrapAsByteBuffer();
}
 

@Test
public void testIterator() throws Exception {

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final long seed = new Random().nextLong();
	final int numElements = 1000;

	final SocketStreamIterator<Long> iterator = new SocketStreamIterator<>(LongSerializer.INSTANCE);

	Thread writer = new Thread() {

		@Override
		public void run() {
			try {
				try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort());
					DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) {

					final TypeSerializer<Long> serializer = LongSerializer.INSTANCE;
					final Random rnd = new Random(seed);

					for (int i = 0; i < numElements; i++) {
						serializer.serialize(rnd.nextLong(), out);
					}
				}
			}
			catch (Throwable t) {
				error.set(t);
			}
		}
	};

	writer.start();

	final Random validator = new Random(seed);
	for (int i = 0; i < numElements; i++) {
		assertTrue(iterator.hasNext());
		assertTrue(iterator.hasNext());
		assertEquals(validator.nextLong(), iterator.next().longValue());
	}

	assertFalse(iterator.hasNext());
	writer.join();
	assertFalse(iterator.hasNext());
}
 

/**
 * Serializes the key and namespace into a {@link ByteBuffer}.
 *
 * <p>The serialized format matches the RocksDB state backend key format, i.e.
 * the key and namespace don't have to be deserialized for RocksDB lookups.
 *
 * @param key                 Key to serialize
 * @param keySerializer       Serializer for the key
 * @param namespace           Namespace to serialize
 * @param namespaceSerializer Serializer for the namespace
 * @param <K>                 Key type
 * @param <N>                 Namespace type
 * @return Buffer holding the serialized key and namespace
 * @throws IOException Serialization errors are forwarded
 */
public static <K, N> byte[] serializeKeyAndNamespace(
		K key,
		TypeSerializer<K> keySerializer,
		N namespace,
		TypeSerializer<N> namespaceSerializer) throws IOException {

	DataOutputSerializer dos = new DataOutputSerializer(32);

	keySerializer.serialize(key, dos);
	dos.writeByte(MAGIC_NUMBER);
	namespaceSerializer.serialize(namespace, dos);

	return dos.getCopyOfBuffer();
}
 

/**
 * Serialize an object using a TypeSerializer.
 *
 * @param object     to serialize
 * @param serializer to use;
 * @param <IN>       type of the object
 * @return serialized object
 * @throws IOException
 */
public static <IN> byte[] serialize(IN object, TypeSerializer<IN> serializer) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(new DataOutputStream(baos));
    serializer.serialize(object, wrapper);
    return baos.toByteArray();
}