org.apache.flink.core.memory.DataOutputView Java Examples

/**
 * Write a list of serializers and their corresponding config snapshots to the provided
 * data output view. This method writes in a fault tolerant way, so that when read again
 * using {@link #readSerializersAndConfigsWithResilience(DataInputView, ClassLoader)}, if
 * deserialization of the serializer fails, its configuration snapshot will remain intact.
 *
 * <p>Specifically, all written serializers and their config snapshots are indexed by their
 * offset positions within the serialized bytes. The serialization format is as follows:
 * <ul>
 *     <li>1. number of serializer and configuration snapshot pairs.</li>
 *     <li>2. offsets of each serializer and configuration snapshot, in order.</li>
 *     <li>3. total number of bytes for the serialized serializers and the config snapshots.</li>
 *     <li>4. serialized serializers and the config snapshots.</li>
 * </ul>
 *
 * @param out the data output view.
 * @param serializersAndConfigs serializer and configuration snapshot pairs
 *
 * @throws IOException
 */
public static void writeSerializersAndConfigsWithResilience(
		DataOutputView out,
		List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigs) throws IOException {

	try (
		ByteArrayOutputStreamWithPos bufferWithPos = new ByteArrayOutputStreamWithPos();
		DataOutputViewStreamWrapper bufferWrapper = new DataOutputViewStreamWrapper(bufferWithPos)) {

		out.writeInt(serializersAndConfigs.size());
		for (Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>> serAndConfSnapshot : serializersAndConfigs) {
			out.writeInt(bufferWithPos.getPosition());
			writeSerializer(bufferWrapper, serAndConfSnapshot.f0);

			out.writeInt(bufferWithPos.getPosition());
			TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
				bufferWrapper, (TypeSerializerSnapshot) serAndConfSnapshot.f1, serAndConfSnapshot.f0);
		}

		out.writeInt(bufferWithPos.getPosition());
		out.write(bufferWithPos.getBuf(), 0, bufferWithPos.getPosition());
	}
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	int val = source.readUnsignedByte();
	target.writeByte(val);
	
	if (val >= MAX_BIT) {
		int shift = 7;
		int curr;
		val = val & 0x7f;
		while ((curr = source.readUnsignedByte()) >= MAX_BIT) {
			target.writeByte(curr);
			val |= (curr & 0x7f) << shift;
			shift += 7;
		}
		target.writeByte(curr);
		val |= curr << shift;
	}
	
	target.write(source, val);
}
 

@Override
public void serialize(T value, DataOutputView target) throws IOException {
	if (CONCURRENT_ACCESS_CHECK) {
		enterExclusiveThread();
	}

	try {
		checkAvroInitialized();
		this.encoder.setOut(target);
		this.writer.write(value, this.encoder);
	}
	finally {
		if (CONCURRENT_ACCESS_CHECK) {
			exitExclusiveThread();
		}
	}
}
 

@Override
public void copy(
		DataInputView source, DataOutputView target) throws IOException {
	TXN pendingTxnHandle = transactionSerializer.deserialize(source);
	transactionSerializer.serialize(pendingTxnHandle, target);
	final long pendingTxnStartTime = source.readLong();
	target.writeLong(pendingTxnStartTime);

	int numPendingCommitTxns = source.readInt();
	target.writeInt(numPendingCommitTxns);
	for (int i = 0; i < numPendingCommitTxns; i++) {
		TXN pendingCommitTxnHandle = transactionSerializer.deserialize(source);
		transactionSerializer.serialize(pendingCommitTxnHandle, target);
		final long pendingCommitTxnStartTime = source.readLong();
		target.writeLong(pendingCommitTxnStartTime);
	}

	boolean hasContext = source.readBoolean();
	target.writeBoolean(hasContext);
	if (hasContext) {
		CONTEXT context = contextSerializer.deserialize(source);
		contextSerializer.serialize(context, target);
	}
}
 

@Override
public void serialize(
		State<TXN, CONTEXT> record,
		DataOutputView target) throws IOException {
	final TransactionHolder<TXN> pendingTransaction = record.getPendingTransaction();
	transactionSerializer.serialize(pendingTransaction.handle, target);
	target.writeLong(pendingTransaction.transactionStartTime);

	final List<TransactionHolder<TXN>> pendingCommitTransactions = record.getPendingCommitTransactions();
	target.writeInt(pendingCommitTransactions.size());
	for (TransactionHolder<TXN> pendingTxn : pendingCommitTransactions) {
		transactionSerializer.serialize(pendingTxn.handle, target);
		target.writeLong(pendingTxn.transactionStartTime);
	}

	Optional<CONTEXT> context = record.getContext();
	if (context.isPresent()) {
		target.writeBoolean(true);
		contextSerializer.serialize(context.get(), target);
	} else {
		target.writeBoolean(false);
	}
}
 

@Test
public void testKeyGroupSerializationAndDeserialization() throws Exception {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(8);
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);

	for (int keyGroupPrefixBytes = 1; keyGroupPrefixBytes <= 2; ++keyGroupPrefixBytes) {
		for (int orgKeyGroup = 0; orgKeyGroup < 128; ++orgKeyGroup) {
			outputStream.reset();
			RocksDBKeySerializationUtils.writeKeyGroup(orgKeyGroup, keyGroupPrefixBytes, outputView);
			int deserializedKeyGroup = RocksDBKeySerializationUtils.readKeyGroup(
				keyGroupPrefixBytes,
				new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(outputStream.toByteArray())));
			Assert.assertEquals(orgKeyGroup, deserializedKeyGroup);
		}
	}
}
 

public static void writeNullMask(
		Object[] array, int length, DataOutputView target) throws IOException {

	int b;
	int bytePos;

	for(int fieldPos = 0; fieldPos < length; ) {
		b = 0x00;
		// set bits in byte
		for(bytePos = 0; bytePos < 8 && fieldPos < length; bytePos++, fieldPos++) {
			b = b << 1;
			// set bit if field is null
			if(array[fieldPos] == null) {
				b |= 0x01;
			}
		}
		// shift bits if last byte is not completely filled
		for(; bytePos < 8; bytePos++) {
			b = b << 1;
		}
		// write byte
		target.writeByte(b);
	}
}
 

@Override
protected void snapshotQueueState(PriorityQueue<StreamRecord<Tuple2<String, IN>>> queue, DataOutputView dataOutputView) throws IOException {
    dataOutputView.writeInt(queue.size());
    for (StreamRecord<Tuple2<String, IN>> record : queue) {
        String streamId = record.getValue().f0;
        dataOutputView.writeUTF(streamId);
        this.getStreamRecordSerializer(streamId).serialize(record, dataOutputView);
    }
}
 

@Override
public void write(DataOutputView out) throws IOException {
	super.write(out);

	out.writeInt(kryoRegistrations.size());
	for (Map.Entry<String, KryoRegistration> kryoRegistrationEntry : kryoRegistrations.entrySet()) {
		out.writeUTF(kryoRegistrationEntry.getKey());
		new KryoRegistrationSerializationProxy<>(kryoRegistrationEntry.getValue()).write(out);
	}
}
 

@Override
public void write(DataOutputView out) throws IOException {
	out.writeInt(position);

	for (int i = 0; i < position; i++) {
		out.writeByte(data[i]);
	}
}
 

@Override
public void serialize(TestElement record, DataOutputView target) throws IOException {
	// serialize priority first, so that we have correct order in RocksDB. We flip the sign bit for correct
	// lexicographic order.
	target.writeLong(MathUtils.flipSignBit(record.getPriority()));
	target.writeLong(record.getKey());
}
 

public static <K, V> void writeOptionalMap(
	DataOutputView out,
	LinkedOptionalMap<K, V> map,
	BiConsumerWithException<DataOutputView, K, IOException> keyWriter,
	BiConsumerWithException<DataOutputView, V, IOException> valueWriter) throws IOException {

	out.writeLong(HEADER);
	out.writeInt(map.size());
	map.forEach(((keyName, key, value) -> {
		out.writeUTF(keyName);

		if (key == null) {
			out.writeBoolean(false);
		}
		else {
			out.writeBoolean(true);
			writeFramed(out, keyWriter, key);
		}

		if (value == null) {
			out.writeBoolean(false);
		}
		else {
			out.writeBoolean(true);
			writeFramed(out, valueWriter, value);
		}
	}));
}
 

@Override
public void serialize(TestElement record, DataOutputView target) throws IOException {
	// serialize priority first, so that we have correct order in RocksDB. We flip the sign bit for correct
	// lexicographic order.
	target.writeLong(MathUtils.flipSignBit(record.getPriority()));
	target.writeLong(record.getKey());
}
 

protected static void copyInternal(DataInputView source, DataOutputView target) throws IOException {
	int count = source.readInt();
	target.writeInt(count);

	int bytes = ELEMENT_LENGTH_IN_BYTES * count;
	target.write(source, bytes);
}
 

@Override
public void serialize(ComplexPayload record, DataOutputView target) throws IOException {
	try {
		if (currentOwnerThread.compareAndSet(null, Thread.currentThread())) {
			target.write(InstantiationUtil.serializeObject(record));
		} else {
			throw new IllegalStateException("Concurrent access to type serializer detected!");
		}
	} finally {
		currentOwnerThread.set(null);
	}
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int size = source.readInt();
	target.writeInt(size);
	for (int i = 0; i < size; i++) {
		target.writeInt(source.readInt());
	}
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final boolean isNull = BigIntSerializer.copyBigInteger(source, target);
	if (!isNull) {
		final int scale = source.readInt();
		target.writeInt(scale);
	}
}
 

@Override
public byte[] serializeValue(Tuple3<Integer, Integer, String> element) {
	ByteArrayOutputStream by = new ByteArrayOutputStream();
	DataOutputView out = new DataOutputViewStreamWrapper(by);
	try {
		ts.serialize(new Tuple2<>(element.f0, element.f1), out);
	} catch (IOException e) {
		throw new RuntimeException("Error" , e);
	}
	return by.toByteArray();
}
 

@Override
public final void writeTimersSnapshot(DataOutputView out) throws IOException {
	writeKeyAndNamespaceSerializers(out);

	LegacyTimerSerializer<K, N> timerSerializer = new LegacyTimerSerializer<>(
		keySerializer,
		namespaceSerializer);

	// write the event time timers
	Set<TimerHeapInternalTimer<K, N>> eventTimers = timersSnapshot.getEventTimeTimers();
	if (eventTimers != null) {
		out.writeInt(eventTimers.size());
		for (TimerHeapInternalTimer<K, N> eventTimer : eventTimers) {
			timerSerializer.serialize(eventTimer, out);
		}
	} else {
		out.writeInt(0);
	}

	// write the processing time timers
	Set<TimerHeapInternalTimer<K, N>> processingTimers = timersSnapshot.getProcessingTimeTimers();
	if (processingTimers != null) {
		out.writeInt(processingTimers.size());
		for (TimerHeapInternalTimer<K, N> processingTimer : processingTimers) {
			timerSerializer.serialize(processingTimer, out);
		}
	} else {
		out.writeInt(0);
	}
}
 

private void serializeSingleComputationState(
		ComputationState computationState,
		DataOutputView target) throws IOException {

	StringValue.writeString(computationState.getCurrentStateName(), target);
	nodeIdSerializer.serialize(computationState.getPreviousBufferEntry(), target);
	versionSerializer.serialize(computationState.getVersion(), target);
	target.writeLong(computationState.getStartTimestamp());
	serializeStartEvent(computationState.getStartEventID(), target);
}
 

@Override
public void serialize(BinaryRow record, DataOutputView target) throws IOException {
	target.writeInt(record.getSizeInBytes());
	if (target instanceof MemorySegmentWritable) {
		serializeWithoutLength(record, (MemorySegmentWritable) target);
	} else {
		SegmentsUtil.copyToView(
				record.getSegments(), record.getOffset(),
				record.getSizeInBytes(), target);
	}
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	if (Decimal.isCompact(precision)) {
		target.writeLong(source.readLong());
	} else {
		int len = source.readInt();
		target.writeInt(len);
		target.write(source, len);
	}
}
 

@Override
public void write(DataOutputView out) throws IOException {
	out.writeInt(position);

	for (int i = 0; i < position; i++) {
		out.writeInt(data[i]);
	}
}
 

@Benchmark
public byte[] stringWrite() throws IOException {
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
    DataOutputView out = new DataOutputViewStreamWrapper(buffer);
    serializer.serialize(input, out);
    return buffer.toByteArray();
}
 

private void copyStates(DataInputView source, DataOutputView target) throws IOException {
	int computationStateNo = source.readInt();
	target.writeInt(computationStateNo);

	for (int i = 0; i < computationStateNo; i++) {
		copySingleComputationState(source, target);
	}
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	target.writeInt(source.readInt());
	int len = source.readInt();
	target.writeInt(len);
	for (int i = 0; i < len; i++) {
		target.writeInt(source.readInt());
	}
}
 

@Override
protected void writeOuterSnapshot(DataOutputView out) throws IOException {
	out.writeUTF(componentClass.getName());
}
 

@Override
public void serialize(Integer record, DataOutputView target) throws IOException {
	Assert.assertFalse(disabled);
	target.writeInt(record);
}
 

@Override
public void serialize(StringValueArray record, DataOutputView target) throws IOException {
	record.write(target);
}
 

@Override
protected void writeRecord(Tuple2<Integer, String> record, DataOutputView outputView) throws IOException {
	serializer.serialize(record, outputView);
}