org.apache.flink.core.memory.DataInputView Java Examples

/**
 * This read attempts to first identify if the input view contains the special
 * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
 * If identified to be versioned, the usual version resolution read path
 * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
 * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
 * into the stream.
 */
public final void read(InputStream inputStream) throws IOException {
	byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
	inputStream.read(tmp);

	if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
		DataInputView inputView = new DataInputViewStreamWrapper(inputStream);

		super.read(inputView);
		read(inputView, true);
	} else {
		PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
		resetStream.unread(tmp);

		read(new DataInputViewStreamWrapper(resetStream), false);
	}
}
 

private <S> void deserializeOperatorStateValues(
	PartitionableListState<S> stateListForName,
	FSDataInputStream in,
	OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {

	if (null != metaInfo) {
		long[] offsets = metaInfo.getOffsets();
		if (null != offsets) {
			DataInputView div = new DataInputViewStreamWrapper(in);
			TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
			for (long offset : offsets) {
				in.seek(offset);
				stateListForName.add(serializer.deserialize(div));
			}
		}
	}
}
 

@SuppressWarnings("unchecked")
@Override
protected void restoreKeyAndNamespaceSerializers(
		InternalTimersSnapshot<K, N> restoredTimersSnapshot,
		DataInputView in) throws IOException {

	DataInputViewStream dis = new DataInputViewStream(in);
	try {
		final TypeSerializer<K> keySerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);
		final TypeSerializer<N> namespaceSerializer = InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true);

		restoredTimersSnapshot.setKeySerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(keySerializer));
		restoredTimersSnapshot.setNamespaceSerializerSnapshot(new BackwardsCompatibleSerializerSnapshot<>(namespaceSerializer));
	} catch (ClassNotFoundException exception) {
		throw new IOException(exception);
	}
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	final int size = source.readInt();
	target.writeInt(size);

	for (int i = 0; i < size; ++i) {
		keySerializer.copy(source, target);
		
		boolean isNull = source.readBoolean();
		target.writeBoolean(isNull);
		
		if (!isNull) {
			valueSerializer.copy(source, target);
		}
	}
}
 

private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
 

@Override
protected void read(DataInputView in, boolean wasVersioned) throws IOException {
	int noOfTimerServices = in.readInt();

	for (int i = 0; i < noOfTimerServices; i++) {
		String serviceName = in.readUTF();

		int readerVersion = wasVersioned ? getReadVersion() : InternalTimersSnapshotReaderWriters.NO_VERSION;
		InternalTimersSnapshot<?, ?> restoredTimersSnapshot = InternalTimersSnapshotReaderWriters
			.getReaderForVersion(readerVersion, userCodeClassLoader)
			.readTimersSnapshot(in);

		InternalTimerServiceImpl<K, ?> timerService = registerOrGetTimerService(
			serviceName,
			restoredTimersSnapshot);

		timerService.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIdx);
	}
}
 

/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 

@Override
public T deserialize(T reuse, DataInputView source) throws IOException {
	if (CONCURRENT_ACCESS_CHECK) {
		enterExclusiveThread();
	}

	try {
		checkAvroInitialized();
		this.decoder.setIn(source);
		return this.reader.read(reuse, this.decoder);
	}
	finally {
		if (CONCURRENT_ACCESS_CHECK) {
			exitExclusiveThread();
		}
	}
}
 

private static BiFunctionWithException<DataInputView, String, TypeSerializerSnapshot<?>, IOException> snapshotReader(ClassLoader cl) {
	return (input, unused) -> {
		try {
			return TypeSerializerSnapshot.readVersionedSnapshot(input, cl);
		}
		catch (Throwable t) {
			LOG.warn("Exception while reading serializer snapshot.", t);
			return null;
		}
	};
}
 

@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readFloat();
	}
}
 

@Override
public final void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	if (readVersion != ADAPTER_VERSION) {
		throw new IOException("Wrong/unexpected version for the TypeSerializerConfigSnapshot: " + readVersion);
	}

	serializer = TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader, true);

	// now delegate to the snapshots own reading code
	setUserCodeClassLoader(userCodeClassLoader);
	read(in);
}
 

@Override
public void read(DataInputView in) throws IOException {
	super.read(in);

	try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) {
		tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader(), true);
	} catch (ClassNotFoundException e) {
		throw new IOException("Could not find requested tuple class in classpath.", e);
	}
}
 

@Override
public Integer deserialize(DataInputView source) throws IOException {
	int header = source.readInt();
	if (header != MAGIC_VALUE) {
		throw new RuntimeException(String.format("Invalid magic value, expected %d found %d", MAGIC_VALUE, header));
	}
	return source.readInt();
}
 

@SuppressWarnings("unchecked")
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	if (deserializedFields1 == null) {
		instantiateDeserializationUtils();
	}
	
	int i = 0;
	try {
		for (; i < serializers.length; i++) {
			deserializedFields1[i] = serializers[i].deserialize(deserializedFields1[i], firstSource);
			deserializedFields2[i] = serializers[i].deserialize(deserializedFields2[i], secondSource);
		}
		
		for (i = 0; i < keyPositions.length; i++) {
			int keyPos = keyPositions[i];
			int cmp = comparators[i].compare(deserializedFields1[keyPos], deserializedFields2[keyPos]);
			if (cmp != 0) {
				return cmp;
			}
		}
		
		return 0;
	} catch (NullPointerException npex) {
		throw new NullKeyFieldException(keyPositions[i]);
	} catch (IndexOutOfBoundsException iobex) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i], iobex);
	}
}
 

@Override
public String deserialize(byte[] messageKey, byte[] message, String topic, int partition,
	long offset) throws IOException {
	cnt++;
	DataInputView in = new DataInputViewStreamWrapper(new ByteArrayInputStream(message));
	String e = ser.deserialize(in);
	return e;
}
 

private boolean deserializeNull(DataInputView source) throws IOException {
	boolean isNull = source.readBoolean();
	if (isNull) {
		source.skipBytesToRead(padding.length);
	}
	return isNull;
}
 

@Override
public void copy(
	DataInputView source, DataOutputView target) throws IOException {
	boolean hasTransactionalId = source.readBoolean();
	target.writeBoolean(hasTransactionalId);
	if (hasTransactionalId) {
		target.writeUTF(source.readUTF());
	}
	target.writeLong(source.readLong());
	target.writeShort(source.readShort());
}
 

@Override
public int[] deserialize(DataInputView source) throws IOException {
	final int len = source.readInt();
	int[] result = new int[len];
	
	for (int i = 0; i < len; i++) {
		result[i] = source.readInt();
	}
	
	return result;
}
 

private void assertSerializerIsAbleToReadOldData(TypeSerializer<ElementT> serializer) throws IOException {
	DataInputView input = dataUnderTest();

	final Matcher<ElementT> matcher = testSpecification.testDataElementMatcher;
	for (int i = 0; i < testSpecification.testDataCount; i++) {
		final ElementT result = serializer.deserialize(input);
		assertThat(result, matcher);
	}
}
 

@Override
public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) throws IOException {
	try {
		DataInputViewStream inStream = new DataInputViewStream(in);
		this.previousType = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
		this.previousEleSer = InstantiationUtil.deserializeObject(inStream, userCodeClassLoader);
	} catch (ClassNotFoundException e) {
		throw new IOException(e);
	}
}
 

@Override
public void copy(DataInputView in, DataOutputView target) throws IOException {
	int len = in.readUnsignedByte();
	target.writeByte(len);

	if (len >= HIGH_BIT) {
		int shift = 7;
		int curr;
		len = len & 0x7f;
		while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
			len |= (curr & 0x7f) << shift;
			shift += 7;
			target.writeByte(curr);
		}
		len |= curr << shift;
		target.writeByte(curr);
	}

	for (int i = 0; i < len; i++) {
		int c = in.readUnsignedByte();
		target.writeByte(c);
		while (c >= HIGH_BIT) {
			c = in.readUnsignedByte();
			target.writeByte(c);
		}
	}
}
 

@Override
public void read(DataInputView in) throws IOException {
	position = in.readInt();
	mark = 0;

	ensureCapacity(position);

	for (int i = 0; i < position; i++) {
		data[i] = in.readByte();
	}
}
 

@Override
public ElementType deserialize(ElementType reuse, DataInputView source) throws IOException {
	if (failOnRead) {
		throw new TestException();
	}
	return new ElementType(source.readInt());
}
 

@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
	int i1 = firstSource.readInt();
	int i2 = secondSource.readInt();
	int comp = (i1 < i2 ? -1 : (i1 == i2 ? 0 : 1)); 
	return ascendingComparison ? comp : -comp; 
}
 

public static <K, V> LinkedOptionalMap<K, V> readOptionalMap(
	DataInputView in,
	BiFunctionWithException<DataInputView, String, K, IOException> keyReader,
	BiFunctionWithException<DataInputView, String, V, IOException> valueReader) throws IOException {

	final long header = in.readLong();
	checkState(header == HEADER, "Corrupted stream received header %s", header);

	long mapSize = in.readInt();
	LinkedOptionalMap<K, V> map = new LinkedOptionalMap<>();
	for (int i = 0; i < mapSize; i++) {
		String keyName = in.readUTF();

		final K key;
		if (in.readBoolean()) {
			key = tryReadFrame(in, keyName, keyReader);
		}
		else {
			key = null;
		}

		final V value;
		if (in.readBoolean()) {
			value = tryReadFrame(in, keyName, valueReader);
		}
		else {
			value = null;
		}

		map.put(keyName, key, value);
	}
	return map;
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	int len = fieldSerializers.length;

	// copy bitmask
	readIntoAndCopyMask(source, target, mask);

	// copy non-null fields
	for (int fieldPos = 0; fieldPos < len; fieldPos++) {
		if (!mask[legacyOffset + fieldPos]) {
			fieldSerializers[fieldPos].copy(source, target);
		}
	}
}
 

@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
	for (int i = 0; i < arity; i++) {
		fieldSerializers[i].copy(source, target);
	}
}
 

@Override
public T deserialize(T reuse, DataInputView source) throws IOException {
	reuse.readFields(source);
	return reuse;
}
 

@Override
public ByteValueArray deserialize(ByteValueArray reuse, DataInputView source) throws IOException {
	reuse.read(source);
	return reuse;
}
 

@Override
public void read(DataInputView in) throws IOException {
	// read 8 bytes
	in.readLong();
}