Java Code Examples for org.apache.flink.core.memory.DataInputView#readFully()

@Override
public void read(DataInputView in) throws IOException {
	final int len = readVarLengthInt(in);
	this.binaryLen = len;
		
	// ensure out byte array is large enough
	byte[] data = this.binaryData;
	if (data == null || data.length < len) {
		data = new byte[len];
		this.binaryData = data;
	}
	
	// read the binary data
	in.readFully(data, 0, len);
	initFields(data, 0, len);
}
 

@Override
public void write(DataInputView source, int numBytes) throws IOException {
	if (numBytes > this.end - this.position) {
		throw new IOException("Could not write " + numBytes + " bytes since the buffer is full.");
	}

	source.readFully(this.memory,this.position, numBytes);
	this.position += numBytes;
}
 

@Override
public DecimalData deserialize(DataInputView source) throws IOException {
	if (DecimalData.isCompact(precision)) {
		long longVal = source.readLong();
		return DecimalData.fromUnscaledLong(longVal, precision, scale);
	} else {
		int length = source.readInt();
		byte[] bytes = new byte[length];
		source.readFully(bytes);
		return DecimalData.fromUnscaledBytes(bytes, precision, scale);
	}
}
 

@Override
public BinaryRowData deserialize(DataInputView source) throws IOException {
	BinaryRowData row = new BinaryRowData(numFields);
	int length = source.readInt();
	byte[] bytes = new byte[length];
	source.readFully(bytes);
	row.pointTo(MemorySegmentFactory.wrap(bytes), 0, length);
	return row;
}
 

@Override
public byte[] deserialize(DataInputView source) throws IOException {
  final int len = source.readInt();
  byte[] result = new byte[len];
  source.readFully(result);
  return result;
}
 

@Override
public void write(DataInputView source, int numBytes) throws IOException {
	if (numBytes > this.end - this.position) {
		throw new IOException("Could not write " + numBytes + " bytes since the buffer is full.");
	}

	source.readFully(this.memory,this.position, numBytes);
	this.position += numBytes;
}
 

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}
 

@Override
public void read(DataInputView in) throws IOException {
	in.readFully(bytes);
}
 

@Override
public void read(DataInputView in) throws IOException {
	byte[] bytes = new byte[32941];
	in.readFully(bytes);
}
 

@Override
public void read(DataInputView in) throws IOException {
	in.readFully(bytes);
}
 

@Override
public void read(DataInputView in) throws IOException {
	this.len = in.readInt();
	in.readFully(this.data, 0, this.len);
}
 

@Override
public void read(DataInputView in) throws IOException {
	this.len = in.readInt();
	in.readFully(this.data, 0, this.len);
}
 

@Override
public void read(DataInputView in) throws IOException {
	this.magicNumber = in.readLong();
	this.payload = new byte[in.readInt()];
	in.readFully(this.payload);
}
 

@Override
public void read(DataInputView in) throws IOException {
	synchronized (this.confData) {
		final int numberOfProperties = in.readInt();

		for (int i = 0; i < numberOfProperties; i++) {
			String key = StringValue.readString(in);
			Object value;

			byte type = in.readByte();
			switch (type) {
				case TYPE_STRING:
					value = StringValue.readString(in);
					break;
				case TYPE_INT:
					value = in.readInt();
					break;
				case TYPE_LONG:
					value = in.readLong();
					break;
				case TYPE_FLOAT:
					value = in.readFloat();
					break;
				case TYPE_DOUBLE:
					value = in.readDouble();
					break;
				case TYPE_BOOLEAN:
					value = in.readBoolean();
					break;
				case TYPE_BYTES:
					byte[] bytes = new byte[in.readInt()];
					in.readFully(bytes);
					value = bytes;
					break;
				default:
					throw new IOException(String.format("Unrecognized type: %s. This method is deprecated and" +
						" might not work for all supported types.", type));
			}

			this.confData.put(key, value);
		}
	}
}
 

/**
 * Reads from a data input view a list of serializers and their corresponding config snapshots
 * written using {@link #writeSerializersAndConfigsWithResilience(DataOutputView, List)}.
 *
 * <p>If deserialization for serializers fails due to any exception, users can opt to use a dummy
 * {@link UnloadableDummyTypeSerializer} to hold the serializer bytes
 *
 * @param in the data input view.
 * @param userCodeClassLoader the user code class loader to use.
 *
 * @return the deserialized serializer and config snapshot pairs.
 *
 * @throws IOException
 */
public static List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> readSerializersAndConfigsWithResilience(
		DataInputView in,
		ClassLoader userCodeClassLoader) throws IOException {

	int numSerializersAndConfigSnapshots = in.readInt();

	int[] offsets = new int[numSerializersAndConfigSnapshots * 2];

	for (int i = 0; i < numSerializersAndConfigSnapshots; i++) {
		offsets[i * 2] = in.readInt();
		offsets[i * 2 + 1] = in.readInt();
	}

	int totalBytes = in.readInt();
	byte[] buffer = new byte[totalBytes];
	in.readFully(buffer);

	List<Tuple2<TypeSerializer<?>, TypeSerializerSnapshot<?>>> serializersAndConfigSnapshots =
		new ArrayList<>(numSerializersAndConfigSnapshots);

	TypeSerializer<?> serializer;
	TypeSerializerSnapshot<?> configSnapshot;
	try (
		ByteArrayInputStreamWithPos bufferWithPos = new ByteArrayInputStreamWithPos(buffer);
		DataInputViewStreamWrapper bufferWrapper = new DataInputViewStreamWrapper(bufferWithPos)) {

		for (int i = 0; i < numSerializersAndConfigSnapshots; i++) {

			bufferWithPos.setPosition(offsets[i * 2]);
			serializer = tryReadSerializer(bufferWrapper, userCodeClassLoader, true);

			bufferWithPos.setPosition(offsets[i * 2 + 1]);

			configSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
					bufferWrapper, userCodeClassLoader, serializer);

			if (serializer instanceof LegacySerializerSnapshotTransformer) {
				configSnapshot = transformLegacySnapshot(serializer, configSnapshot);
			}

			serializersAndConfigSnapshots.add(new Tuple2<>(serializer, configSnapshot));
		}
	}

	return serializersAndConfigSnapshots;
}
 

@Override
public void read(DataInputView in) throws IOException {
	in.readFully(bytes);
}
 

private BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {

		final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> commitableSerializer = getCommitableSerializer();
		final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumableSerializer = getResumableSerializer();

		final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
		final String bucketPathStr = in.readUTF();
		final long creationTime = in.readLong();

		// then get the current resumable stream
		InProgressFileWriter.InProgressFileRecoverable current = null;
		if (in.readBoolean()) {
			current =
				new OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverable(
					SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in));
		}

		final int committableVersion = in.readInt();
		final int numCheckpoints = in.readInt();
		final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablePerCheckpoint = new HashMap<>(numCheckpoints);

		for (int i = 0; i < numCheckpoints; i++) {
			final long checkpointId = in.readLong();
			final int noOfResumables = in.readInt();

			final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(noOfResumables);
			for (int j = 0; j < noOfResumables; j++) {
				final byte[] bytes = new byte[in.readInt()];
				in.readFully(bytes);
				pendingFileRecoverables.add(
					new OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable(commitableSerializer.deserialize(committableVersion, bytes)));
			}
			pendingFileRecoverablePerCheckpoint.put(checkpointId, pendingFileRecoverables);
		}

		return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			pendingFileRecoverablePerCheckpoint);
	}
 

/**
 * Deserializes the version and datum from a stream.
 *
 * <p>This method deserializes data serialized via
 * {@link #writeVersionAndSerialize(SimpleVersionedSerializer, Object, DataOutputView)}.
 *
 * <p>The first four bytes will be interpreted as the version. The next four bytes will be
 * interpreted as the length of the datum bytes, then length-many bytes will be read.
 * Finally, the datum is deserialized via the {@link SimpleVersionedSerializer#deserialize(int, byte[])}
 * method.
 *
 * @param serializer The serializer to serialize the datum with.
 * @param in The stream to deserialize from.
 */
public static <T> T readVersionAndDeSerialize(SimpleVersionedSerializer<T> serializer, DataInputView in) throws IOException {
	checkNotNull(serializer, "serializer");
	checkNotNull(in, "in");

	final int version = in.readInt();
	final int length = in.readInt();
	final byte[] data = new byte[length];
	in.readFully(data);

	return serializer.deserialize(version, data);
}
 

@Override
public void write(DataInputView source, int numBytes) throws IOException {
	checkSize(numBytes);

	byte[] tempBuffer = new byte[numBytes];

	source.readFully(tempBuffer);

	System.arraycopy(tempBuffer, 0, buffer, position, numBytes);

	position += numBytes;
}
 

@Override
public void write(DataInputView source, int numBytes) throws IOException {
	checkSize(numBytes);

	byte[] tempBuffer = new byte[numBytes];

	source.readFully(tempBuffer);

	System.arraycopy(tempBuffer, 0, buffer, position, numBytes);

	position += numBytes;
}