Java Code Examples for org.apache.flink.core.memory.DataInputDeserializer#available()

/**
 * Deserializes the value with the given serializer.
 *
 * @param serializedValue Serialized value of type T
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized value or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> T deserializeValue(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue == null) {
		return null;
	} else {
		final DataInputDeserializer deser = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);
		final T value = serializer.deserialize(deser);
		if (deser.available() > 0) {
			throw new IOException(
				"Unconsumed bytes in the deserialized value. " +
					"This indicates a mismatch in the value serializers " +
					"used by the KvState instance and this access.");
		}
		return value;
	}
}
 

/**
 * Deserializes all kv pairs with the given serializer.
 *
 * @param serializedValue Serialized value of type Map<UK, UV>
 * @param keySerializer   Serializer for UK
 * @param valueSerializer Serializer for UV
 * @param <UK>            Type of the key
 * @param <UV>            Type of the value.
 * @return Deserialized map or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <UK, UV> Map<UK, UV> deserializeMap(byte[] serializedValue, TypeSerializer<UK> keySerializer, TypeSerializer<UV> valueSerializer) throws IOException {
	if (serializedValue != null) {
		DataInputDeserializer in = new DataInputDeserializer(serializedValue, 0, serializedValue.length);

		Map<UK, UV> result = new HashMap<>();
		while (in.available() > 0) {
			UK key = keySerializer.deserialize(in);

			boolean isNull = in.readBoolean();
			UV value = isNull ? null : valueSerializer.deserialize(in);

			result.put(key, value);
		}

		return result;
	} else {
		return null;
	}
}
 

/**
 * Deserializes the value with the given serializer.
 *
 * @param serializedValue Serialized value of type T
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized value or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> T deserializeValue(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue == null) {
		return null;
	} else {
		final DataInputDeserializer deser = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);
		final T value = serializer.deserialize(deser);
		if (deser.available() > 0) {
			throw new IOException(
				"Unconsumed bytes in the deserialized value. " +
					"This indicates a mismatch in the value serializers " +
					"used by the KvState instance and this access.");
		}
		return value;
	}
}
 

/**
 * Deserializes all kv pairs with the given serializer.
 *
 * @param serializedValue Serialized value of type Map<UK, UV>
 * @param keySerializer   Serializer for UK
 * @param valueSerializer Serializer for UV
 * @param <UK>            Type of the key
 * @param <UV>            Type of the value.
 * @return Deserialized map or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <UK, UV> Map<UK, UV> deserializeMap(byte[] serializedValue, TypeSerializer<UK> keySerializer, TypeSerializer<UV> valueSerializer) throws IOException {
	if (serializedValue != null) {
		DataInputDeserializer in = new DataInputDeserializer(serializedValue, 0, serializedValue.length);

		Map<UK, UV> result = new HashMap<>();
		while (in.available() > 0) {
			UK key = keySerializer.deserialize(in);

			boolean isNull = in.readBoolean();
			UV value = isNull ? null : valueSerializer.deserialize(in);

			result.put(key, value);
		}

		return result;
	} else {
		return null;
	}
}
 

/**
 * Deserializes all kv pairs with the given serializer.
 *
 * @param serializedValue Serialized value of type Map<UK, UV>
 * @param keySerializer   Serializer for UK
 * @param valueSerializer Serializer for UV
 * @param <UK>            Type of the key
 * @param <UV>            Type of the value.
 * @return Deserialized map or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <UK, UV> Map<UK, UV> deserializeMap(byte[] serializedValue, TypeSerializer<UK> keySerializer, TypeSerializer<UV> valueSerializer) throws IOException {
	if (serializedValue != null) {
		DataInputDeserializer in = new DataInputDeserializer(serializedValue, 0, serializedValue.length);

		Map<UK, UV> result = new HashMap<>();
		while (in.available() > 0) {
			UK key = keySerializer.deserialize(in);

			boolean isNull = in.readBoolean();
			UV value = isNull ? null : valueSerializer.deserialize(in);

			result.put(key, value);
		}

		return result;
	} else {
		return null;
	}
}
 

/**
 * Deserializes the value with the given serializer.
 *
 * @param serializedValue Serialized value of type T
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized value or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> T deserializeValue(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue == null) {
		return null;
	} else {
		final DataInputDeserializer deser = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);
		final T value = serializer.deserialize(deser);
		if (deser.available() > 0) {
			throw new IOException(
				"Unconsumed bytes in the deserialized value. " +
					"This indicates a mismatch in the value serializers " +
					"used by the KvState instance and this access.");
		}
		return value;
	}
}
 

/**
 * Deserializes all values with the given serializer.
 *
 * @param serializedValue Serialized value of type List<T>
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized list or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> List<T> deserializeList(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue != null) {
		final DataInputDeserializer in = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);

		try {
			final List<T> result = new ArrayList<>();
			while (in.available() > 0) {
				result.add(serializer.deserialize(in));

				// The expected binary format has a single byte separator. We
				// want a consistent binary format in order to not need any
				// special casing during deserialization. A "cleaner" format
				// would skip this extra byte, but would require a memory copy
				// for RocksDB, which stores the data serialized in this way
				// for lists.
				if (in.available() > 0) {
					in.readByte();
				}
			}

			return result;
		} catch (IOException e) {
			throw new IOException(
					"Unable to deserialize value. " +
						"This indicates a mismatch in the value serializers " +
						"used by the KvState instance and this access.", e);
		}
	} else {
		return null;
	}
}
 

private static <V> V deserializeNextElement(DataInputDeserializer in, TypeSerializer<V> elementSerializer) {
	try {
		if (in.available() > 0) {
			V element = elementSerializer.deserialize(in);
			if (in.available() > 0) {
				in.readByte();
			}
			return element;
		}
	} catch (IOException e) {
		throw new FlinkRuntimeException("Unexpected list element deserialization failure", e);
	}
	return null;
}
 

@Override
public void migrateSerializedValue(
		DataInputDeserializer serializedOldValueInput,
		DataOutputSerializer serializedMigratedValueOutput,
		TypeSerializer<List<V>> priorSerializer,
		TypeSerializer<List<V>> newSerializer) throws StateMigrationException {

	Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
	Preconditions.checkArgument(newSerializer instanceof ListSerializer);

	TypeSerializer<V> priorElementSerializer =
		((ListSerializer<V>) priorSerializer).getElementSerializer();

	TypeSerializer<V> newElementSerializer =
		((ListSerializer<V>) newSerializer).getElementSerializer();

	try {
		while (serializedOldValueInput.available() > 0) {
			V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer);
			newElementSerializer.serialize(element, serializedMigratedValueOutput);
			if (serializedOldValueInput.available() > 0) {
				serializedMigratedValueOutput.write(DELIMITER);
			}
		}
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
	}
}
 

/**
 * Deserializes the key and namespace into a {@link Tuple2}.
 *
 * @param serializedKeyAndNamespace Serialized key and namespace
 * @param keySerializer             Serializer for the key
 * @param namespaceSerializer       Serializer for the namespace
 * @param <K>                       Key type
 * @param <N>                       Namespace
 * @return Tuple2 holding deserialized key and namespace
 * @throws IOException              if the deserialization fails for any reason
 */
public static <K, N> Tuple2<K, N> deserializeKeyAndNamespace(
		byte[] serializedKeyAndNamespace,
		TypeSerializer<K> keySerializer,
		TypeSerializer<N> namespaceSerializer) throws IOException {

	DataInputDeserializer dis = new DataInputDeserializer(
			serializedKeyAndNamespace,
			0,
			serializedKeyAndNamespace.length);

	try {
		K key = keySerializer.deserialize(dis);
		byte magicNumber = dis.readByte();
		if (magicNumber != MAGIC_NUMBER) {
			throw new IOException("Unexpected magic number " + magicNumber + ".");
		}
		N namespace = namespaceSerializer.deserialize(dis);

		if (dis.available() > 0) {
			throw new IOException("Unconsumed bytes in the serialized key and namespace.");
		}

		return new Tuple2<>(key, namespace);
	} catch (IOException e) {
		throw new IOException("Unable to deserialize key " +
			"and namespace. This indicates a mismatch in the key/namespace " +
			"serializers used by the KvState instance and this access.", e);
	}
}
 

private static <V> V deserializeNextElement(DataInputDeserializer in, TypeSerializer<V> elementSerializer) {
	try {
		if (in.available() > 0) {
			V element = elementSerializer.deserialize(in);
			if (in.available() > 0) {
				in.readByte();
			}
			return element;
		}
	} catch (IOException e) {
		throw new FlinkRuntimeException("Unexpected list element deserialization failure");
	}
	return null;
}
 

/**
 * Deserializes all values with the given serializer.
 *
 * @param serializedValue Serialized value of type List<T>
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized list or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> List<T> deserializeList(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue != null) {
		final DataInputDeserializer in = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);

		try {
			final List<T> result = new ArrayList<>();
			while (in.available() > 0) {
				result.add(serializer.deserialize(in));

				// The expected binary format has a single byte separator. We
				// want a consistent binary format in order to not need any
				// special casing during deserialization. A "cleaner" format
				// would skip this extra byte, but would require a memory copy
				// for RocksDB, which stores the data serialized in this way
				// for lists.
				if (in.available() > 0) {
					in.readByte();
				}
			}

			return result;
		} catch (IOException e) {
			throw new IOException(
					"Unable to deserialize value. " +
						"This indicates a mismatch in the value serializers " +
						"used by the KvState instance and this access.", e);
		}
	} else {
		return null;
	}
}
 

private static <V> V deserializeNextElement(DataInputDeserializer in, TypeSerializer<V> elementSerializer) {
	try {
		if (in.available() > 0) {
			V element = elementSerializer.deserialize(in);
			if (in.available() > 0) {
				in.readByte();
			}
			return element;
		}
	} catch (IOException e) {
		throw new FlinkRuntimeException("Unexpected list element deserialization failure", e);
	}
	return null;
}
 

@Override
public void migrateSerializedValue(
		DataInputDeserializer serializedOldValueInput,
		DataOutputSerializer serializedMigratedValueOutput,
		TypeSerializer<List<V>> priorSerializer,
		TypeSerializer<List<V>> newSerializer) throws StateMigrationException {

	Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
	Preconditions.checkArgument(newSerializer instanceof ListSerializer);

	TypeSerializer<V> priorElementSerializer =
		((ListSerializer<V>) priorSerializer).getElementSerializer();

	TypeSerializer<V> newElementSerializer =
		((ListSerializer<V>) newSerializer).getElementSerializer();

	try {
		while (serializedOldValueInput.available() > 0) {
			V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer);
			newElementSerializer.serialize(element, serializedMigratedValueOutput);
			if (serializedOldValueInput.available() > 0) {
				serializedMigratedValueOutput.write(DELIMITER);
			}
		}
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
	}
}
 

/**
 * Deserializes the key and namespace into a {@link Tuple2}.
 *
 * @param serializedKeyAndNamespace Serialized key and namespace
 * @param keySerializer             Serializer for the key
 * @param namespaceSerializer       Serializer for the namespace
 * @param <K>                       Key type
 * @param <N>                       Namespace
 * @return Tuple2 holding deserialized key and namespace
 * @throws IOException              if the deserialization fails for any reason
 */
public static <K, N> Tuple2<K, N> deserializeKeyAndNamespace(
		byte[] serializedKeyAndNamespace,
		TypeSerializer<K> keySerializer,
		TypeSerializer<N> namespaceSerializer) throws IOException {

	DataInputDeserializer dis = new DataInputDeserializer(
			serializedKeyAndNamespace,
			0,
			serializedKeyAndNamespace.length);

	try {
		K key = keySerializer.deserialize(dis);
		byte magicNumber = dis.readByte();
		if (magicNumber != MAGIC_NUMBER) {
			throw new IOException("Unexpected magic number " + magicNumber + ".");
		}
		N namespace = namespaceSerializer.deserialize(dis);

		if (dis.available() > 0) {
			throw new IOException("Unconsumed bytes in the serialized key and namespace.");
		}

		return new Tuple2<>(key, namespace);
	} catch (IOException e) {
		throw new IOException("Unable to deserialize key " +
			"and namespace. This indicates a mismatch in the key/namespace " +
			"serializers used by the KvState instance and this access.", e);
	}
}
 

/**
 * Deserializes the key and namespace into a {@link Tuple2}.
 *
 * @param serializedKeyAndNamespace Serialized key and namespace
 * @param keySerializer             Serializer for the key
 * @param namespaceSerializer       Serializer for the namespace
 * @param <K>                       Key type
 * @param <N>                       Namespace
 * @return Tuple2 holding deserialized key and namespace
 * @throws IOException              if the deserialization fails for any reason
 */
public static <K, N> Tuple2<K, N> deserializeKeyAndNamespace(
		byte[] serializedKeyAndNamespace,
		TypeSerializer<K> keySerializer,
		TypeSerializer<N> namespaceSerializer) throws IOException {

	DataInputDeserializer dis = new DataInputDeserializer(
			serializedKeyAndNamespace,
			0,
			serializedKeyAndNamespace.length);

	try {
		K key = keySerializer.deserialize(dis);
		byte magicNumber = dis.readByte();
		if (magicNumber != MAGIC_NUMBER) {
			throw new IOException("Unexpected magic number " + magicNumber + ".");
		}
		N namespace = namespaceSerializer.deserialize(dis);

		if (dis.available() > 0) {
			throw new IOException("Unconsumed bytes in the serialized key and namespace.");
		}

		return new Tuple2<>(key, namespace);
	} catch (IOException e) {
		throw new IOException("Unable to deserialize key " +
			"and namespace. This indicates a mismatch in the key/namespace " +
			"serializers used by the KvState instance and this access.", e);
	}
}
 

/**
 * Deserializes all values with the given serializer.
 *
 * @param serializedValue Serialized value of type List<T>
 * @param serializer      Serializer for T
 * @param <T>             Type of the value
 * @return Deserialized list or <code>null</code> if the serialized value
 * is <code>null</code>
 * @throws IOException On failure during deserialization
 */
public static <T> List<T> deserializeList(byte[] serializedValue, TypeSerializer<T> serializer) throws IOException {
	if (serializedValue != null) {
		final DataInputDeserializer in = new DataInputDeserializer(
			serializedValue, 0, serializedValue.length);

		try {
			final List<T> result = new ArrayList<>();
			while (in.available() > 0) {
				result.add(serializer.deserialize(in));

				// The expected binary format has a single byte separator. We
				// want a consistent binary format in order to not need any
				// special casing during deserialization. A "cleaner" format
				// would skip this extra byte, but would require a memory copy
				// for RocksDB, which stores the data serialized in this way
				// for lists.
				if (in.available() > 0) {
					in.readByte();
				}
			}

			return result;
		} catch (IOException e) {
			throw new IOException(
					"Unable to deserialize value. " +
						"This indicates a mismatch in the value serializers " +
						"used by the KvState instance and this access.", e);
		}
	} else {
		return null;
	}
}
 

@Override
public void migrateSerializedValue(
		DataInputDeserializer serializedOldValueInput,
		DataOutputSerializer serializedMigratedValueOutput,
		TypeSerializer<List<V>> priorSerializer,
		TypeSerializer<List<V>> newSerializer) throws StateMigrationException {

	Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
	Preconditions.checkArgument(newSerializer instanceof ListSerializer);

	TypeSerializer<V> priorElementSerializer =
		((ListSerializer<V>) priorSerializer).getElementSerializer();

	TypeSerializer<V> newElementSerializer =
		((ListSerializer<V>) newSerializer).getElementSerializer();

	try {
		while (serializedOldValueInput.available() > 0) {
			V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer);
			newElementSerializer.serialize(element, serializedMigratedValueOutput);
			if (serializedOldValueInput.available() > 0) {
				serializedMigratedValueOutput.write(DELIMITER);
			}
		}
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
	}
}