Java Code Examples for org.apache.flink.api.common.typeutils.TypeSerializer#duplicate()

/**
 * Returns the {@link TypeSerializer} that can be used to serialize the value in the state.
 * Note that the serializer may initialized lazily and is only guaranteed to exist after
 * calling {@link #initializeSerializerUnlessSet(ExecutionConfig)}.
 */
public TypeSerializer<T> getSerializer() {
	TypeSerializer<T> serializer = serializerAtomicReference.get();
	if (serializer != null) {
		return serializer.duplicate();
	} else {
		throw new IllegalStateException("Serializer not yet initialized.");
	}
}
 

private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 

/**
 * Returns the {@link TypeSerializer} that can be used to serialize the value in the state.
 * Note that the serializer may initialized lazily and is only guaranteed to exist after
 * calling {@link #initializeSerializerUnlessSet(ExecutionConfig)}.
 */
public TypeSerializer<T> getSerializer() {
	TypeSerializer<T> serializer = serializerAtomicReference.get();
	if (serializer != null) {
		return serializer.duplicate();
	} else {
		throw new IllegalStateException("Serializer not yet initialized.");
	}
}
 

private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 

/**
 * Returns the {@link TypeSerializer} that can be used to serialize the value in the state.
 * Note that the serializer may initialized lazily and is only guaranteed to exist after
 * calling {@link #initializeSerializerUnlessSet(ExecutionConfig)}.
 */
public TypeSerializer<T> getSerializer() {
	TypeSerializer<T> serializer = serializerAtomicReference.get();
	if (serializer != null) {
		return serializer.duplicate();
	} else {
		throw new IllegalStateException("Serializer not yet initialized.");
	}
}
 

private void writeObject(final ObjectOutputStream out) throws IOException {
	// write all the non-transient fields
	out.defaultWriteObject();

	// write the non-serializable default value field
	if (defaultValue == null) {
		// we don't have a default value
		out.writeBoolean(false);
	} else {
		TypeSerializer<T> serializer = serializerAtomicReference.get();
		checkNotNull(serializer, "Serializer not initialized.");

		// we have a default value
		out.writeBoolean(true);

		byte[] serializedDefaultValue;
		try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
				DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) {

			TypeSerializer<T> duplicateSerializer = serializer.duplicate();
			duplicateSerializer.serialize(defaultValue, outView);

			outView.flush();
			serializedDefaultValue = baos.toByteArray();
		}
		catch (Exception e) {
			throw new IOException("Unable to serialize default value of type " +
					defaultValue.getClass().getSimpleName() + ".", e);
		}

		out.writeInt(serializedDefaultValue.length);
		out.write(serializedDefaultValue);
	}
}
 

private <S> ListState<S> getListState(
		ListStateDescriptor<S> stateDescriptor,
		OperatorStateHandle.Mode mode) throws StateMigrationException {

	Preconditions.checkNotNull(stateDescriptor);
	String name = Preconditions.checkNotNull(stateDescriptor.getName());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> previous = (PartitionableListState<S>) accessedStatesByName.get(name);
	if (previous != null) {
		checkStateNameAndMode(
				previous.getStateMetaInfo().getName(),
				name,
				previous.getStateMetaInfo().getAssignmentMode(),
				mode);
		return previous;
	}

	// end up here if its the first time access after execution for the
	// provided state name; check compatibility of restored state, if any
	// TODO with eager registration in place, these checks should be moved to restore()

	stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
	TypeSerializer<S> partitionStateSerializer = Preconditions.checkNotNull(stateDescriptor.getElementSerializer());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredOperatorStates.get(name);

	if (null == partitionableListState) {
		// no restored state for the state name; simply create new state holder

		partitionableListState = new PartitionableListState<>(
			new RegisteredOperatorStateBackendMetaInfo<>(
				name,
				partitionStateSerializer,
				mode));

		registeredOperatorStates.put(name, partitionableListState);
	} else {
		// has restored state; check compatibility of new state access

		checkStateNameAndMode(
				partitionableListState.getStateMetaInfo().getName(),
				name,
				partitionableListState.getStateMetaInfo().getAssignmentMode(),
				mode);

		RegisteredOperatorStateBackendMetaInfo<S> restoredPartitionableListStateMetaInfo =
			partitionableListState.getStateMetaInfo();

		// check compatibility to determine if new serializers are incompatible
		TypeSerializer<S> newPartitionStateSerializer = partitionStateSerializer.duplicate();

		TypeSerializerSchemaCompatibility<S> stateCompatibility =
			restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer);
		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new state typeSerializer for operator state must not be incompatible.");
		}

		partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo);
	}

	accessedStatesByName.put(name, partitionableListState);
	return partitionableListState;
}
 

private <S> ListState<S> getListState(
		ListStateDescriptor<S> stateDescriptor,
		OperatorStateHandle.Mode mode) throws StateMigrationException {

	Preconditions.checkNotNull(stateDescriptor);
	String name = Preconditions.checkNotNull(stateDescriptor.getName());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> previous = (PartitionableListState<S>) accessedStatesByName.get(name);
	if (previous != null) {
		checkStateNameAndMode(
				previous.getStateMetaInfo().getName(),
				name,
				previous.getStateMetaInfo().getAssignmentMode(),
				mode);
		return previous;
	}

	// end up here if its the first time access after execution for the
	// provided state name; check compatibility of restored state, if any
	// TODO with eager registration in place, these checks should be moved to restore()

	stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
	TypeSerializer<S> partitionStateSerializer = Preconditions.checkNotNull(stateDescriptor.getElementSerializer());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredOperatorStates.get(name);

	if (null == partitionableListState) {
		// no restored state for the state name; simply create new state holder

		partitionableListState = new PartitionableListState<>(
			new RegisteredOperatorStateBackendMetaInfo<>(
				name,
				partitionStateSerializer,
				mode));

		registeredOperatorStates.put(name, partitionableListState);
	} else {
		// has restored state; check compatibility of new state access

		checkStateNameAndMode(
				partitionableListState.getStateMetaInfo().getName(),
				name,
				partitionableListState.getStateMetaInfo().getAssignmentMode(),
				mode);

		RegisteredOperatorStateBackendMetaInfo<S> restoredPartitionableListStateMetaInfo =
			partitionableListState.getStateMetaInfo();

		// check compatibility to determine if new serializers are incompatible
		TypeSerializer<S> newPartitionStateSerializer = partitionStateSerializer.duplicate();

		TypeSerializerSchemaCompatibility<S> stateCompatibility =
			restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer);
		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new state typeSerializer for operator state must not be incompatible.");
		}

		partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo);
	}

	accessedStatesByName.put(name, partitionableListState);
	return partitionableListState;
}
 

@Override
public void initializeState(StateInitializationContext context) throws Exception {
  super.initializeState(context);

  final Configuration configuration = getConfiguration();
  final StatefulFunctionsUniverse statefulFunctionsUniverse =
      statefulFunctionsUniverse(configuration);

  final TypeSerializer<Message> envelopeSerializer =
      getOperatorConfig().getTypeSerializerIn1(getContainingTask().getUserCodeClassLoader());
  final Executor checkpointLockExecutor =
      new UnderCheckpointLockExecutor(
          getContainingTask().getCheckpointLock(), () -> closedOrDisposed, getContainingTask());
  final MapStateDescriptor<Long, Message> asyncOperationStateDescriptor =
      new MapStateDescriptor<>(
          "asyncOperations", LongSerializer.INSTANCE, envelopeSerializer.duplicate());
  final ListStateDescriptor<Message> delayedMessageStateDescriptor =
      new ListStateDescriptor<>(
          FlinkStateDelayedMessagesBuffer.BUFFER_STATE_NAME, envelopeSerializer.duplicate());
  final MapState<Long, Message> asyncOperationState =
      getRuntimeContext().getMapState(asyncOperationStateDescriptor);

  Objects.requireNonNull(mailboxExecutor, "MailboxExecutor is unexpectedly NULL");

  //
  // the core logic of applying messages to functions.
  //
  this.reductions =
      Reductions.create(
          configuration,
          statefulFunctionsUniverse,
          getRuntimeContext(),
          getKeyedStateBackend(),
          new FlinkTimerServiceFactory(super.timeServiceManager),
          delayedMessagesBufferState(delayedMessageStateDescriptor),
          sideOutputs,
          output,
          MessageFactory.forType(statefulFunctionsUniverse.messageFactoryType()),
          new MailboxExecutorFacade(mailboxExecutor, "Stateful Functions Mailbox"),
          getRuntimeContext().getMetricGroup().addGroup("functions"),
          asyncOperationState,
          checkpointLockExecutor);
  //
  // expire all the pending async operations.
  //
  AsyncOperationFailureNotifier.fireExpiredAsyncOperations(
      asyncOperationStateDescriptor, reductions, asyncOperationState, getKeyedStateBackend());
}
 

@Override
public void initializeState(StateInitializationContext context) throws Exception {
  super.initializeState(context);

  final StatefulFunctionsUniverse statefulFunctionsUniverse =
      statefulFunctionsUniverse(configuration);

  final TypeSerializer<Message> envelopeSerializer =
      getOperatorConfig().getTypeSerializerIn1(getContainingTask().getUserCodeClassLoader());
  final MapStateDescriptor<Long, Message> asyncOperationStateDescriptor =
      new MapStateDescriptor<>(
          "asyncOperations", LongSerializer.INSTANCE, envelopeSerializer.duplicate());
  final ListStateDescriptor<Message> delayedMessageStateDescriptor =
      new ListStateDescriptor<>(
          FlinkStateDelayedMessagesBuffer.BUFFER_STATE_NAME, envelopeSerializer.duplicate());
  final MapState<Long, Message> asyncOperationState =
      getRuntimeContext().getMapState(asyncOperationStateDescriptor);

  Objects.requireNonNull(mailboxExecutor, "MailboxExecutor is unexpectedly NULL");

  this.backPressureValve =
      new ThresholdBackPressureValve(configuration.getMaxAsyncOperationsPerTask());

  //
  // the core logic of applying messages to functions.
  //
  this.reductions =
      Reductions.create(
          backPressureValve,
          statefulFunctionsUniverse,
          getRuntimeContext(),
          getKeyedStateBackend(),
          new FlinkTimerServiceFactory(super.timeServiceManager),
          delayedMessagesBufferState(delayedMessageStateDescriptor),
          sideOutputs,
          output,
          MessageFactory.forType(statefulFunctionsUniverse.messageFactoryType()),
          new MailboxExecutorFacade(mailboxExecutor, "Stateful Functions Mailbox"),
          getRuntimeContext().getMetricGroup().addGroup("functions"),
          asyncOperationState);
  //
  // expire all the pending async operations.
  //
  AsyncOperationFailureNotifier.fireExpiredAsyncOperations(
      asyncOperationStateDescriptor, reductions, getKeyedStateBackend());
}
 

private <S> ListState<S> getListState(
		ListStateDescriptor<S> stateDescriptor,
		OperatorStateHandle.Mode mode) throws StateMigrationException {

	Preconditions.checkNotNull(stateDescriptor);
	String name = Preconditions.checkNotNull(stateDescriptor.getName());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> previous = (PartitionableListState<S>) accessedStatesByName.get(name);
	if (previous != null) {
		checkStateNameAndMode(
				previous.getStateMetaInfo().getName(),
				name,
				previous.getStateMetaInfo().getAssignmentMode(),
				mode);
		return previous;
	}

	// end up here if its the first time access after execution for the
	// provided state name; check compatibility of restored state, if any
	// TODO with eager registration in place, these checks should be moved to restore()

	stateDescriptor.initializeSerializerUnlessSet(getExecutionConfig());
	TypeSerializer<S> partitionStateSerializer = Preconditions.checkNotNull(stateDescriptor.getElementSerializer());

	@SuppressWarnings("unchecked")
	PartitionableListState<S> partitionableListState = (PartitionableListState<S>) registeredOperatorStates.get(name);

	if (null == partitionableListState) {
		// no restored state for the state name; simply create new state holder

		partitionableListState = new PartitionableListState<>(
			new RegisteredOperatorStateBackendMetaInfo<>(
				name,
				partitionStateSerializer,
				mode));

		registeredOperatorStates.put(name, partitionableListState);
	} else {
		// has restored state; check compatibility of new state access

		checkStateNameAndMode(
				partitionableListState.getStateMetaInfo().getName(),
				name,
				partitionableListState.getStateMetaInfo().getAssignmentMode(),
				mode);

		RegisteredOperatorStateBackendMetaInfo<S> restoredPartitionableListStateMetaInfo =
			partitionableListState.getStateMetaInfo();

		// check compatibility to determine if new serializers are incompatible
		TypeSerializer<S> newPartitionStateSerializer = partitionStateSerializer.duplicate();

		TypeSerializerSchemaCompatibility<S> stateCompatibility =
			restoredPartitionableListStateMetaInfo.updatePartitionStateSerializer(newPartitionStateSerializer);
		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("The new state typeSerializer for operator state must not be incompatible.");
		}

		partitionableListState.setStateMetaInfo(restoredPartitionableListStateMetaInfo);
	}

	accessedStatesByName.put(name, partitionableListState);
	return partitionableListState;
}
 

private ServerThread(TypeSerializer<IN> serializer) throws Exception {
	this.serializer = serializer.duplicate();
	this.serverSocket = new ServerSocket(0, 0, getBindAddress());
	this.running = true;
}