Java Code Examples for org.apache.flink.util.InstantiationUtil#clone()

private SimpleConsumerThread<T> createAndStartSimpleConsumerThread(
		List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions,
		Node leader,
		ExceptionProxy errorHandler) throws IOException, ClassNotFoundException {
	// each thread needs its own copy of the deserializer, because the deserializer is
	// not necessarily thread safe
	final KafkaDeserializationSchema<T> clonedDeserializer =
			InstantiationUtil.clone(deserializer, runtimeContext.getUserCodeClassLoader());

	// seed thread with list of fetch partitions (otherwise it would shut down immediately again
	SimpleConsumerThread<T> brokerThread = new SimpleConsumerThread<>(
			this, errorHandler, kafkaConfig, leader, seedPartitions, unassignedPartitionsQueue,
			clonedDeserializer, invalidOffsetBehavior);

	brokerThread.setName(String.format("SimpleConsumer - %s - broker-%s (%s:%d)",
			runtimeContext.getTaskName(), leader.id(), leader.host(), leader.port()));
	brokerThread.setDaemon(true);
	brokerThread.start();

	LOG.info("Starting thread {}", brokerThread.getName());
	return brokerThread;
}
 

private SimpleConsumerThread<T> createAndStartSimpleConsumerThread(
		List<KafkaTopicPartitionState<TopicAndPartition>> seedPartitions,
		Node leader,
		ExceptionProxy errorHandler) throws IOException, ClassNotFoundException {
	// each thread needs its own copy of the deserializer, because the deserializer is
	// not necessarily thread safe
	final KafkaDeserializationSchema<T> clonedDeserializer =
			InstantiationUtil.clone(deserializer, runtimeContext.getUserCodeClassLoader());

	// seed thread with list of fetch partitions (otherwise it would shut down immediately again
	SimpleConsumerThread<T> brokerThread = new SimpleConsumerThread<>(
			this, errorHandler, kafkaConfig, leader, seedPartitions, unassignedPartitionsQueue,
			clonedDeserializer, invalidOffsetBehavior);

	brokerThread.setName(String.format("SimpleConsumer - %s - broker-%s (%s:%d)",
			runtimeContext.getTaskName(), leader.id(), leader.host(), leader.port()));
	brokerThread.setDaemon(true);
	brokerThread.start();

	LOG.info("Starting thread {}", brokerThread.getName());
	return brokerThread;
}
 

@Override
public AggregateFunction makeFunction(Object[] constantArguments, LogicalType[] argTypes) {
	AggregateFunction clone;
	try {
		clone = InstantiationUtil.clone(aggregateFunction);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
	return (AggregateFunction) invokeSetArgs(clone, constantArguments, argTypes);
}
 

@Override
public ScalarFunction makeFunction(Object[] constantArguments, LogicalType[] argTypes) {
	ScalarFunction clone;
	try {
		clone = InstantiationUtil.clone(function);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
	return (ScalarFunction) invokeSetArgs(clone, constantArguments, argTypes);
}
 

@Override
public AggregateFunction makeFunction(Object[] constantArguments, LogicalType[] argTypes) {
	AggregateFunction clone;
	try {
		clone = InstantiationUtil.clone(aggregateFunction);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
	return (AggregateFunction) invokeSetArgs(clone, constantArguments, argTypes);
}
 

/**
 * Register a new subscribed shard state.
 *
 * @param newSubscribedShardState the new shard state that this fetcher is to be subscribed to
 */
public int registerNewSubscribedShardState(KinesisStreamShardState newSubscribedShardState) {
	synchronized (checkpointLock) {
		subscribedShardsState.add(newSubscribedShardState);

		// If a registered shard has initial state that is not SENTINEL_SHARD_ENDING_SEQUENCE_NUM (will be the case
		// if the consumer had already finished reading a shard before we failed and restored), we determine that
		// this subtask has a new active shard
		if (!newSubscribedShardState.getLastProcessedSequenceNum().equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) {
			this.numberOfActiveShards.incrementAndGet();
		}

		int shardStateIndex = subscribedShardsState.size() - 1;

		// track all discovered shards for watermark determination
		ShardWatermarkState sws = shardWatermarks.get(shardStateIndex);
		if (sws == null) {
			sws = new ShardWatermarkState();
			try {
				sws.periodicWatermarkAssigner = InstantiationUtil.clone(periodicWatermarkAssigner);
			} catch (Exception e) {
				throw new RuntimeException("Failed to instantiate new WatermarkAssigner", e);
			}
			sws.emitQueue = recordEmitter.getQueue(shardStateIndex);
			sws.lastUpdated = getCurrentTimeMillis();
			sws.lastRecordTimestamp = Long.MIN_VALUE;
			shardWatermarks.put(shardStateIndex, sws);
		}

		return shardStateIndex;
	}
}
 

private KinesisDeserializationSchema<T> getClonedDeserializationSchema() {
	try {
		return InstantiationUtil.clone(deserializationSchema, runtimeContext.getUserCodeClassLoader());
	} catch (IOException | ClassNotFoundException ex) {
		// this really shouldn't happen; simply wrap it around a runtime exception
		throw new RuntimeException(ex);
	}
}
 

@Override
public TimestampedHiveInputSplit copy(TimestampedHiveInputSplit from) {
	try {
		return InstantiationUtil.clone(from, Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException e) {
		throw new FlinkRuntimeException("Could not copy element via serialization: " + from, e);
	}
}
 

@Override
public T copy(T from) {
	try {
		return InstantiationUtil.clone(from, Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException e) {
		throw new FlinkRuntimeException("Could not copy element via serialization: " + from, e);
	}
}
 

private ExecutionConfig.SerializableSerializer<? extends Serializer<?>> deepCopySerializer(
	ExecutionConfig.SerializableSerializer<? extends Serializer<?>> original) {
	try {
		return InstantiationUtil.clone(original, Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException ex) {
		throw new CloneFailedException(
			"Could not clone serializer instance of class " + original.getClass(),
			ex);
	}
}
 

@Override
public TableFunction makeFunction(Object[] constantArguments, LogicalType[] argTypes) {
	TableFunction clone;
	try {
		clone = InstantiationUtil.clone(hiveUdtf);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
	return (TableFunction) invokeSetArgs(clone, constantArguments, argTypes);
}
 

@Override
public ScalarFunction makeFunction(Object[] constantArguments, LogicalType[] argTypes) {
	ScalarFunction clone;
	try {
		clone = InstantiationUtil.clone(function);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
	return (ScalarFunction) invokeSetArgs(clone, constantArguments, argTypes);
}
 

@Override
public TableSink<Row> configure(String[] fieldNames, TypeInformation<?>[] fieldTypes) {
	int[] types = outputFormat.getTypesArray();

	String sinkSchema =
		String.join(", ", IntStream.of(types).mapToObj(JDBCTypeUtil::getTypeName).collect(Collectors.toList()));
	String tableSchema =
		String.join(", ", Stream.of(fieldTypes).map(JDBCTypeUtil::getTypeName).collect(Collectors.toList()));
	String msg = String.format("Schema of output table is incompatible with JDBCAppendTableSink schema. " +
		"Table schema: [%s], sink schema: [%s]", tableSchema, sinkSchema);

	Preconditions.checkArgument(fieldTypes.length == types.length, msg);
	for (int i = 0; i < types.length; ++i) {
		Preconditions.checkArgument(
			JDBCTypeUtil.typeInformationToSqlType(fieldTypes[i]) == types[i],
			msg);
	}

	JDBCAppendTableSink copy;
	try {
		copy = new JDBCAppendTableSink(InstantiationUtil.clone(outputFormat));
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}

	copy.fieldNames = fieldNames;
	copy.fieldTypes = fieldTypes;
	return copy;
}
 

/**
 * Register a new subscribed shard state.
 *
 * @param newSubscribedShardState the new shard state that this fetcher is to be subscribed to
 */
public int registerNewSubscribedShardState(KinesisStreamShardState newSubscribedShardState) {
	synchronized (checkpointLock) {
		subscribedShardsState.add(newSubscribedShardState);

		// If a registered shard has initial state that is not SENTINEL_SHARD_ENDING_SEQUENCE_NUM (will be the case
		// if the consumer had already finished reading a shard before we failed and restored), we determine that
		// this subtask has a new active shard
		if (!newSubscribedShardState.getLastProcessedSequenceNum().equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) {
			this.numberOfActiveShards.incrementAndGet();
		}

		int shardStateIndex = subscribedShardsState.size() - 1;

		// track all discovered shards for watermark determination
		ShardWatermarkState sws = shardWatermarks.get(shardStateIndex);
		if (sws == null) {
			sws = new ShardWatermarkState();
			try {
				sws.periodicWatermarkAssigner = InstantiationUtil.clone(periodicWatermarkAssigner);
			} catch (Exception e) {
				throw new RuntimeException("Failed to instantiate new WatermarkAssigner", e);
			}
			sws.emitQueue = recordEmitter.getQueue(shardStateIndex);
			sws.lastUpdated = getCurrentTimeMillis();
			sws.lastRecordTimestamp = Long.MIN_VALUE;
			shardWatermarks.put(shardStateIndex, sws);
		}

		return shardStateIndex;
	}
}
 

protected KinesisDeserializationSchema<T> getClonedDeserializationSchema() {
	try {
		return InstantiationUtil.clone(deserializationSchema, runtimeContext.getUserCodeClassLoader());
	} catch (IOException | ClassNotFoundException ex) {
		// this really shouldn't happen; simply wrap it around a runtime exception
		throw new RuntimeException(ex);
	}
}
 

@Override
public T copy(T from) {
	try {
		return InstantiationUtil.clone(from, Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException e) {
		throw new FlinkRuntimeException("Could not copy element via serialization: " + from, e);
	}
}
 

private ExecutionConfig.SerializableSerializer<? extends Serializer<?>> deepCopySerializer(
	ExecutionConfig.SerializableSerializer<? extends Serializer<?>> original) {
	try {
		return InstantiationUtil.clone(original, Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException ex) {
		throw new CloneFailedException(
			"Could not clone serializer instance of class " + original.getClass(),
			ex);
	}
}
 

@Override
public TableSink<Row> configure(String[] fieldNames, TypeInformation<?>[] fieldTypes) {
	int[] types = outputFormat.getTypesArray();

	String sinkSchema =
		String.join(", ", IntStream.of(types).mapToObj(JDBCTypeUtil::getTypeName).collect(Collectors.toList()));
	String tableSchema =
		String.join(", ", Stream.of(fieldTypes).map(JDBCTypeUtil::getTypeName).collect(Collectors.toList()));
	String msg = String.format("Schema of output table is incompatible with JDBCAppendTableSink schema. " +
		"Table schema: [%s], sink schema: [%s]", tableSchema, sinkSchema);

	Preconditions.checkArgument(fieldTypes.length == types.length, msg);
	for (int i = 0; i < types.length; ++i) {
		Preconditions.checkArgument(
			JDBCTypeUtil.typeInformationToSqlType(fieldTypes[i]) == types[i],
			msg);
	}

	JDBCAppendTableSink copy;
	try {
		copy = new JDBCAppendTableSink(InstantiationUtil.clone(outputFormat));
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}

	copy.fieldNames = fieldNames;
	copy.fieldTypes = fieldTypes;
	return copy;
}
 

private ExecutionConfig.SerializableSerializer<? extends Serializer<?>> deepCopySerializer(
	ExecutionConfig.SerializableSerializer<? extends Serializer<?>> original) {
	try {
		return InstantiationUtil.clone(original, Thread.currentThread().getContextClassLoader());
	} catch (IOException | ClassNotFoundException ex) {
		throw new CloneFailedException(
			"Could not clone serializer instance of class " + original.getClass(),
			ex);
	}
}
 

protected KinesisDeserializationSchema<T> getClonedDeserializationSchema() {
	try {
		return InstantiationUtil.clone(deserializationSchema, runtimeContext.getUserCodeClassLoader());
	} catch (IOException | ClassNotFoundException ex) {
		// this really shouldn't happen; simply wrap it around a runtime exception
		throw new RuntimeException(ex);
	}
}