Java Code Examples for org.apache.flink.api.java.ClosureCleaner#ensureSerializable()

@Test
public void testWrapperClass() throws Exception {
	MapCreator creator = new NonSerializableMapCreator();
	MapFunction<Integer, Integer> notCleanedMap = creator.getMap();

	WrapperMapFunction wrapped = new WrapperMapFunction(notCleanedMap);

	ClosureCleaner.clean(wrapped, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(wrapped);

	int result = wrapped.map(3);
	Assert.assertEquals(result, 4);
}
 

protected void validate(ParameterTool parameter) {
	ClosureCleaner.ensureSerializable(parameter);
	internalValidate(parameter);

	// -------- test behaviour after serialization ------------
	try {
		byte[] b = InstantiationUtil.serializeObject(parameter);
		final ParameterTool copy = InstantiationUtil.deserializeObject(b, getClass().getClassLoader());
		internalValidate(copy);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
}
 

@Test
public void testRecursiveClass() {
	RecursiveClass recursiveClass = new RecursiveClass(new RecursiveClass());

	ClosureCleaner.clean(recursiveClass, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(recursiveClass);
}
 

/**
 * Returns a "closure-cleaned" version of the given function. Cleans only if closure cleaning
 * is not disabled in the {@link org.apache.flink.api.common.ExecutionConfig}
 */
@Internal
public <F> F clean(F f) {
	if (getConfig().isClosureCleanerEnabled()) {
		ClosureCleaner.clean(f, getConfig().getClosureCleanerLevel(), true);
	}
	ClosureCleaner.ensureSerializable(f);
	return f;
}
 

public FlinkPulsarProducer(ClientConfigurationData clientConfigurationData,
                           ProducerConfigurationData producerConfigurationData,
                           SerializationSchema<T> serializationSchema,
                           PulsarKeyExtractor<T> keyExtractor,
                           PulsarPropertiesExtractor<T> propertiesExtractor) {
    this.clientConf = checkNotNull(clientConfigurationData, "client conf can not be null");
    this.producerConf = checkNotNull(producerConfigurationData, "producer conf can not be null");
    this.schema = checkNotNull(serializationSchema, "Serialization Schema not set");
    this.flinkPulsarKeyExtractor = getOrNullKeyExtractor(keyExtractor);
    this.flinkPulsarPropertiesExtractor = getOrNullPropertiesExtractor(propertiesExtractor);
    ClosureCleaner.ensureSerializable(serializationSchema);
}
 

@Test(expected = InvalidProgramException.class)
public void testNonSerializable() throws Exception  {
	MapCreator creator = new NonSerializableMapCreator();
	MapFunction<Integer, Integer> map = creator.getMap();

	ClosureCleaner.ensureSerializable(map);

	int result = map.map(3);
	Assert.assertEquals(result, 4);
}
 

@Test
public void testWrapperClass() throws Exception {
	MapCreator creator = new NonSerializableMapCreator();
	MapFunction<Integer, Integer> notCleanedMap = creator.getMap();

	WrapperMapFunction wrapped = new WrapperMapFunction(notCleanedMap);

	ClosureCleaner.clean(wrapped, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(wrapped);

	int result = wrapped.map(3);
	Assert.assertEquals(result, 4);
}
 

@Test
public void testNestedSerializable() throws Exception  {
	MapCreator creator = new NestedSerializableMapCreator(1);
	MapFunction<Integer, Integer> map = creator.getMap();

	ClosureCleaner.clean(map, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(map);

	int result = map.map(3);
	Assert.assertEquals(result, 4);
}
 

@Test
public void testNestedSerializable() throws Exception  {
	MapCreator creator = new NestedSerializableMapCreator(1);
	MapFunction<Integer, Integer> map = creator.getMap();

	ClosureCleaner.clean(map, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(map);

	int result = map.map(3);
	Assert.assertEquals(result, 4);
}
 

protected void validate(ParameterTool parameter) {
	ClosureCleaner.ensureSerializable(parameter);
	internalValidate(parameter);

	// -------- test behaviour after serialization ------------
	try {
		byte[] b = InstantiationUtil.serializeObject(parameter);
		final ParameterTool copy = InstantiationUtil.deserializeObject(b, getClass().getClassLoader());
		internalValidate(copy);
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
}
 

/**
 * The main constructor for creating a FlinkKafkaProducer.
 *
 * @param defaultTopicId The default topic to write data to
 * @param serializationSchema A serializable serialization schema for turning user objects into a kafka-consumable byte[] supporting key/value messages
 * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is the only required argument.
 * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions. Passing null will use Kafka's partitioner.
 */
public FlinkKafkaProducerBase(String defaultTopicId, KeyedSerializationSchema<IN> serializationSchema, Properties producerConfig, FlinkKafkaPartitioner<IN> customPartitioner) {
	requireNonNull(defaultTopicId, "TopicID not set");
	requireNonNull(serializationSchema, "serializationSchema not set");
	requireNonNull(producerConfig, "producerConfig not set");
	ClosureCleaner.clean(customPartitioner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
	ClosureCleaner.ensureSerializable(serializationSchema);

	this.defaultTopicId = defaultTopicId;
	this.schema = serializationSchema;
	this.producerConfig = producerConfig;
	this.flinkKafkaPartitioner = customPartitioner;

	// set the producer configuration properties for kafka record key value serializers.
	if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
	}

	if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
	}

	// eagerly ensure that bootstrap servers are set.
	if (!this.producerConfig.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
		throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be supplied in the producer config properties.");
	}

	this.topicPartitionsMap = new HashMap<>();
}
 

@Test(expected = InvalidProgramException.class)
public void testNestedNonSerializable() throws Exception  {
	MapCreator creator = new NestedNonSerializableMapCreator(1);
	MapFunction<Integer, Integer> map = creator.getMap();

	ClosureCleaner.clean(map, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(map);

	int result = map.map(3);
	Assert.assertEquals(result, 4);
}
 

/**
 * Set a custom endpoint initializer.
 */
public void setCustomEndpointInitializer(EndpointInitializer initializer) {
	Objects.requireNonNull(initializer, "Initializer has to be set");
	ClosureCleaner.ensureSerializable(initializer);
	this.initializer = initializer;
}
 

/**
 * Set a custom endpoint initializer.
 */
public void setCustomEndpointInitializer(EndpointInitializer initializer) {
	Objects.requireNonNull(initializer, "Initializer has to be set");
	ClosureCleaner.ensureSerializable(initializer);
	this.initializer = initializer;
}
 

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces its input to
 * the topic. It accepts a keyed {@link KeyedSerializationSchema} and possibly a custom {@link FlinkKafkaPartitioner}.
 *
 * <p>If a partitioner is not provided, written records will be partitioned by the attached key of each
 * record (as determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If written records do not
 * have a key (i.e., {@link KeyedSerializationSchema#serializeKey(Object)} returns {@code null}), they
 * will be distributed to Kafka partitions in a round-robin fashion.
 *
 * @param defaultTopicId The default topic to write data to
 * @param serializationSchema A serializable serialization schema for turning user objects into a kafka-consumable byte[] supporting key/value messages
 * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is the only required argument.
 * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions.
 *                          If a partitioner is not provided, records will be partitioned by the key of each record
 *                          (determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If the keys
 *                          are {@code null}, then records will be distributed to Kafka partitions in a
 *                          round-robin fashion.
 * @param semantic Defines semantic that will be used by this producer (see {@link Semantic}).
 * @param kafkaProducersPoolSize Overwrite default KafkaProducers pool size (see {@link Semantic#EXACTLY_ONCE}).
 */
public FlinkKafkaProducer011(
		String defaultTopicId,
		KeyedSerializationSchema<IN> serializationSchema,
		Properties producerConfig,
		Optional<FlinkKafkaPartitioner<IN>> customPartitioner,
		Semantic semantic,
		int kafkaProducersPoolSize) {
	super(new TransactionStateSerializer(), new ContextStateSerializer());

	this.defaultTopicId = checkNotNull(defaultTopicId, "defaultTopicId is null");
	this.schema = checkNotNull(serializationSchema, "serializationSchema is null");
	this.producerConfig = checkNotNull(producerConfig, "producerConfig is null");
	this.flinkKafkaPartitioner = checkNotNull(customPartitioner, "customPartitioner is null").orElse(null);
	this.semantic = checkNotNull(semantic, "semantic is null");
	this.kafkaProducersPoolSize = kafkaProducersPoolSize;
	checkState(kafkaProducersPoolSize > 0, "kafkaProducersPoolSize must be non empty");

	ClosureCleaner.clean(this.flinkKafkaPartitioner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
	ClosureCleaner.ensureSerializable(serializationSchema);

	// set the producer configuration properties for kafka record key value serializers.
	if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
	}

	if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
	}

	// eagerly ensure that bootstrap servers are set.
	if (!this.producerConfig.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
		throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be supplied in the producer config properties.");
	}

	if (!producerConfig.containsKey(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG)) {
		long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
		checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
		this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
		LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
	}

	// Enable transactionTimeoutWarnings to avoid silent data loss
	// See KAFKA-6119 (affects versions 0.11.0.0 and 0.11.0.1):
	// The KafkaProducer may not throw an exception if the transaction failed to commit
	if (semantic == Semantic.EXACTLY_ONCE) {
		final Object object = this.producerConfig.get(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG);
		final long transactionTimeout;
		if (object instanceof String && StringUtils.isNumeric((String) object)) {
			transactionTimeout = Long.parseLong((String) object);
		} else if (object instanceof Number) {
			transactionTimeout = ((Number) object).longValue();
		} else {
			throw new IllegalArgumentException(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG
				+ " must be numeric, was " + object);
		}
		super.setTransactionTimeout(transactionTimeout);
		super.enableTransactionTimeoutWarnings(0.8);
	}

	this.topicPartitionsMap = new HashMap<>();
}
 

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces its input to
 * the topic. It accepts a keyed {@link KeyedSerializationSchema} and possibly a custom {@link FlinkKafkaPartitioner}.
 *
 * <p>If a partitioner is not provided, written records will be partitioned by the attached key of each
 * record (as determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If written records do not
 * have a key (i.e., {@link KeyedSerializationSchema#serializeKey(Object)} returns {@code null}), they
 * will be distributed to Kafka partitions in a round-robin fashion.
 *
 * @param defaultTopicId The default topic to write data to
 * @param serializationSchema A serializable serialization schema for turning user objects into a kafka-consumable byte[] supporting key/value messages
 * @param producerConfig Configuration properties for the KafkaProducer. 'bootstrap.servers.' is the only required argument.
 * @param customPartitioner A serializable partitioner for assigning messages to Kafka partitions.
 *                          If a partitioner is not provided, records will be partitioned by the key of each record
 *                          (determined by {@link KeyedSerializationSchema#serializeKey(Object)}). If the keys
 *                          are {@code null}, then records will be distributed to Kafka partitions in a
 *                          round-robin fashion.
 * @param semantic Defines semantic that will be used by this producer (see {@link Semantic}).
 * @param kafkaProducersPoolSize Overwrite default KafkaProducers pool size (see {@link Semantic#EXACTLY_ONCE}).
 */
public FlinkKafkaProducer011(
		String defaultTopicId,
		KeyedSerializationSchema<IN> serializationSchema,
		Properties producerConfig,
		Optional<FlinkKafkaPartitioner<IN>> customPartitioner,
		Semantic semantic,
		int kafkaProducersPoolSize) {
	super(new TransactionStateSerializer(), new ContextStateSerializer());

	this.defaultTopicId = checkNotNull(defaultTopicId, "defaultTopicId is null");
	this.schema = checkNotNull(serializationSchema, "serializationSchema is null");
	this.producerConfig = checkNotNull(producerConfig, "producerConfig is null");
	this.flinkKafkaPartitioner = checkNotNull(customPartitioner, "customPartitioner is null").orElse(null);
	this.semantic = checkNotNull(semantic, "semantic is null");
	this.kafkaProducersPoolSize = kafkaProducersPoolSize;
	checkState(kafkaProducersPoolSize > 0, "kafkaProducersPoolSize must be non empty");

	ClosureCleaner.clean(this.flinkKafkaPartitioner, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
	ClosureCleaner.ensureSerializable(serializationSchema);

	// set the producer configuration properties for kafka record key value serializers.
	if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
	}

	if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) {
		this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
	} else {
		LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
	}

	// eagerly ensure that bootstrap servers are set.
	if (!this.producerConfig.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)) {
		throw new IllegalArgumentException(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be supplied in the producer config properties.");
	}

	if (!producerConfig.containsKey(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG)) {
		long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
		checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
		this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
		LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
	}

	// Enable transactionTimeoutWarnings to avoid silent data loss
	// See KAFKA-6119 (affects versions 0.11.0.0 and 0.11.0.1):
	// The KafkaProducer may not throw an exception if the transaction failed to commit
	if (semantic == Semantic.EXACTLY_ONCE) {
		final Object object = this.producerConfig.get(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG);
		final long transactionTimeout;
		if (object instanceof String && StringUtils.isNumeric((String) object)) {
			transactionTimeout = Long.parseLong((String) object);
		} else if (object instanceof Number) {
			transactionTimeout = ((Number) object).longValue();
		} else {
			throw new IllegalArgumentException(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG
				+ " must be numeric, was " + object);
		}
		super.setTransactionTimeout(transactionTimeout);
		super.enableTransactionTimeoutWarnings(0.8);
	}

	this.topicPartitionsMap = new HashMap<>();
}
 

@Test
public void testRealOuterStaticClassInnerStaticClassInnerAnonymousOrLocalClass() {
	MapFunction<Integer, Integer> nestedMap = new OuterMapCreator().getMap();

	MapFunction<Integer, Integer> wrappedMap = new WrapperMapFunction(nestedMap);

	Tuple1<MapFunction<Integer, Integer>> tuple = new Tuple1<>(wrappedMap);

	ClosureCleaner.clean(tuple, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(tuple);
}
 

@Test
public void testOuterStaticClassInnerStaticClassInnerAnonymousOrLocalClass() {
	MapFunction<Integer, Integer> nestedMap = new OuterMapCreator().getMap();

	MapFunction<Integer, Integer> wrappedMap = new WrapperMapFunction(nestedMap);

	ClosureCleaner.clean(wrappedMap, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(wrappedMap);
}
 

@Test
public void testOuterStaticClassInnerStaticClassInnerAnonymousOrLocalClass() {
	MapFunction<Integer, Integer> nestedMap = new OuterMapCreator().getMap();

	MapFunction<Integer, Integer> wrappedMap = new WrapperMapFunction(nestedMap);

	ClosureCleaner.clean(wrappedMap, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(wrappedMap);
}
 

@Test
public void testRealOuterStaticClassInnerStaticClassInnerAnonymousOrLocalClass() {
	MapFunction<Integer, Integer> nestedMap = new OuterMapCreator().getMap();

	MapFunction<Integer, Integer> wrappedMap = new WrapperMapFunction(nestedMap);

	Tuple1<MapFunction<Integer, Integer>> tuple = new Tuple1<>(wrappedMap);

	ClosureCleaner.clean(tuple, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);

	ClosureCleaner.ensureSerializable(tuple);
}