org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner Java Examples

@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 

@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 

/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 

/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 

/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 

@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 

@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc);
}
 

@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc);
}
 

/**
 * Tests for when there are more partitions than sinks.
 * <pre>
 * 		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2	---------------->	2
 * 									3
 * 									4
 * 									5
 *
 * </pre>
 */
@Test
public void testFewerPartitions() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0, 1, 2, 3, 4};
	part.open(0, 2);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 2);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
}
 

@Override
public <T> DataStreamSink<T> writeToKafkaWithTimestamps(DataStream<T> stream, String topic, KeyedSerializationSchema<T> serSchema, Properties props) {
	FlinkKafkaProducer011<T> prod = new FlinkKafkaProducer011<>(
		topic, serSchema, props, Optional.of(new FlinkFixedPartitioner<>()), producerSemantic, FlinkKafkaProducer011.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE);

	prod.setWriteTimestampToKafka(true);

	return stream.addSink(prod);
}
 

public static void testFlinkAvroSQLJson() {
    System.out.println("TestCase_Test Avro SQL to Json Sink");
    final String STATIC_USER_SCHEMA = "{"
            + "\"type\":\"record\","
            + "\"name\":\"myrecord\","
            + "\"fields\":["
            + "  { \"name\":\"symbol\", \"type\":\"string\" },"
            + "  { \"name\":\"name\", \"type\":\"string\" },"
            + "  { \"name\":\"exchangecode\", \"type\":\"string\" }"
            + "]}";

    String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    DFRemoteStreamEnvironment env = new DFRemoteStreamEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("group.id", "consumer_test");
    properties.setProperty("schema.subject", "test-value");
    properties.setProperty("schema.registry", "localhost:8081");
    properties.setProperty("useAvro", "avro");
    properties.setProperty("static.avro.schema",
            SchemaRegistryClient.getSchemaFromRegistry("http://localhost:8081", "test-value", "latest").toString());

    try {
        HashMap<String, String> hm = new HashMap<>();
        Kafka09AvroTableSource kafkaAvroTableSource =  new Kafka09AvroTableSource("test", properties);
        tableEnv.registerTableSource("Orders", kafkaAvroTableSource);

        Table result = tableEnv.sql("SELECT name, symbol, exchangecode FROM Orders");
        //Kafka09JsonTableSink json_sink = new Kafka09JsonTableSink ("test_json", properties, new FlinkFixedPartitioner());
        Kafka09AvroTableSink json_sink = new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());

        // write the result Table to the TableSink
        result.writeToSink(json_sink);
        env.executeWithDFObj("Flink AVRO SQL KAFKA Test", new DFJobPOPJ().setJobConfig(hm) );
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

public static void tcFlinkAvroSQL(String KafkaServerHostPort, String SchemaRegistryHostPort,
                                  String srcTopic, String targetTopic,
                                  String consumerGroupId, String sinkKeys, String sqlState) {

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result = tableEnv.sql(sqlState);
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
        // For old producer, we need to create topic-value subject as well
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic + "-value", result);

        // delivered properties for sink
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        env.execute("DF_FlinkSQL_Client_" + srcTopic + "-" + targetTopic);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

/**
 * 构建 FlinkKafka09OutputComponent 对应的业务逻辑
 * @param flinkComponent
 */
private void dealFlinkKafka09OutputComponent(FlinkKafka09OutputComponent flinkComponent) {

    Properties kafkaProperties = new Properties();
    kafkaProperties.setProperty("bootstrap.servers",flinkComponent.getBrokerHostPorts());

    Kafka09JsonTableSink kafkaTableSink = new Kafka09JsonTableSink(
            flinkComponent.getTopic(),
            kafkaProperties,
            new FlinkFixedPartitioner());

    tableSinkMap.put(flinkComponent.getName(),kafkaTableSink);
}
 

@Test
public void testMixedCase() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();
	int[] partitions = new int[]{0, 1};

	part.open(0, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 3);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));

	part.open(2, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

}
 

/**
 * Tests for when there are more partitions than sinks.
 * <pre>
 * 		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2	---------------->	2
 * 									3
 * 									4
 * 									5
 *
 * </pre>
 */
@Test
public void testFewerPartitions() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0, 1, 2, 3, 4};
	part.open(0, 2);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 2);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
}
 

@Test
public void testMixedCase() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();
	int[] partitions = new int[]{0, 1};

	part.open(0, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 3);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));

	part.open(2, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

}
 

@Test
public void testTableSink() {
	final DataType consumedDataType = SINK_SCHEMA.toPhysicalRowDataType();
	EncodingFormat<SerializationSchema<RowData>> encodingFormat =
			new TestFormatFactory.EncodingFormatMock(",");

	// Construct table sink using options and table sink factory.
	ObjectIdentifier objectIdentifier = ObjectIdentifier.of(
			"default",
			"default",
			"sinkTable");
	final CatalogTable sinkTable = createKafkaSinkCatalogTable();
	final DynamicTableSink actualSink = FactoryUtil.createTableSink(
			null,
			objectIdentifier,
			sinkTable,
			new Configuration(),
			Thread.currentThread().getContextClassLoader());

	final DynamicTableSink expectedSink = getExpectedSink(
			consumedDataType,
			TOPIC,
			KAFKA_PROPERTIES,
			Optional.of(new FlinkFixedPartitioner<>()),
			encodingFormat);
	assertEquals(expectedSink, actualSink);

	// Test sink format.
	final KafkaDynamicSinkBase actualKafkaSink = (KafkaDynamicSinkBase) actualSink;
	assertEquals(encodingFormat, actualKafkaSink.encodingFormat);

	// Test kafka producer.
	DynamicTableSink.SinkRuntimeProvider provider =
			actualKafkaSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
	assertThat(provider, instanceOf(SinkFunctionProvider.class));
	final SinkFunctionProvider sinkFunctionProvider = (SinkFunctionProvider) provider;
	final SinkFunction<RowData> sinkFunction = sinkFunctionProvider.createSinkFunction();
	assertThat(sinkFunction, instanceOf(getExpectedProducerClass()));
}
 

public static void tcFlinkAvroTableAPI(String KafkaServerHostPort, String SchemaRegistryHostPort,
                                  String srcTopic, String targetTopic,
                                  String consumerGroupId, String sinkKeys, String transScript) {

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopic, new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result;
        Table ingest = tableEnv.scan(srcTopic);
        String className = "dynamic.FlinkScript";
        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";
        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl." + transScript + ";\n" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";
        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        result = runner.transTableObj(ingest);

        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
        // delivered properties for sink
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        env.execute("DF_FlinkTableAPI_Client_" + srcTopic + "-" + targetTopic);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets", "partition:0,offset:42;partition:1,offset:300");
	props2.put("connector.properties.zookeeper.stuff", "12");
	props2.put("connector.properties.kafka.stuff", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets", "partition:0,offset:42;partition:1,offset:300");
	props3.put("connector.properties.zookeeper.stuff", "12");
	props3.put("connector.properties.kafka.stuff", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	final Map<String, String> props4 = new HashMap<>();
	props4.put("connector.property-version", "1");
	props4.put("connector.type", "kafka");
	props4.put("connector.version", "0.11");
	props4.put("connector.topic", "MyTable");
	props4.put("connector.startup-mode", "timestamp");
	props4.put("connector.startup-timestamp-millis", "1577014729000");

	return Arrays.asList(props1, props2, props3, props4);
}
 

@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets.0.partition", "0");
	props2.put("connector.specific-offsets.0.offset", "42");
	props2.put("connector.specific-offsets.1.partition", "1");
	props2.put("connector.specific-offsets.1.offset", "300");
	props2.put("connector.properties.0.key", "zookeeper.stuff");
	props2.put("connector.properties.0.value", "12");
	props2.put("connector.properties.1.key", "kafka.stuff");
	props2.put("connector.properties.1.value", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets.0.partition", "0");
	props3.put("connector.specific-offsets.0.offset", "42");
	props3.put("connector.specific-offsets.1.partition", "1");
	props3.put("connector.specific-offsets.1.offset", "300");
	props3.put("connector.properties.0.key", "zookeeper.stuff");
	props3.put("connector.properties.0.value", "12");
	props3.put("connector.properties.1.key", "kafka.stuff");
	props3.put("connector.properties.1.value", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	return Arrays.asList(props1, props2, props3);
}
 

@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets.0.partition", "0");
	props2.put("connector.specific-offsets.0.offset", "42");
	props2.put("connector.specific-offsets.1.partition", "1");
	props2.put("connector.specific-offsets.1.offset", "300");
	props2.put("connector.properties.0.key", "zookeeper.stuff");
	props2.put("connector.properties.0.value", "12");
	props2.put("connector.properties.1.key", "kafka.stuff");
	props2.put("connector.properties.1.value", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets.0.partition", "0");
	props3.put("connector.specific-offsets.0.offset", "42");
	props3.put("connector.specific-offsets.1.partition", "1");
	props3.put("connector.specific-offsets.1.offset", "300");
	props3.put("connector.properties.0.key", "zookeeper.stuff");
	props3.put("connector.properties.0.value", "12");
	props3.put("connector.properties.1.key", "kafka.stuff");
	props3.put("connector.properties.1.value", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	return Arrays.asList(props1, props2, props3);
}
 

/**
 * This test can be unified with the corresponding source test once we have fixed FLINK-9870.
 */
@Test
public void testTableSink() {
	// prepare parameters for Kafka table sink

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.build();

	final KafkaTableSinkBase expected = getExpectedKafkaTableSink(
		schema,
		TOPIC,
		KAFKA_PROPERTIES,
		Optional.of(new FlinkFixedPartitioner<>()),
		new TestSerializationSchema(schema.toRowType()));

	// construct table sink using descriptors and table sink factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerFixed()
				.startFromSpecificOffsets(OFFSETS)) // test if they accepted although not needed
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING())
				.field(COUNT, Types.DECIMAL())
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()))
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
		.createStreamTableSink(propertiesMap);

	assertEquals(expected, actualSink);

	// test Kafka producer
	final KafkaTableSinkBase actualKafkaSink = (KafkaTableSinkBase) actualSink;
	final DataStreamMock streamMock = new DataStreamMock(new StreamExecutionEnvironmentMock(), schema.toRowType());
	actualKafkaSink.emitDataStream(streamMock);
	assertTrue(getExpectedFlinkKafkaProducer().isAssignableFrom(streamMock.sinkFunction.getClass()));
}
 

@Test
public void testTableSinkWithLegacyProperties() {
	// prepare parameters for Kafka table sink
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(10, 4))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final KafkaTableSinkBase expected = getExpectedKafkaTableSink(
		schema,
		TOPIC,
		KAFKA_PROPERTIES,
		Optional.of(new FlinkFixedPartitioner<>()),
		new TestSerializationSchema(schema.toRowType()));

	// construct table sink using descriptors and table sink factory
	final Map<String, String> legacyPropertiesMap = new HashMap<>();
	legacyPropertiesMap.putAll(createKafkaSinkProperties());

	// use legacy properties
	legacyPropertiesMap.remove("connector.specific-offsets");
	legacyPropertiesMap.remove("connector.properties.bootstrap.servers");
	legacyPropertiesMap.remove("connector.properties.group.id");

	// keep compatible with a specified update-mode
	legacyPropertiesMap.put("update-mode", "append");

	// legacy properties for specific-offsets and properties
	legacyPropertiesMap.put("connector.specific-offsets.0.partition", "0");
	legacyPropertiesMap.put("connector.specific-offsets.0.offset", "100");
	legacyPropertiesMap.put("connector.specific-offsets.1.partition", "1");
	legacyPropertiesMap.put("connector.specific-offsets.1.offset", "123");
	legacyPropertiesMap.put("connector.properties.0.key", "bootstrap.servers");
	legacyPropertiesMap.put("connector.properties.0.value", "dummy");
	legacyPropertiesMap.put("connector.properties.1.key", "group.id");
	legacyPropertiesMap.put("connector.properties.1.value", "dummy");

	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, legacyPropertiesMap)
		.createStreamTableSink(legacyPropertiesMap);

	assertEquals(expected, actualSink);

	// test Kafka producer
	final KafkaTableSinkBase actualKafkaSink = (KafkaTableSinkBase) actualSink;
	final DataStreamMock streamMock = new DataStreamMock(new StreamExecutionEnvironmentMock(), schema.toRowType());
	actualKafkaSink.consumeDataStream(streamMock);
	assertTrue(getExpectedFlinkKafkaProducer().isAssignableFrom(streamMock.sinkFunction.getClass()));
}
 

public static void testFlinkAvroScriptWithStaticSchema() {
    System.out.println("TestCase_Test Avro Table API Script with static Schema");

    final String STATIC_USER_SCHEMA = "{"
            + "\"type\":\"record\","
            + "\"name\":\"myrecord\","
            + "\"fields\":["
            + "  { \"name\":\"symbol\", \"type\":\"string\" },"
            + "  { \"name\":\"name\", \"type\":\"string\" },"
            + "  { \"name\":\"exchangecode\", \"type\":\"string\" }"
            + "]}";

    String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("group.id", "consumer_test");
    properties.setProperty("schema.subject", "test-value");
    properties.setProperty("schema.registry", "localhost:8081");
    properties.setProperty("static.avro.schema", STATIC_USER_SCHEMA);

    try {
        Kafka09AvroTableSource kafkaAvroTableSource =  new Kafka09AvroTableSource("test", properties);
        tableEnv.registerTableSource("Orders", kafkaAvroTableSource);

        Table ingest = tableEnv.scan("Orders");

        String className = "dynamic.FlinkScript";

        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";

        String transScript = "select(\"name\")";

        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl."+ transScript + ";" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";

        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        Table result = runner.transTableObj(ingest);

        Kafka09AvroTableSink sink =
                new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());
        // write the result Table to the TableSink
        result.writeToSink(sink);
        env.execute("Flink AVRO SQL KAFKA Test");
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	final Descriptor timestampDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromTimestamp(1577014729000L);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc, timestampDesc);
}
 

public static void tcFlinkAvroSQL(String SchemaRegistryHostPort, String srcTopic, String targetTopic, String sqlState) {
    System.out.println("tcFlinkAvroSQL");
    String resultFile = "testResult";

    String jarPath = "C:/Users/dadu/Coding/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
    //String jarPath = "/Users/will/Documents/Coding/GitHub/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), "localhost:9092");
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, "consumer_test");
    //properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, "test");
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, "symbol");

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result = tableEnv.sql(sqlState);
        result.printSchema();
        System.out.println("generated avro schema is = " + SchemaRegistryClient.tableAPIToAvroSchema(result, targetTopic));
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);

        // delivered properties
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        System.out.println(Paths.get(resultFile).toAbsolutePath());
        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        //result.writeToSink(new CsvTableSink(resultFile, "|", 1, FileSystem.WriteMode.OVERWRITE));
        env.execute("tcFlinkAvroSQL");
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * <p>To use a custom partitioner, please use
 * {@link #FlinkKafkaProducer011(String, SerializationSchema, Properties, Optional)} instead.
 *
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined key-less serialization schema.
 * @param producerConfig
 * 			Properties with the producer configuration.
 */
public FlinkKafkaProducer011(String topicId, SerializationSchema<IN> serializationSchema, Properties producerConfig) {
	this(
		topicId,
		new KeyedSerializationSchemaWrapper<>(serializationSchema),
		producerConfig,
		Optional.of(new FlinkFixedPartitioner<IN>()));
}
 

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * @param brokerList
 *			Comma separated addresses of the brokers
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined (keyless) serialization schema.
 */
public FlinkKafkaProducer011(String brokerList, String topicId, SerializationSchema<IN> serializationSchema) {
	this(
		topicId,
		new KeyedSerializationSchemaWrapper<>(serializationSchema),
		getPropertiesFromBrokerList(brokerList),
		Optional.of(new FlinkFixedPartitioner<IN>()));
}
 

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined serialization schema supporting key/value messages
 * @param producerConfig
 * 			Properties with the producer configuration.
 * @param semantic
 * 			Defines semantic that will be used by this producer (see {@link FlinkKafkaProducer.Semantic}).
 *
 * @deprecated use {@link #FlinkKafkaProducer(String, KafkaSerializationSchema, Properties, FlinkKafkaProducer.Semantic)}
 */
@Deprecated
public FlinkKafkaProducer(
	String topicId,
	KeyedSerializationSchema<IN> serializationSchema,
	Properties producerConfig,
	FlinkKafkaProducer.Semantic semantic) {
	this(topicId,
		serializationSchema,
		producerConfig,
		Optional.of(new FlinkFixedPartitioner<IN>()),
		semantic,
		DEFAULT_KAFKA_PRODUCERS_POOL_SIZE);
}
 

/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * <p>To use a custom partitioner, please use
 * {@link #FlinkKafkaProducer011(String, KeyedSerializationSchema, Properties, Optional)} instead.
 *
 * @param brokerList
 *			Comma separated addresses of the brokers
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined serialization schema supporting key/value messages
 */
public FlinkKafkaProducer011(String brokerList, String topicId, KeyedSerializationSchema<IN> serializationSchema) {
	this(
		topicId,
		serializationSchema,
		getPropertiesFromBrokerList(brokerList),
		Optional.of(new FlinkFixedPartitioner<IN>()));
}