Java Code Examples for org.apache.flink.table.descriptors.DescriptorProperties#putTableSchema()

private static Map<String, String> hbaseTableProperties() {
	Map<String, String> properties = new HashMap<>();
	properties.put(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_HBASE);
	properties.put(CONNECTOR_VERSION, CONNECTOR_VERSION_VALUE_143);
	properties.put(CONNECTOR_PROPERTY_VERSION, "1");
	properties.put(CONNECTOR_TABLE_NAME, TEST_TABLE_1);
	// get zk quorum from "hbase-site.xml" in classpath
	String hbaseZk = HBaseConfiguration.create().get(HConstants.ZOOKEEPER_QUORUM);
	properties.put(CONNECTOR_ZK_QUORUM, hbaseZk);
	// schema
	String[] columnNames = {FAMILY1, ROWKEY, FAMILY2, FAMILY3};
	TypeInformation<Row> f1 = Types.ROW_NAMED(new String[]{F1COL1}, Types.INT);
	TypeInformation<Row> f2 = Types.ROW_NAMED(new String[]{F2COL1, F2COL2}, Types.STRING, Types.LONG);
	TypeInformation<Row> f3 = Types.ROW_NAMED(new String[]{F3COL1, F3COL2, F3COL3}, Types.DOUBLE, Types.BOOLEAN, Types.STRING);
	TypeInformation[] columnTypes = new TypeInformation[]{f1, Types.INT, f2, f3};

	DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	TableSchema tableSchema = new TableSchema(columnNames, columnTypes);
	descriptorProperties.putTableSchema(SCHEMA, tableSchema);
	descriptorProperties.putProperties(properties);
	return descriptorProperties.asMap();
}
 

private DescriptorProperties createDescriptor(String[] columnNames, TypeInformation[] columnTypes) {
	TableSchema tableSchema = new TableSchema(columnNames, columnTypes);

	Map<String, String> tableProperties = new HashMap<>();
	tableProperties.put("connector.type", "hbase");
	tableProperties.put("connector.version", "1.4.3");
	tableProperties.put("connector.property-version", "1");
	tableProperties.put("connector.table-name", "testHBastTable");
	tableProperties.put("connector.zookeeper.quorum", "localhost:2181");
	tableProperties.put("connector.zookeeper.znode.parent", "/flink");
	tableProperties.put("connector.write.buffer-flush.max-size", "10mb");
	tableProperties.put("connector.write.buffer-flush.max-rows", "1000");
	tableProperties.put("connector.write.buffer-flush.interval", "10s");

	DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putTableSchema(SCHEMA, tableSchema);
	descriptorProperties.putProperties(tableProperties);
	return descriptorProperties;
}
 

private DescriptorProperties createDescriptor(TableSchema tableSchema) {
	Map<String, String> tableProperties = new HashMap<>();
	tableProperties.put("connector.type", "hbase");
	tableProperties.put("connector.version", "1.4.3");
	tableProperties.put("connector.property-version", "1");
	tableProperties.put("connector.table-name", "testHBastTable");
	tableProperties.put("connector.zookeeper.quorum", "localhost:2181");
	tableProperties.put("connector.zookeeper.znode.parent", "/flink");
	tableProperties.put("connector.write.buffer-flush.max-size", "10mb");
	tableProperties.put("connector.write.buffer-flush.max-rows", "1000");
	tableProperties.put("connector.write.buffer-flush.interval", "10s");

	DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putTableSchema(SCHEMA, tableSchema);
	descriptorProperties.putProperties(tableProperties);
	return descriptorProperties;
}
 

private DescriptorProperties createDescriptor(TableSchema schema) {
	Map<String, String> properties = new HashMap<>();
	properties.put("connector.type", "filesystem");
	properties.put("connector.property-version", "1");
	properties.put("connector.path", "/path/to/csv");

	// schema
	properties.put("format.type", "csv");
	properties.put("format.property-version", "1");
	properties.put("format.field-delimiter", ";");

	DescriptorProperties descriptor = new DescriptorProperties(true);
	descriptor.putProperties(properties);
	descriptor.putTableSchema(SCHEMA, schema);
	if (deriveSchema == TernaryBoolean.TRUE) {
		descriptor.putBoolean("format.derive-schema", true);
	} else if (deriveSchema == TernaryBoolean.FALSE) {
		descriptor.putTableSchema(FORMAT_FIELDS, testingSchema);
	} // nothing to put for UNDEFINED
	return descriptor;
}
 

ExternalCatalogTable toExternalCatalogTable() {
  TableSchema tableSchema = new TableSchema(schema.getFieldNames(), schema.getFieldTypes());
  ConnectorDescriptor descriptor = new ConnectorDescriptor(CONNECTOR_TYPE, CONNECTOR_VERSION, false) {
    @Override
    public void addConnectorProperties(DescriptorProperties properties) {
      properties.putTableSchema(TABLE_SCHEMA_CONNECTOR_PROPERTY, tableSchema);
      properties.putString(TABLE_DATA_CONNECTOR_PROPERTY, serializeRows());
    }
  };
  return new ExternalCatalogTable(descriptor, Option.empty(), Option.empty(), Option.empty(), Option.empty());
}
 

private static ExternalCatalogTable mockExternalCatalogTable(String topic, String brokerAddress) {
  TableSchema schema = new TableSchema(new String[] {"foo"}, new TypeInformation[] {INT_TYPE_INFO});
  ConnectorDescriptor descriptor = new ConnectorDescriptor("kafka+json", 1, false) {
    @Override
    public void addConnectorProperties(DescriptorProperties properties) {
      properties.putTableSchema(TOPIC_SCHEMA_KEY, schema);
      properties.putString(TOPIC_NAME_KEY, topic);
      properties.putString(KAFKA_CONFIG_PREFIX + "." + ConsumerConfig.GROUP_ID_CONFIG, "foo");
      properties.putString(KAFKA_CONFIG_PREFIX + "." + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerAddress);
      properties.putString(KAFKA_CONFIG_PREFIX + "." + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    }
  };

  return new ExternalCatalogTable(descriptor, Option.empty(), Option.empty(), Option.empty(), Option.empty());
}
 

static KafkaInputExternalCatalogTable getKafkaExternalCatalogTable(Map<String, String> props) {
  ConnectorDescriptor descriptor = new ConnectorDescriptor("kafka+json", 1, false) {
    @Override
    public void addConnectorProperties(DescriptorProperties properties) {
      properties.putTableSchema("athenax.kafka.topic.schema", KafkaInputExternalCatalogTable.SCHEMA);
      properties.putProperties(props);
    }
  };
  return new KafkaInputExternalCatalogTable(descriptor);
}
 

private Map<String, String> getBasicProperties() {
	Map<String, String> properties = new HashMap<>();

	properties.put("connector.type", "jdbc");
	properties.put("connector.property-version", "1");

	properties.put("connector.url", "jdbc:derby:memory:mydb");
	properties.put("connector.table", "mytable");

	DescriptorProperties descriptorProperties = new DescriptorProperties();
	descriptorProperties.putProperties(properties);
	descriptorProperties.putTableSchema("schema", schema);

	return new HashMap<>(descriptorProperties.asMap());
}
 

@Override
public Map<String, String> toProperties() {
	DescriptorProperties descriptor = new DescriptorProperties();

	descriptor.putTableSchema(Schema.SCHEMA, getSchema());
	descriptor.putPartitionKeys(getPartitionKeys());

	Map<String, String> properties = new HashMap<>(getProperties());
	properties.remove(CatalogConfig.IS_GENERIC);

	descriptor.putProperties(properties);

	return descriptor.asMap();
}
 

/**
 * Construct catalog table properties from {@link #toProperties()}.
 */
public static Map<String, String> removeRedundant(
		Map<String, String> properties,
		TableSchema schema,
		List<String> partitionKeys) {
	Map<String, String> ret = new HashMap<>(properties);
	DescriptorProperties descriptorProperties = new DescriptorProperties();
	descriptorProperties.putTableSchema(Schema.SCHEMA, schema);
	descriptorProperties.putPartitionKeys(partitionKeys);
	descriptorProperties.asMap().keySet().forEach(ret::remove);
	return ret;
}
 

@Test
public void testTableSink() throws Exception {
	HBaseTableSchema schema = new HBaseTableSchema();
	schema.addColumn(FAMILY1, F1COL1, Integer.class);
	schema.addColumn(FAMILY2, F2COL1, String.class);
	schema.addColumn(FAMILY2, F2COL2, Long.class);
	schema.setRowKey("rk", Integer.class);
	schema.addColumn(FAMILY3, F3COL1, Double.class);
	schema.addColumn(FAMILY3, F3COL2, Boolean.class);
	schema.addColumn(FAMILY3, F3COL3, String.class);

	Map<String, String> tableProperties = new HashMap<>();
	tableProperties.put("connector.type", "hbase");
	tableProperties.put("connector.version", "1.4.3");
	tableProperties.put("connector.property-version", "1");
	tableProperties.put("connector.table-name", TEST_TABLE_2);
	tableProperties.put("connector.zookeeper.quorum", getZookeeperQuorum());
	tableProperties.put("connector.zookeeper.znode.parent", "/hbase");
	DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putTableSchema(SCHEMA, schema.convertsToTableSchema());
	descriptorProperties.putProperties(tableProperties);
	TableSink tableSink = TableFactoryService
		.find(HBaseTableFactory.class, descriptorProperties.asMap())
		.createTableSink(descriptorProperties.asMap());

	StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment();
	StreamTableEnvironment tEnv = StreamTableEnvironment.create(execEnv, streamSettings);

	DataStream<Row> ds = execEnv.fromCollection(testData1).returns(testTypeInfo1);
	tEnv.registerDataStream("src", ds);
	tEnv.registerTableSink("hbase", tableSink);

	String query = "INSERT INTO hbase SELECT ROW(f1c1), ROW(f2c1, f2c2), rowkey, ROW(f3c1, f3c2, f3c3) FROM src";
	tEnv.sqlUpdate(query);

	// wait to finish
	tEnv.execute("HBase Job");

	// start a batch scan job to verify contents in HBase table
	// start a batch scan job to verify contents in HBase table
	TableEnvironment batchTableEnv = createBatchTableEnv();

	HBaseTableSource hbaseTable = new HBaseTableSource(getConf(), TEST_TABLE_2);
	hbaseTable.setRowKey("rowkey", Integer.class);
	hbaseTable.addColumn(FAMILY1, F1COL1, Integer.class);
	hbaseTable.addColumn(FAMILY2, F2COL1, String.class);
	hbaseTable.addColumn(FAMILY2, F2COL2, Long.class);
	hbaseTable.addColumn(FAMILY3, F3COL1, Double.class);
	hbaseTable.addColumn(FAMILY3, F3COL2, Boolean.class);
	hbaseTable.addColumn(FAMILY3, F3COL3, String.class);
	batchTableEnv.registerTableSource("hTable", hbaseTable);

	Table table = batchTableEnv.sqlQuery(
		"SELECT " +
			"  h.rowkey, " +
			"  h.family1.col1, " +
			"  h.family2.col1, " +
			"  h.family2.col2, " +
			"  h.family3.col1, " +
			"  h.family3.col2, " +
			"  h.family3.col3 " +
			"FROM hTable AS h"
	);

	List<Row> results = collectBatchResult(table);
	String expected =
		"1,10,Hello-1,100,1.01,false,Welt-1\n" +
			"2,20,Hello-2,200,2.02,true,Welt-2\n" +
			"3,30,Hello-3,300,3.03,false,Welt-3\n" +
			"4,40,,400,4.04,true,Welt-4\n" +
			"5,50,Hello-5,500,5.05,false,Welt-5\n" +
			"6,60,Hello-6,600,6.06,true,Welt-6\n" +
			"7,70,Hello-7,700,7.07,false,Welt-7\n" +
			"8,80,,800,8.08,true,Welt-8\n";

	TestBaseUtils.compareResultAsText(results, expected);
}
 

public static Table instantiateHiveTable(ObjectPath tablePath, CatalogBaseTable table, HiveConf hiveConf) {
	if (!(table instanceof CatalogTableImpl) && !(table instanceof CatalogViewImpl)) {
		throw new CatalogException(
				"HiveCatalog only supports CatalogTableImpl and CatalogViewImpl");
	}
	// let Hive set default parameters for us, e.g. serialization.format
	Table hiveTable = org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable(tablePath.getDatabaseName(),
			tablePath.getObjectName());
	hiveTable.setCreateTime((int) (System.currentTimeMillis() / 1000));

	Map<String, String> properties = new HashMap<>(table.getProperties());
	// Table comment
	if (table.getComment() != null) {
		properties.put(HiveCatalogConfig.COMMENT, table.getComment());
	}

	boolean isGeneric = HiveCatalog.isGenericForCreate(properties);

	// Hive table's StorageDescriptor
	StorageDescriptor sd = hiveTable.getSd();
	HiveTableUtil.setDefaultStorageFormat(sd, hiveConf);

	if (isGeneric) {
		DescriptorProperties tableSchemaProps = new DescriptorProperties(true);
		tableSchemaProps.putTableSchema(Schema.SCHEMA, table.getSchema());

		if (table instanceof CatalogTable) {
			tableSchemaProps.putPartitionKeys(((CatalogTable) table).getPartitionKeys());
		}

		properties.putAll(tableSchemaProps.asMap());
		properties = maskFlinkProperties(properties);
		hiveTable.setParameters(properties);
	} else {
		HiveTableUtil.initiateTableFromProperties(hiveTable, properties, hiveConf);
		List<FieldSchema> allColumns = HiveTableUtil.createHiveColumns(table.getSchema());
		// Table columns and partition keys
		if (table instanceof CatalogTableImpl) {
			CatalogTable catalogTable = (CatalogTableImpl) table;

			if (catalogTable.isPartitioned()) {
				int partitionKeySize = catalogTable.getPartitionKeys().size();
				List<FieldSchema> regularColumns = allColumns.subList(0, allColumns.size() - partitionKeySize);
				List<FieldSchema> partitionColumns = allColumns.subList(allColumns.size() - partitionKeySize, allColumns.size());

				sd.setCols(regularColumns);
				hiveTable.setPartitionKeys(partitionColumns);
			} else {
				sd.setCols(allColumns);
				hiveTable.setPartitionKeys(new ArrayList<>());
			}
		} else {
			sd.setCols(allColumns);
		}
		// Table properties
		hiveTable.getParameters().putAll(properties);
	}

	if (table instanceof CatalogViewImpl) {
		// TODO: [FLINK-12398] Support partitioned view in catalog API
		hiveTable.setPartitionKeys(new ArrayList<>());

		CatalogView view = (CatalogView) table;
		hiveTable.setViewOriginalText(view.getOriginalQuery());
		hiveTable.setViewExpandedText(view.getExpandedQuery());
		hiveTable.setTableType(TableType.VIRTUAL_VIEW.name());
	}

	return hiveTable;
}
 

@Override
public Map<String, String> toProperties() {
	DescriptorProperties descriptor = new DescriptorProperties();

	descriptor.putTableSchema(Schema.SCHEMA, getSchema());

	Map<String, String> properties = new HashMap<>(getProperties());
	properties.remove(CatalogConfig.IS_GENERIC);

	descriptor.putProperties(properties);

	return descriptor.asMap();
}