org.apache.flink.table.catalog.config.CatalogConfig Java Examples

public static void checkEquals(CatalogTable t1, CatalogTable t2) {
	assertEquals(t1.getClass(), t2.getClass());
	assertEquals(t1.getSchema(), t2.getSchema());
	assertEquals(t1.getComment(), t2.getComment());
	assertEquals(t1.getPartitionKeys(), t2.getPartitionKeys());
	assertEquals(t1.isPartitioned(), t2.isPartitioned());

	assertEquals(
		t1.getProperties().get(CatalogConfig.IS_GENERIC),
		t2.getProperties().get(CatalogConfig.IS_GENERIC));

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(t1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(t1.getProperties(), t2.getProperties());
	} else {
		assertTrue(t2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(t2.getProperties().entrySet().containsAll(t1.getProperties().entrySet()));
	}
}
 

@Test
public void testCreateHiveTable() {
	Map<String, String> map = new HashMap<>(new FileSystem().path("/test_path").toProperties());

	map.put(CatalogConfig.IS_GENERIC, String.valueOf(false));

	Table hiveTable = HiveTableUtil.instantiateHiveTable(
		new ObjectPath("test", "test"),
		new CatalogTableImpl(
			schema,
			map,
			null
		),
		HiveTestUtils.createHiveConf());

	Map<String, String> prop = hiveTable.getParameters();
	assertEquals(prop.remove(CatalogConfig.IS_GENERIC), String.valueOf(false));
	assertTrue(prop.keySet().stream().noneMatch(k -> k.startsWith(CatalogConfig.FLINK_PROPERTY_PREFIX)));
}
 

@Override
public TableSink createTableSink(TableSinkFactory.Context context) {
	CatalogTable table = checkNotNull(context.getTable());
	Preconditions.checkArgument(table instanceof CatalogTableImpl);

	boolean isGeneric = Boolean.parseBoolean(table.getProperties().get(CatalogConfig.IS_GENERIC));

	if (!isGeneric) {
		return new HiveTableSink(
				context.getConfiguration().get(
						HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER),
				context.isBounded(),
				new JobConf(hiveConf),
				context.getObjectIdentifier(),
				table);
	} else {
		return TableFactoryUtil.findAndCreateTableSink(context);
	}
}
 

@Override
public TableSource<RowData> createTableSource(TableSourceFactory.Context context) {
	CatalogTable table = checkNotNull(context.getTable());
	Preconditions.checkArgument(table instanceof CatalogTableImpl);

	boolean isGeneric = Boolean.parseBoolean(table.getProperties().get(CatalogConfig.IS_GENERIC));

	if (!isGeneric) {
		return new HiveTableSource(
				new JobConf(hiveConf),
				context.getConfiguration(),
				context.getObjectIdentifier().toObjectPath(),
				table);
	} else {
		return TableFactoryUtil.findAndCreateTableSource(context);
	}
}
 

private void checkStatistics(int inputStat, int expectStat) throws Exception {
	catalog.dropTable(path1, true);

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, "false");
	properties.put(StatsSetupConst.ROW_COUNT, String.valueOf(inputStat));
	properties.put(StatsSetupConst.NUM_FILES, String.valueOf(inputStat));
	properties.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(inputStat));
	properties.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(inputStat));
	CatalogTable catalogTable = new CatalogTableImpl(
			TableSchema.builder().field("f0", DataTypes.INT()).build(),
			properties,
			"");
	catalog.createTable(path1, catalogTable, false);

	CatalogTableStatistics statistics = catalog.getTableStatistics(path1);
	assertEquals(expectStat, statistics.getRowCount());
	assertEquals(expectStat, statistics.getFileCount());
	assertEquals(expectStat, statistics.getRawDataSize());
	assertEquals(expectStat, statistics.getTotalSize());
}
 

private CatalogTable createCatalogTable(DataType[] types) {
	String[] colNames = new String[types.length];

	for (int i = 0; i < types.length; i++) {
		colNames[i] = String.format("%s_%d", types[i].toString().toLowerCase(), i);
	}

	TableSchema schema = TableSchema.builder()
		.fields(colNames, types)
		.build();

	return new CatalogTableImpl(
		schema,
		new HashMap<String, String>() {{
			put("is_streaming", "false");
			put(CatalogConfig.IS_GENERIC, String.valueOf(false));
		}},
		""
	);
}
 

public static boolean isGenericForCreate(Map<String, String> properties) {
	// When creating an object, a hive object needs explicitly have a key is_generic = false
	// otherwise, this is a generic object if 1) the key is missing 2) is_generic = true
	// this is opposite to reading an object. See getObjectIsGeneric().
	if (properties == null) {
		return true;
	}
	boolean isGeneric;
	if (!properties.containsKey(CatalogConfig.IS_GENERIC)) {
		// must be a generic object
		isGeneric = true;
		properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
	} else {
		isGeneric = Boolean.parseBoolean(properties.get(CatalogConfig.IS_GENERIC));
	}
	return isGeneric;
}
 

private CatalogTable createHiveCatalogTable(TableSchema tableSchema, int numPartCols) {
	if (numPartCols == 0) {
		return new CatalogTableImpl(
			tableSchema,
			new HashMap<String, String>() {{
				// creating a hive table needs explicit is_generic=false flag
				put(CatalogConfig.IS_GENERIC, String.valueOf(false));
			}},
			"");
	}
	String[] partCols = new String[numPartCols];
	System.arraycopy(tableSchema.getFieldNames(), tableSchema.getFieldNames().length - numPartCols, partCols, 0, numPartCols);
	return new CatalogTableImpl(
		tableSchema,
		Arrays.asList(partCols),
		new HashMap<String, String>() {{
			// creating a hive table needs explicit is_generic=false flag
			put(CatalogConfig.IS_GENERIC, String.valueOf(false));
		}},
		"");
}
 

public SqlCreateHiveView(SqlParserPos pos, SqlIdentifier viewName, SqlNodeList fieldList, SqlNode query,
		boolean ifNotExists, SqlCharStringLiteral comment, SqlNodeList properties) {
	super(
			pos,
			viewName,
			fieldList,
			query,
			false,
			false,
			ifNotExists,
			HiveDDLUtils.unescapeStringLiteral(comment),
			properties
	);
	HiveDDLUtils.unescapeProperties(properties);
	originPropList = new SqlNodeList(properties.getList(), properties.getParserPosition());
	// mark it as a hive view
	properties.add(HiveDDLUtils.toTableOption(CatalogConfig.IS_GENERIC, "false", pos));
}
 

public SqlCreateHiveDatabase(SqlParserPos pos, SqlIdentifier databaseName, SqlNodeList propertyList,
		SqlCharStringLiteral comment, SqlCharStringLiteral location, boolean ifNotExists) throws ParseException {
	super(
			pos,
			databaseName,
			HiveDDLUtils.checkReservedDBProperties(propertyList),
			HiveDDLUtils.unescapeStringLiteral(comment),
			ifNotExists
	);
	HiveDDLUtils.ensureNonGeneric(propertyList);
	originPropList = new SqlNodeList(propertyList.getList(), propertyList.getParserPosition());
	// mark it as a hive database
	propertyList.add(HiveDDLUtils.toTableOption(CatalogConfig.IS_GENERIC, "false", pos));
	if (location != null) {
		propertyList.add(new SqlTableOption(
				SqlLiteral.createCharString(DATABASE_LOCATION_URI, location.getParserPosition()),
				location,
				location.getParserPosition()));
	}
	this.location = location;
}
 

public static void checkEquals(CatalogView v1, CatalogView v2) {
	assertEquals(v1.getClass(), v2.getClass());
	assertEquals(v1.getSchema(), v1.getSchema());
	assertEquals(v1.getComment(), v2.getComment());
	assertEquals(v1.getOriginalQuery(), v2.getOriginalQuery());
	assertEquals(v1.getExpandedQuery(), v2.getExpandedQuery());

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(v1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(v1.getProperties(), v2.getProperties());
	} else {
		assertTrue(v2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(v2.getProperties().entrySet().containsAll(v1.getProperties().entrySet()));
	}
}
 

public static void checkEquals(CatalogTable t1, CatalogTable t2) {
	assertEquals(t1.getClass(), t2.getClass());
	assertEquals(t1.getSchema(), t2.getSchema());
	assertEquals(t1.getComment(), t2.getComment());
	assertEquals(t1.getPartitionKeys(), t2.getPartitionKeys());
	assertEquals(t1.isPartitioned(), t2.isPartitioned());

	assertEquals(
		t1.getProperties().get(CatalogConfig.IS_GENERIC),
		t2.getProperties().get(CatalogConfig.IS_GENERIC));

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(t1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(t1.getProperties(), t2.getProperties());
	} else {
		assertTrue(t2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(t2.getProperties().entrySet().containsAll(t1.getProperties().entrySet()));
	}
}
 

@Test
public void testGenericTable() throws Exception {
	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
	properties.put("connector", "COLLECTION");

	catalog.createDatabase("mydb", new CatalogDatabaseImpl(new HashMap<>(), ""), true);
	ObjectPath path = new ObjectPath("mydb", "mytable");
	CatalogTable table = new CatalogTableImpl(schema, properties, "csv table");
	catalog.createTable(path, table, true);
	Optional<TableFactory> opt = catalog.getTableFactory();
	assertTrue(opt.isPresent());
	HiveTableFactory tableFactory = (HiveTableFactory) opt.get();
	TableSource tableSource = tableFactory.createTableSource(path, table);
	assertTrue(tableSource instanceof StreamTableSource);
	TableSink tableSink = tableFactory.createTableSink(path, table);
	assertTrue(tableSink instanceof StreamTableSink);
}
 

private CatalogTable createCatalogTable(DataType[] types) {
	String[] colNames = new String[types.length];

	for (int i = 0; i < types.length; i++) {
		colNames[i] = String.format("%s_%d", types[i].toString().toLowerCase(), i);
	}

	TableSchema schema = TableSchema.builder()
		.fields(colNames, types)
		.build();

	return new CatalogTableImpl(
		schema,
		new HashMap<String, String>() {{
			put("is_streaming", "false");
			put(CatalogConfig.IS_GENERIC, String.valueOf(false));
		}},
		""
	);
}
 

public static void checkEquals(CatalogView v1, CatalogView v2) {
	assertEquals(v1.getClass(), v2.getClass());
	assertEquals(v1.getSchema(), v1.getSchema());
	assertEquals(v1.getComment(), v2.getComment());
	assertEquals(v1.getOriginalQuery(), v2.getOriginalQuery());
	assertEquals(v1.getExpandedQuery(), v2.getExpandedQuery());

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(v1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(v1.getProperties(), v2.getProperties());
	} else {
		assertTrue(v2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(v2.getProperties().entrySet().containsAll(v1.getProperties().entrySet()));
	}
}
 

private static Function instantiateHiveFunction(ObjectPath functionPath, CatalogFunction function) {

		boolean isGeneric = Boolean.valueOf(function.getProperties().get(CatalogConfig.IS_GENERIC));

		// Hive Function does not have properties map
		// thus, use a prefix in class name to distinguish Flink and Hive functions
		String functionClassName = isGeneric ?
			FLINK_FUNCTION_PREFIX + function.getClassName() :
			function.getClassName();

		return new Function(
			// due to https://issues.apache.org/jira/browse/HIVE-22053, we have to normalize function name ourselves
			HiveStringUtils.normalizeIdentifier(functionPath.getObjectName()),
			functionPath.getDatabaseName(),
			functionClassName,
			null,			// Owner name
			PrincipalType.GROUP,	// Temporarily set to GROUP type because it's required by Hive. May change later
			(int) (System.currentTimeMillis() / 1000),
			FunctionType.JAVA,		// FunctionType only has JAVA now
			new ArrayList<>()		// Resource URIs
		);
	}
 

/**
 * Add a prefix to Flink-created properties to distinguish them from Hive-created properties.
 * Note that 'is_generic' is a special key and this method will leave it as-is.
 */
public static Map<String, String> maskFlinkProperties(Map<String, String> properties) {
	return properties.entrySet().stream()
			.filter(e -> e.getKey() != null && e.getValue() != null)
			.map(e -> new Tuple2<>(
					e.getKey().equals(CatalogConfig.IS_GENERIC) ? e.getKey() : FLINK_PROPERTY_PREFIX + e.getKey(),
					e.getValue()))
			.collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 

public static SqlNodeList ensureNonGeneric(SqlNodeList props) throws ParseException {
	for (SqlNode node : props) {
		if (node instanceof SqlTableOption && ((SqlTableOption) node).getKeyString().equalsIgnoreCase(CatalogConfig.IS_GENERIC)) {
			if (!((SqlTableOption) node).getValueString().equalsIgnoreCase("false")) {
				throw new ParseException("Creating generic object with Hive dialect is not allowed");
			}
		}
	}
	return props;
}
 

public static void checkEquals(CatalogPartition p1, CatalogPartition p2) {
	assertEquals(p1.getClass(), p2.getClass());
	assertEquals(p1.getComment(), p2.getComment());

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(p1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(p1.getProperties(), p2.getProperties());
	} else {
		assertTrue(p2.getProperties().entrySet().containsAll(p1.getProperties().entrySet()));
	}
}
 

@Test
public void testHiveTable() throws Exception {
	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, String.valueOf(false));

	catalog.createDatabase("mydb", new CatalogDatabaseImpl(new HashMap<>(), ""), true);
	ObjectPath path = new ObjectPath("mydb", "mytable");
	CatalogTable table = new CatalogTableImpl(schema, properties, "hive table");
	catalog.createTable(path, table, true);
	Optional<TableFactory> opt = catalog.getTableFactory();
	assertTrue(opt.isPresent());
	HiveTableFactory tableFactory = (HiveTableFactory) opt.get();
	TableSink tableSink = tableFactory.createTableSink(new TableSinkFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"),
			table,
			new Configuration(),
			true));
	assertTrue(tableSink instanceof HiveTableSink);
	TableSource tableSource = tableFactory.createTableSource(new TableSourceFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"), table, new Configuration()));
	assertTrue(tableSource instanceof HiveTableSource);
}
 

@Test
public void testGenericTable() throws Exception {
	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
	properties.put("connector", "COLLECTION");

	catalog.createDatabase("mydb", new CatalogDatabaseImpl(new HashMap<>(), ""), true);
	ObjectPath path = new ObjectPath("mydb", "mytable");
	CatalogTable table = new CatalogTableImpl(schema, properties, "csv table");
	catalog.createTable(path, table, true);
	Optional<TableFactory> opt = catalog.getTableFactory();
	assertTrue(opt.isPresent());
	HiveTableFactory tableFactory = (HiveTableFactory) opt.get();
	TableSource tableSource = tableFactory.createTableSource(new TableSourceFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"), table, new Configuration()));
	assertTrue(tableSource instanceof StreamTableSource);
	TableSink tableSink = tableFactory.createTableSink(new TableSinkFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"),
			table,
			new Configuration(),
			true));
	assertTrue(tableSink instanceof StreamTableSink);
}
 

@Test
public void testCreateDatabase() throws Exception {
	tableEnv.executeSql("create database db1 comment 'db1 comment'");
	Database db = hiveCatalog.getHiveDatabase("db1");
	assertEquals("db1 comment", db.getDescription());
	assertFalse(Boolean.parseBoolean(db.getParameters().get(CatalogConfig.IS_GENERIC)));

	String db2Location = warehouse + "/db2_location";
	tableEnv.executeSql(String.format("create database db2 location '%s' with dbproperties('k1'='v1')", db2Location));
	db = hiveCatalog.getHiveDatabase("db2");
	assertEquals(db2Location, locationPath(db.getLocationUri()));
	assertEquals("v1", db.getParameters().get("k1"));
}
 

@Override
public Map<String, String> toProperties() {
	DescriptorProperties descriptor = new DescriptorProperties();

	descriptor.putTableSchema(Schema.SCHEMA, getSchema());
	descriptor.putPartitionKeys(getPartitionKeys());

	Map<String, String> properties = new HashMap<>(getProperties());
	properties.remove(CatalogConfig.IS_GENERIC);

	descriptor.putProperties(properties);

	return descriptor.asMap();
}
 

@Override
public CatalogTable createStreamingTable() {
	Map<String, String> prop = getBatchTableProperties();
	prop.put(CatalogConfig.IS_GENERIC, String.valueOf(false));

	return new CatalogTableImpl(
		createTableSchema(),
		getStreamingTableProperties(),
		TEST_COMMENT);
}
 

@Override
public List<String> supportedProperties() {
	List<String> list = super.supportedProperties();
	list.add(CatalogConfig.IS_GENERIC);

	return list;
}
 

@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	// Developers may already have their own production/testing hive-site.xml set in their environment,
	// and Flink tests should avoid using those hive-site.xml.
	// Thus, explicitly create a testing HiveConf for unit tests here
	Catalog hiveCatalog = HiveTestUtils.createHiveCatalog(name, properties.get(HiveCatalogValidator.CATALOG_HIVE_VERSION));

	// Creates an additional database to test tableEnv.useDatabase() will switch current database of the catalog
	hiveCatalog.open();
	try {
		hiveCatalog.createDatabase(
			ADDITIONAL_TEST_DATABASE,
			new CatalogDatabaseImpl(new HashMap<>(), null),
			false);
		hiveCatalog.createTable(
			new ObjectPath(ADDITIONAL_TEST_DATABASE, TEST_TABLE),
			new CatalogTableImpl(
				TableSchema.builder()
					.field("testcol", DataTypes.INT())
					.build(),
				new HashMap<String, String>() {{
					put(CatalogConfig.IS_GENERIC, String.valueOf(false));
				}},
				""
			),
			false
		);
		// create a table to test parameterized types
		hiveCatalog.createTable(new ObjectPath("default", TABLE_WITH_PARAMETERIZED_TYPES),
				tableWithParameterizedTypes(),
				false);
	} catch (DatabaseAlreadyExistException | TableAlreadyExistException | DatabaseNotExistException e) {
		throw new CatalogException(e);
	}

	return hiveCatalog;
}
 

private CatalogTable tableWithParameterizedTypes() {
	TableSchema tableSchema = TableSchema.builder().fields(new String[]{"dec", "ch", "vch"},
			new DataType[]{DataTypes.DECIMAL(10, 10), DataTypes.CHAR(5), DataTypes.VARCHAR(15)}).build();
	return new CatalogTableImpl(
		tableSchema,
		new HashMap<String, String>() {{
			put(CatalogConfig.IS_GENERIC, String.valueOf(false));
		}},
		"");
}
 

@Test
public void testCreateGenericTable() {
	Table hiveTable = HiveTableUtil.instantiateHiveTable(
		new ObjectPath("test", "test"),
		new CatalogTableImpl(
			schema,
			new FileSystem().path("/test_path").toProperties(),
			null
		),
		HiveTestUtils.createHiveConf());

	Map<String, String> prop = hiveTable.getParameters();
	assertEquals(prop.remove(CatalogConfig.IS_GENERIC), String.valueOf("true"));
	assertTrue(prop.keySet().stream().allMatch(k -> k.startsWith(CatalogConfig.FLINK_PROPERTY_PREFIX)));
}
 

/**
 * Add a prefix to Flink-created properties to distinguish them from Hive-created properties.
 * Note that 'is_generic' is a special key and this method will leave it as-is.
 */
private static Map<String, String> maskFlinkProperties(Map<String, String> properties) {
	return properties.entrySet().stream()
		.filter(e -> e.getKey() != null && e.getValue() != null)
		.map(e -> new Tuple2<>(
			e.getKey().equals(CatalogConfig.IS_GENERIC) ? e.getKey() : FLINK_PROPERTY_PREFIX + e.getKey(),
			e.getValue()))
		.collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 

@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	// Test HiveCatalogFactory.createCatalog
	// But not use it for testing purpose
	assertTrue(super.createCatalog(name, properties) != null);

	// Developers may already have their own production/testing hive-site.xml set in their environment,
	// and Flink tests should avoid using those hive-site.xml.
	// Thus, explicitly create a testing HiveConf for unit tests here
	Catalog hiveCatalog = HiveTestUtils.createHiveCatalog(name, properties.get(HiveCatalogValidator.CATALOG_HIVE_VERSION));

	// Creates an additional database to test tableEnv.useDatabase() will switch current database of the catalog
	hiveCatalog.open();
	try {
		hiveCatalog.createDatabase(
			ADDITIONAL_TEST_DATABASE,
			new CatalogDatabaseImpl(new HashMap<>(), null),
			false);
		hiveCatalog.createTable(
			new ObjectPath(ADDITIONAL_TEST_DATABASE, TEST_TABLE),
			new CatalogTableImpl(
				TableSchema.builder()
					.field("testcol", DataTypes.INT())
					.build(),
				new HashMap<String, String>() {{
					put(CatalogConfig.IS_GENERIC, String.valueOf(true));
				}},
				""
			),
			false
		);
	} catch (DatabaseAlreadyExistException | TableAlreadyExistException | DatabaseNotExistException e) {
		throw new CatalogException(e);
	}

	return hiveCatalog;
}