org.apache.flink.table.catalog.exceptions.PartitionSpecInvalidException Java Examples

The following examples show how to use org.apache.flink.table.catalog.exceptions.PartitionSpecInvalidException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HiveCatalog.java    From flink with Apache License 2.0 6 votes vote down vote up 
private Partition instantiateHivePartition(Table hiveTable, CatalogPartitionSpec partitionSpec, CatalogPartition catalogPartition)
		throws PartitionSpecInvalidException {
	List<String> partCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partValues = getOrderedFullPartitionValues(
		partitionSpec, partCols, new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()));
	// validate partition values
	for (int i = 0; i < partCols.size(); i++) {
		if (StringUtils.isNullOrWhitespaceOnly(partValues.get(i))) {
			throw new PartitionSpecInvalidException(getName(), partCols,
				new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()), partitionSpec);
		}
	}
	// TODO: handle GenericCatalogPartition
	StorageDescriptor sd = hiveTable.getSd().deepCopy();
	sd.setLocation(catalogPartition.getProperties().remove(HiveCatalogConfig.PARTITION_LOCATION));

	Map<String, String> properties = new HashMap<>(catalogPartition.getProperties());
	properties.put(HiveCatalogConfig.COMMENT, catalogPartition.getComment());

	return HiveTableUtil.createHivePartition(
			hiveTable.getDbName(),
			hiveTable.getTableName(),
			partValues,
			sd,
			properties);
}
Example #2
Source File: HiveCatalog.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Get a list of ordered partition values by re-arranging them based on the given list of partition keys.
 *
 * @param partitionSpec a partition spec.
 * @param partitionKeys a list of partition keys.
 * @param tablePath path of the table to which the partition belongs.
 * @return A list of partition values ordered according to partitionKeys.
 * @throws PartitionSpecInvalidException thrown if partitionSpec and partitionKeys have different sizes,
 *                                       or any key in partitionKeys doesn't exist in partitionSpec.
 */
private List<String> getOrderedFullPartitionValues(CatalogPartitionSpec partitionSpec, List<String> partitionKeys, ObjectPath tablePath)
		throws PartitionSpecInvalidException {
	Map<String, String> spec = partitionSpec.getPartitionSpec();
	if (spec.size() != partitionKeys.size()) {
		throw new PartitionSpecInvalidException(getName(), partitionKeys, tablePath, partitionSpec);
	}

	List<String> values = new ArrayList<>(spec.size());
	for (String key : partitionKeys) {
		if (!spec.containsKey(key)) {
			throw new PartitionSpecInvalidException(getName(), partitionKeys, tablePath, partitionSpec);
		} else {
			values.add(spec.get(key));
		}
	}

	return values;
}
Example #3
Source File: GenericInMemoryCatalog.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
		throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partition);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);
	ensureFullPartitionSpec(tablePath, partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		if (!ignoreIfExists) {
			throw new PartitionAlreadyExistsException(getName(), tablePath, partitionSpec);
		}
	}

	partitions.get(tablePath).put(partitionSpec, partition.copy());
}
Example #4
Source File: HiveCatalog.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Get a list of ordered partition values by re-arranging them based on the given list of partition keys.
 *
 * @param partitionSpec a partition spec.
 * @param partitionKeys a list of partition keys.
 * @param tablePath path of the table to which the partition belongs.
 * @return A list of partition values ordered according to partitionKeys.
 * @throws PartitionSpecInvalidException thrown if partitionSpec and partitionKeys have different sizes,
 *                                       or any key in partitionKeys doesn't exist in partitionSpec.
 */
private List<String> getOrderedFullPartitionValues(CatalogPartitionSpec partitionSpec, List<String> partitionKeys, ObjectPath tablePath)
		throws PartitionSpecInvalidException {
	Map<String, String> spec = partitionSpec.getPartitionSpec();
	if (spec.size() != partitionKeys.size()) {
		throw new PartitionSpecInvalidException(getName(), partitionKeys, tablePath, partitionSpec);
	}

	List<String> values = new ArrayList<>(spec.size());
	for (String key : partitionKeys) {
		if (!spec.containsKey(key)) {
			throw new PartitionSpecInvalidException(getName(), partitionKeys, tablePath, partitionSpec);
		} else {
			values.add(spec.get(key));
		}
	}

	return values;
}
Example #5
Source File: GenericInMemoryCatalog.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
		throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partition);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);
	ensureFullPartitionSpec(tablePath, partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		if (!ignoreIfExists) {
			throw new PartitionAlreadyExistsException(getName(), tablePath, partitionSpec);
		}
	}

	partitions.get(tablePath).put(partitionSpec, partition.copy());
}
Example #6
Source File: GenericInMemoryCatalog.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void ensureFullPartitionSpec(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
	throws TableNotExistException, PartitionSpecInvalidException {
	if (!isFullPartitionSpec(tablePath, partitionSpec)) {
		throw new PartitionSpecInvalidException(getName(), ((CatalogTable) getTable(tablePath)).getPartitionKeys(),
			tablePath, partitionSpec);
	}
}
Example #7
Source File: CatalogTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCreatePartition_PartitionSpecInvalidException() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable table = createPartitionedTable();
	catalog.createTable(path1, table, false);

	CatalogPartitionSpec partitionSpec = createInvalidPartitionSpecSubset();
	exception.expect(PartitionSpecInvalidException.class);
	exception.expectMessage(
		String.format("PartitionSpec %s does not match partition keys %s of table %s in catalog %s.",
			partitionSpec, table.getPartitionKeys(), path1.getFullName(), TEST_CATALOG_NAME));
	catalog.createPartition(path1, partitionSpec, createPartition(), false);
}
Example #8
Source File: HiveCatalog.java    From flink with Apache License 2.0 5 votes vote down vote up 
private String getPartitionName(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, Table hiveTable) throws PartitionSpecInvalidException {
	List<String> partitionCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partitionVals = getOrderedFullPartitionValues(partitionSpec, partitionCols, tablePath);
	List<String> partKVs = new ArrayList<>();
	for (int i = 0; i < partitionCols.size(); i++) {
		partKVs.add(partitionCols.get(i) + "=" + partitionVals.get(i));
	}
	return String.join("/", partKVs);
}
Example #9
Source File: HiveCatalog.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public Partition getHivePartition(Table hiveTable, CatalogPartitionSpec partitionSpec)
		throws PartitionSpecInvalidException, TException {
	return client.getPartition(hiveTable.getDbName(), hiveTable.getTableName(),
		getOrderedFullPartitionValues(partitionSpec, getFieldNames(hiveTable.getPartitionKeys()),
			new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName())));
}
Example #10
Source File: HiveCatalog.java    From flink with Apache License 2.0 5 votes vote down vote up 
private Partition instantiateHivePartition(Table hiveTable, CatalogPartitionSpec partitionSpec, CatalogPartition catalogPartition)
		throws PartitionSpecInvalidException {
	List<String> partCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partValues = getOrderedFullPartitionValues(
		partitionSpec, partCols, new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()));
	// validate partition values
	for (int i = 0; i < partCols.size(); i++) {
		if (StringUtils.isNullOrWhitespaceOnly(partValues.get(i))) {
			throw new PartitionSpecInvalidException(getName(), partCols,
				new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName()), partitionSpec);
		}
	}
	// TODO: handle GenericCatalogPartition
	StorageDescriptor sd = hiveTable.getSd().deepCopy();
	sd.setLocation(catalogPartition.getProperties().remove(SqlCreateHiveTable.TABLE_LOCATION_URI));

	Map<String, String> properties = new HashMap<>(catalogPartition.getProperties());
	String comment = catalogPartition.getComment();
	if (comment != null) {
		properties.put(HiveCatalogConfig.COMMENT, comment);
	}

	return HiveTableUtil.createHivePartition(
			hiveTable.getDbName(),
			hiveTable.getTableName(),
			partValues,
			sd,
			properties);
}
Example #11
Source File: GenericInMemoryCatalog.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void ensureFullPartitionSpec(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
	throws TableNotExistException, PartitionSpecInvalidException {
	if (!isFullPartitionSpec(tablePath, partitionSpec)) {
		throw new PartitionSpecInvalidException(getName(), ((CatalogTable) getTable(tablePath)).getPartitionKeys(),
			tablePath, partitionSpec);
	}
}
Example #12
Source File: CatalogTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCreatePartition_PartitionSpecInvalidException() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable table = createPartitionedTable();
	catalog.createTable(path1, table, false);

	CatalogPartitionSpec partitionSpec = createInvalidPartitionSpecSubset();
	exception.expect(PartitionSpecInvalidException.class);
	exception.expectMessage(
		String.format("PartitionSpec %s does not match partition keys %s of table %s in catalog %s.",
			partitionSpec, table.getPartitionKeys(), path1.getFullName(), TEST_CATALOG_NAME));
	catalog.createPartition(path1, partitionSpec, createPartition(), false);
}
Example #13
Source File: HiveCatalog.java    From flink with Apache License 2.0 5 votes vote down vote up 
private String getPartitionName(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, Table hiveTable) throws PartitionSpecInvalidException {
	List<String> partitionCols = getFieldNames(hiveTable.getPartitionKeys());
	List<String> partitionVals = getOrderedFullPartitionValues(partitionSpec, partitionCols, tablePath);
	List<String> partKVs = new ArrayList<>();
	for (int i = 0; i < partitionCols.size(); i++) {
		partKVs.add(partitionCols.get(i) + "=" + partitionVals.get(i));
	}
	return String.join("/", partKVs);
}
Example #14
Source File: AbstractReadOnlyCatalog.java    From bahir-flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
    throw UNSUPPORTED_ERR;
}
Example #15
Source File: AbstractJdbcCatalog.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	throw new UnsupportedOperationException();
}
Example #16
Source File: HiveCatalog.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Partition getHivePartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, PartitionSpecInvalidException, TException {
	return getHivePartition(getHiveTable(tablePath), partitionSpec);
}
Example #17
Source File: HiveCatalog.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Partition getHivePartition(Table hiveTable, CatalogPartitionSpec partitionSpec)
		throws PartitionSpecInvalidException, TException {
	return client.getPartition(hiveTable.getDbName(), hiveTable.getTableName(),
		getOrderedFullPartitionValues(partitionSpec, getFieldNames(hiveTable.getPartitionKeys()),
			new ObjectPath(hiveTable.getDbName(), hiveTable.getTableName())));
}
Example #18
Source File: HiveCatalog.java    From flink with Apache License 2.0 4 votes vote down vote up 
private Partition getHivePartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, PartitionSpecInvalidException, TException {
	return getHivePartition(getHiveTable(tablePath), partitionSpec);
}
Example #19
Source File: PulsarCatalog.java    From pulsar-flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
    throw new UnsupportedOperationException();
}
Example #20
Source File: Catalog.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Create a partition.
 *
 * @param tablePath path of the table.
 * @param partitionSpec partition spec of the partition
 * @param partition the partition to add.
 * @param ignoreIfExists flag to specify behavior if a table with the given name already exists:
 *                       if set to false, it throws a TableAlreadyExistException,
 *                       if set to true, nothing happens.
 *
 * @throws TableNotExistException thrown if the target table does not exist
 * @throws TableNotPartitionedException thrown if the target table is not partitioned
 * @throws PartitionSpecInvalidException thrown if the given partition spec is invalid
 * @throws PartitionAlreadyExistsException thrown if the target partition already exists
 * @throws CatalogException in case of any runtime exception
 */
void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
	throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException;
Example #21
Source File: Catalog.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Create a partition.
 *
 * @param tablePath path of the table.
 * @param partitionSpec partition spec of the partition
 * @param partition the partition to add.
 * @param ignoreIfExists flag to specify behavior if a table with the given name already exists:
 *                       if set to false, it throws a TableAlreadyExistException,
 *                       if set to true, nothing happens.
 *
 * @throws TableNotExistException thrown if the target table does not exist
 * @throws TableNotPartitionedException thrown if the target table is not partitioned
 * @throws PartitionSpecInvalidException thrown if the given partition spec is invalid
 * @throws PartitionAlreadyExistsException thrown if the target partition already exists
 * @throws CatalogException in case of any runtime exception
 */
void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
	throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException;