org.apache.flink.table.catalog.exceptions.TableNotPartitionedException Java Examples

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// pass -1 as max_parts to fetch all partitions
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), (short) -1).stream()
			.map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");
	checkNotNull(partitionSpec, "CatalogPartitionSpec cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// partition spec can be partial
		List<String> partialVals = HiveReflectionUtils.getPvals(hiveShim, hiveTable.getPartitionKeys(),
			partitionSpec.getPartitionSpec());
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), partialVals,
			(short) -1).stream().map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// pass -1 as max_parts to fetch all partitions
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), (short) -1).stream()
			.map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");
	checkNotNull(partitionSpec, "CatalogPartitionSpec cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// partition spec can be partial
		List<String> partialVals = MetaStoreUtils.getPvals(hiveTable.getPartitionKeys(), partitionSpec.getPartitionSpec());
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), partialVals,
			(short) -1).stream().map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	ensurePartitionedTable(tablePath);

	CatalogTable catalogTable = (CatalogTable) getTable(tablePath);
	List<String> partKeys = catalogTable.getPartitionKeys();
	Map<String, String> spec = partitionSpec.getPartitionSpec();
	if (!partKeys.containsAll(spec.keySet())) {
		return new ArrayList<>();
	}

	return partitions.get(tablePath).keySet().stream()
		.filter(ps -> ps.getPartitionSpec().entrySet().containsAll(partitionSpec.getPartitionSpec().entrySet()))
		.collect(Collectors.toList());
}
 

@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
		throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partition);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);
	ensureFullPartitionSpec(tablePath, partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		if (!ignoreIfExists) {
			throw new PartitionAlreadyExistsException(getName(), tablePath, partitionSpec);
		}
	}

	partitions.get(tablePath).put(partitionSpec, partition.copy());
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	ensurePartitionedTable(tablePath);

	CatalogTable catalogTable = (CatalogTable) getTable(tablePath);
	List<String> partKeys = catalogTable.getPartitionKeys();
	Map<String, String> spec = partitionSpec.getPartitionSpec();
	if (!partKeys.containsAll(spec.keySet())) {
		return new ArrayList<>();
	}

	return partitions.get(tablePath).keySet().stream()
		.filter(ps -> ps.getPartitionSpec().entrySet().containsAll(partitionSpec.getPartitionSpec().entrySet()))
		.collect(Collectors.toList());
}
 

@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
		throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partition);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);
	ensureFullPartitionSpec(tablePath, partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		if (!ignoreIfExists) {
			throw new PartitionAlreadyExistsException(getName(), tablePath, partitionSpec);
		}
	}

	partitions.get(tablePath).put(partitionSpec, partition.copy());
}
 

@Test
public void testCreatePartition_TableNotPartitionedException() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	catalog.createTable(path1, createTable(), false);

	exception.expect(TableNotPartitionedException.class);
	exception.expectMessage(
		String.format("Table %s in catalog %s is not partitioned.", path1.getFullName(), TEST_CATALOG_NAME));
	catalog.createPartition(path1, createPartitionSpec(), createPartition(), false);
}
 

@Override
public List<CatalogPartitionSpec> listPartitionsByFilter(ObjectPath tablePath, List<Expression> expressions)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	Table hiveTable = getHiveTable(tablePath);
	ensurePartitionedTable(tablePath, hiveTable);
	List<String> partColNames = getFieldNames(hiveTable.getPartitionKeys());
	Optional<String> filter = HiveTableUtil.makePartitionFilter(
			getNonPartitionFields(hiveConf, hiveTable).size(), partColNames, expressions, hiveShim);
	if (!filter.isPresent()) {
		throw new UnsupportedOperationException(
				"HiveCatalog is unable to handle the partition filter expressions: " + expressions);
	}
	try {
		PartitionSpecProxy partitionSpec = client.listPartitionSpecsByFilter(
				tablePath.getDatabaseName(), tablePath.getObjectName(), filter.get(), (short) -1);
		List<CatalogPartitionSpec> res = new ArrayList<>(partitionSpec.size());
		PartitionSpecProxy.PartitionIterator partitions = partitionSpec.getPartitionIterator();
		while (partitions.hasNext()) {
			Partition partition = partitions.next();
			Map<String, String> spec = new HashMap<>();
			for (int i = 0; i < partColNames.size(); i++) {
				spec.put(partColNames.get(i), partition.getValues().get(i));
			}
			res.add(new CatalogPartitionSpec(spec));
		}
		return res;
	} catch (TException e) {
		throw new UnsupportedOperationException(
				"Failed to list partition by filter from HMS, filter expressions: " + expressions, e);
	}
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);

	return new ArrayList<>(partitions.get(tablePath).keySet());
}
 

@Test
public void testCreatePartition_TableNotPartitionedException() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	catalog.createTable(path1, createTable(), false);

	exception.expect(TableNotPartitionedException.class);
	exception.expectMessage(
		String.format("Table %s in catalog %s is not partitioned.", path1.getFullName(), TEST_CATALOG_NAME));
	catalog.createPartition(path1, createPartitionSpec(), createPartition(), false);
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);

	return new ArrayList<>(partitions.get(tablePath).keySet());
}
 

@Override
public List<CatalogPartitionSpec> listPartitionsByFilter(ObjectPath tablePath, List<Expression> filters) throws TableNotExistException, TableNotPartitionedException, CatalogException {
	return Collections.emptyList();
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException {
	fallback = true;
	return super.listPartitions(tablePath);
}
 

private void ensurePartitionedTable(ObjectPath tablePath, Table hiveTable) throws TableNotPartitionedException {
	if (!isTablePartitioned(hiveTable)) {
		throw new TableNotPartitionedException(getName(), tablePath);
	}
}
 

@Override
public List<CatalogPartitionSpec> listPartitionsByFilter(ObjectPath tablePath, List<Expression> filters)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	throw new UnsupportedOperationException();
}
 

private void ensurePartitionedTable(ObjectPath tablePath) throws TableNotPartitionedException {
	if (!isPartitionedTable(tablePath)) {
		throw new TableNotPartitionedException(getName(), tablePath);
	}
}
 

@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	throw new UnsupportedOperationException();
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException {
    throw new UnsupportedOperationException();
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, CatalogException {
	return Collections.emptyList();
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath) throws TableNotExistException, TableNotPartitionedException, CatalogException {
	return Collections.emptyList();
}
 

@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
    throw UNSUPPORTED_ERR;
}
 

private void ensurePartitionedTable(ObjectPath tablePath) throws TableNotPartitionedException {
	if (!isPartitionedTable(tablePath)) {
		throw new TableNotPartitionedException(getName(), tablePath);
	}
}
 

private void ensurePartitionedTable(ObjectPath tablePath, Table hiveTable) throws TableNotPartitionedException {
	if (!isTablePartitioned(hiveTable)) {
		throw new TableNotPartitionedException(getName(), tablePath);
	}
}
 

@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
    throw new UnsupportedOperationException();
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws TableNotExistException, TableNotPartitionedException, CatalogException {
    throw new UnsupportedOperationException();
}
 

/**
 * Get CatalogPartitionSpec of all partitions of the table.
 *
 * @param tablePath	path of the table
 * @return a list of CatalogPartitionSpec of the table
 *
 * @throws TableNotExistException thrown if the table does not exist in the catalog
 * @throws TableNotPartitionedException thrown if the table is not partitioned
 * @throws CatalogException	in case of any runtime exception
 */
List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
	throws TableNotExistException, TableNotPartitionedException, CatalogException;
 

/**
 * Get CatalogPartitionSpec of all partitions that is under the given CatalogPartitionSpec in the table.
 *
 * @param tablePath	path of the table
 * @param partitionSpec the partition spec to list
 * @return a list of CatalogPartitionSpec that is under the given CatalogPartitionSpec in the table
 *
 * @throws TableNotExistException thrown if the table does not exist in the catalog
 * @throws TableNotPartitionedException thrown if the table is not partitioned
 * @throws CatalogException in case of any runtime exception
 */
List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
	throws TableNotExistException, TableNotPartitionedException, CatalogException;
 

/**
 * Get CatalogPartitionSpec of partitions by expression filters in the table.
 *
 * <p>NOTE: For FieldReferenceExpression, the field index is based on schema of this table
 * instead of partition columns only.
 *
 * <p>The passed in predicates have been translated in conjunctive form.
 *
 * <p>If catalog does not support this interface at present, throw an {@link UnsupportedOperationException}
 * directly. If the catalog does not have a valid filter, throw the {@link UnsupportedOperationException}
 * directly. Planner will fallback to get all partitions and filter by itself.
 *
 * @param tablePath	path of the table
 * @param filters filters to push down filter to catalog
 * @return a list of CatalogPartitionSpec that is under the given CatalogPartitionSpec in the table
 *
 * @throws TableNotExistException thrown if the table does not exist in the catalog
 * @throws TableNotPartitionedException thrown if the table is not partitioned
 * @throws CatalogException in case of any runtime exception
 */
List<CatalogPartitionSpec> listPartitionsByFilter(ObjectPath tablePath, List<Expression> filters)
	throws TableNotExistException, TableNotPartitionedException, CatalogException;