org.apache.hadoop.hive.metastore.api.InvalidObjectException Java Examples

@Test
public void testDeletePartitionsThrowsInvalidInputException() throws Exception {
  Exception e = new com.amazonaws.services.glue.model.InvalidInputException("foo");
  mockBatchDeleteThrowsException(e);

  List<String> values1 = Lists.newArrayList("val1");
  List<String> values2 = Lists.newArrayList("val2");
  List<Partition> partitions = Lists.newArrayList(
        TestObjects.getTestPartition(NAMESPACE_NAME, TABLE_NAME, values1),
        TestObjects.getTestPartition(NAMESPACE_NAME, TABLE_NAME, values2));
  batchDeletePartitionsHelper = new BatchDeletePartitionsHelper(client, NAMESPACE_NAME, TABLE_NAME, null, partitions);

  batchDeletePartitionsHelper.deletePartitions();
  assertTrue(batchDeletePartitionsHelper.getPartitionsDeleted().isEmpty());
  assertThat(batchDeletePartitionsHelper.getFirstTException(), is(instanceOf(InvalidObjectException.class)));
}
 

@Test
public void testDropPartitionsClientException() throws Exception {
  Table table = HiveToCatalogConverter.convertTable(testTable);
  String namespaceName = testDB.getName();
  String tableName = table.getName();
  Partition partition = TestObjects.getTestPartition(namespaceName, tableName, Arrays.asList("foo", "bar"));

  mockGetPartitionsSuccess(Lists.newArrayList(partition));
  mockBatchDeleteThrowsException(new InvalidInputException("InvalidInputException"));
  when(glueClient.getTable(any(GetTableRequest.class)))
      .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));

  try {
    metastoreClient.dropPartitions(namespaceName, tableName,
        Lists.newArrayList(getDumbExpression()), true, false, false);
    fail("should throw");
  } catch (TException e) {
    verify(glueClient, times(1)).batchDeletePartition(any(BatchDeletePartitionRequest.class));
    verify(glueClient, never()).getPartition(any(GetPartitionRequest.class));
    verify(wh, never()).deleteDir(any(Path.class), anyBoolean(), anyBoolean());
    assertThat(e, is(instanceOf(InvalidObjectException.class)));
    assertThat(e.getMessage(), containsString("InvalidInputException"));
  }
  assertDaemonThreadPools();
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String source_db,
    String source_table_name,
    String dest_db,
    String dest_table_name)
    throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(source_db);
  mapping.checkWritePermissions(dest_db);
  Partition result = mapping
      .getClient()
      .exchange_partition(partitionSpecs, mapping.transformInboundDatabaseName(source_db), source_table_name,
          mapping.transformInboundDatabaseName(dest_db), dest_table_name);
  return mapping.transformOutboundPartition(result);
}
 

/**
 * Taken from HiveMetaStore#create_table_core
 * https://github.com/apache/hive/blob/rel/release-2.3.0/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java#L1370-L1383
 */
public static void validateTableObject(Table table, Configuration conf) throws InvalidObjectException {
  checkNotNull(table, "table cannot be null");
  checkNotNull(table.getSd(), "Table#StorageDescriptor cannot be null");

  if (!hiveShims.validateTableName(table.getTableName(), conf)) {
    throw new InvalidObjectException(table.getTableName() + " is not a valid object name");
  }
  String validate = MetaStoreUtils.validateTblColumns(table.getSd().getCols());
  if (validate != null) {
    throw new InvalidObjectException("Invalid column " + validate);
  }

  if (table.getPartitionKeys() != null) {
    validate = MetaStoreUtils.validateTblColumns(table.getPartitionKeys());
    if (validate != null) {
      throw new InvalidObjectException("Invalid partition column " + validate);
    }
  }
}
 

@Test
public void testAddPartitionsThrowsExceptionSecondPage() throws Exception {
  int numPartitions = 200;
  int secondPageSize = numPartitions - BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE;
  when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
    .thenReturn(new BatchCreatePartitionResult())
    .thenThrow(new InvalidInputException("exception"));
  when(glueClient.getTable(any(GetTableRequest.class)))
    .thenReturn(new GetTableResult().withTable(testTbl));

  List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);

  try {
    metastoreClientDelegate.addPartitions(partitions, false, true);
    fail("Should throw");
  } catch (Exception e) {
    assertThat(e, is(instanceOf(InvalidObjectException.class)));
    verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
    verify(glueClient, times(2)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
    verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
    verify(wh, times(secondPageSize)).deleteDir(any(Path.class), eq(true));
    assertDaemonThreadPools();
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<Partition> exchange_partitions(
    Map<String, String> partitionSpecs,
    String source_db,
    String source_table_name,
    String dest_db,
    String dest_table_name)
    throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(source_db);
  mapping.checkWritePermissions(dest_db);
  List<Partition> result = mapping
      .getClient()
      .exchange_partitions(partitionSpecs, mapping.transformInboundDatabaseName(source_db), source_table_name,
          mapping.transformInboundDatabaseName(dest_db), dest_table_name);
  return mapping.transformOutboundPartitions(result);
}
 

private void addPartitionsToSource(Table sourceTable, URI sourceTableLocation)
  throws IOException, InvalidObjectException, AlreadyExistsException, MetaException, TException {

  final File dataFileUk = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileUk, "1\tadam\tlondon\n2\tsusan\tglasgow\n");
  String partitionEurope = "/continent=Europe";
  String partitionUk = partitionEurope + "/country=UK";
  String partitionFileLocationUk = SOURCE_MANAGED_PARTITIONED_TABLE + partitionUk;
  String fileKeyUk = String.format("%s/%s/%s", DATABASE, partitionFileLocationUk, PART_00000);
  s3Client.putObject("source", fileKeyUk, dataFileUk);

  final File dataFileChina = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileChina, "1\tchun\tbeijing\n2\tshanghai\tmilan\n");
  String partitionAsia = "/continent=Asia";
  String partitionChina = partitionAsia + "/country=China";
  String partitionFileLocation = SOURCE_MANAGED_PARTITIONED_TABLE + partitionChina;
  String fileKeyChina = String.format("%s/%s/%s", DATABASE, partitionFileLocation, PART_00000);
  s3Client.putObject("source", fileKeyChina, dataFileChina);

  URI ukPartitionURI = URI.create(sourceTableLocation + partitionUk);
  URI chinaPartitionURI = URI.create(sourceTableLocation + partitionChina);
  sourceCatalog
      .client()
      .add_partitions(Arrays
          .asList(newTablePartition(sourceTable, Arrays.asList("Europe", "UK"), ukPartitionURI),
              newTablePartition(sourceTable, Arrays.asList("Asia", "China"), chinaPartitionURI)));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_environment_context(Table tbl, EnvironmentContext environment_context)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_environment_context(mapping.transformInboundTable(tbl), environment_context);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table(Table tbl)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table(mapping.transformInboundTable(tbl));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_database(Database database)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.primaryDatabaseMapping();
  mapping.createDatabase(mapping.transformInboundDatabase(database));
}
 

@Override
public void createDatabase(Database database)
  throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  if (accessControlHandler.hasCreatePermission()) {
    getClient().create_database(database);
    accessControlHandler.databaseCreatedNotification(database.getName());
  } else {
    throw new NotAllowedException("You cannot create the database '" + database.getName() + "'.");
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_function(Function func)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(func.getDbName());
  mapping.getClient().create_function(mapping.transformInboundFunction(func));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition_with_environment_context(Partition new_part, EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping
      .getClient()
      .add_partition_with_environment_context(mapping.transformInboundPartition(new_part), environment_context);
  return mapping.transformOutboundPartition(result);
}
 

@Override
public org.apache.hadoop.hive.metastore.api.Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String srcDb,
    String srcTbl,
    String dstDb,
    String dstTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartition(partitionSpecs, srcDb, srcTbl, dstDb, dstTbl);
}
 

@Override
public List<org.apache.hadoop.hive.metastore.api.Partition> exchange_partitions(
    Map<String, String> partitionSpecs,
    String sourceDb,
    String sourceTbl,
    String destDb,
    String destTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartitions(partitionSpecs, sourceDb, sourceTbl, destDb, destTbl);
}
 

@Override
public void createTableWithConstraints(
    org.apache.hadoop.hive.metastore.api.Table table,
    List<SQLPrimaryKey> primaryKeys,
    List<SQLForeignKey> foreignKeys
) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  glueMetastoreClientDelegate.createTableWithConstraints(table, primaryKeys, foreignKeys);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(request.getDbName());
  for (Partition partition : request.getParts()) {
    mapping.checkWritePermissions(partition.getDbName());
  }
  AddPartitionsResult result = mapping
      .getClient()
      .add_partitions_req(mapping.transformInboundAddPartitionsRequest(request));
  return mapping.transformOutboundAddPartitionsResult(result);
}
 

@Override
public boolean deleteTableColumnStatistics(
    String dbName, String tableName, String colName
) throws NoSuchObjectException, MetaException, InvalidObjectException,
    TException, org.apache.hadoop.hive.metastore.api.InvalidInputException {
  return glueMetastoreClientDelegate.deleteTableColumnStatistics(dbName, tableName, colName);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition(Partition new_part)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping.getClient().add_partition(mapping.transformInboundPartition(new_part));
  return mapping.transformOutboundPartition(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_constraints(Table tbl, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_constraints(mapping.transformInboundTable(tbl), primaryKeys, foreignKeys);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_with_environment_context(
    String db_name,
    String tbl_name,
    List<String> part_vals,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals,
          environment_context);
  return mapping.transformOutboundPartition(partition);
}
 

@Override
public boolean deletePartitionColumnStatistics(
    String dbName, String tableName, String partName, String colName
) throws NoSuchObjectException, MetaException, InvalidObjectException,
    TException, org.apache.hadoop.hive.metastore.api.InvalidInputException {
  return glueMetastoreClientDelegate.deletePartitionColumnStatistics(dbName, tableName, partName, colName);
}
 

@Override
public void createIndex(Index index, org.apache.hadoop.hive.metastore.api.Table indexTable) throws InvalidObjectException, MetaException, NoSuchObjectException,
      TException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException {
  boolean dirCreated = glueMetastoreClientDelegate.validateNewTableAndCreateDirectory(indexTable);
  boolean indexTableCreated = false;
  String dbName = index.getDbName();
  String indexTableName = index.getIndexTableName();
  String originTableName = index.getOrigTableName();
  Path indexTablePath = new Path(indexTable.getSd().getLocation());
  Table catalogIndexTableObject = HiveToCatalogConverter.convertIndexToTableObject(index);
  String indexTableObjectName = INDEX_PREFIX + index.getIndexName();

  try {
    org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, originTableName);
    Map<String, String> parameters = originTable.getParameters();
    if (parameters.containsKey(indexTableObjectName)){
      throw new org.apache.hadoop.hive.metastore.api.AlreadyExistsException("Index: " + index.getIndexName() + " already exist");
    }
    createTable(indexTable);
    indexTableCreated = true;
    originTable.getParameters().put(indexTableObjectName, catalogTableToString(catalogIndexTableObject));
    alter_table(dbName, originTableName, originTable);
  } catch (Exception e) {
    if (dirCreated){
      wh.deleteDir(indexTablePath, true);
    }
    if (indexTableCreated) {
      dropTable(dbName, indexTableName);
    }
    String msg = "Unable to create index: ";
    logger.error(msg, e);
    if (e instanceof TException) {
      throw e;
    } else {
      throw new MetaException(msg + e);
    }
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_table_column_statistics(String db_name, String tbl_name, String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_table_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions(List<Partition> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (Partition partition : new_parts) {
      mapping.checkWritePermissions(partition.getDbName());
    }
    return mapping.getClient().add_partitions(mapping.transformInboundPartitions(new_parts));
  }
  return 0;
}
 

@Override
public int add_partitions_pspec(
    PartitionSpecProxy pSpec
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException,
    MetaException, TException {
  return glueMetastoreClientDelegate.addPartitionsSpecProxy(pSpec);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Index add_index(Index new_index, Table index_table)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_index.getDbName());
  mapping.checkWritePermissions(index_table.getDbName());
  Index result = mapping
      .getClient()
      .add_index(mapping.transformInboundIndex(new_index), mapping.transformInboundTable(index_table));
  return mapping.transformOutboundIndex(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_by_name_with_environment_context(
    String db_name,
    String tbl_name,
    String part_name,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_by_name_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name,
          part_name, environment_context);
  return mapping.transformOutboundPartition(partition);
}
 

@Override
public org.apache.hadoop.hive.metastore.api.Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String srcDb,
    String srcTbl,
    String dstDb,
    String dstTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartition(partitionSpecs, srcDb, srcTbl, dstDb, dstTbl);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition(String db_name, String tbl_name, List<String> part_vals)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition result = mapping
      .getClient()
      .append_partition(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals);
  return mapping.transformOutboundPartition(result);
}