org.apache.hadoop.hive.metastore.api.AlreadyExistsException Java Examples

private void processResult(List<PartitionError> partitionErrors) {
  if (partitionErrors == null || partitionErrors.isEmpty()) {
    return;
  }

  logger.error(String.format("BatchCreatePartitions failed to create %d out of %d partitions. \n",
      partitionErrors.size(), partitionMap.size()));

  for (PartitionError partitionError : partitionErrors) {
    Partition partitionFailed = partitionMap.remove(new PartitionKey(partitionError.getPartitionValues()));

    TException exception = CatalogToHiveConverter.errorDetailToHiveException(partitionError.getErrorDetail());
    if (ifNotExists && exception instanceof AlreadyExistsException) {
      // AlreadyExistsException is allowed, so we shouldn't add the partition to partitionsFailed list
      continue;
    }
    logger.error(exception);
    if (firstTException == null) {
      firstTException = exception;
    }
    partitionsFailed.add(partitionFailed);
  }
}
 

/**
 * @return boolean
 *    true  -> directory created
 *    false -> directory not created
 */
public boolean validateNewTableAndCreateDirectory(org.apache.hadoop.hive.metastore.api.Table tbl) throws TException {
  checkNotNull(tbl, "tbl cannot be null");
  if (tableExists(tbl.getDbName(), tbl.getTableName())) {
    throw new AlreadyExistsException("Table " + tbl.getTableName() + " already exists.");
  }
  validateTableObject(tbl, conf);

  if (TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
    // we don't need to create directory for virtual views
    return false;
  }

  if (StringUtils.isEmpty(tbl.getSd().getLocation())) {
    org.apache.hadoop.hive.metastore.api.Database db = getDatabase(tbl.getDbName());
    tbl.getSd().setLocation(hiveShims.getDefaultTablePath(db, tbl.getTableName(), wh).toString());
  } else {
    tbl.getSd().setLocation(wh.getDnsPath(new Path(tbl.getSd().getLocation())).toString());
  }

  Path tblPath = new Path(tbl.getSd().getLocation());
  return makeDirs(wh, tblPath);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table(Table tbl)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table(mapping.transformInboundTable(tbl));
}
 

private void onPartitionExistWithoutComputingDiff(Table table, Partition nativePartition, TException e) throws TException {
  if(e == null) {
    return;
  }
  if (e instanceof AlreadyExistsException) {
    log.debug(String.format("Partition %s in table %s with location %s already exists and no need to update",
        stringifyPartition(nativePartition), table.getTableName(), nativePartition.getSd().getLocation()));
  }
  else {
    throw e;
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_environment_context(Table tbl, EnvironmentContext environment_context)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_environment_context(mapping.transformInboundTable(tbl), environment_context);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition(Partition new_part)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping.getClient().add_partition(mapping.transformInboundPartition(new_part));
  return mapping.transformOutboundPartition(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition_with_environment_context(Partition new_part, EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping
      .getClient()
      .add_partition_with_environment_context(mapping.transformInboundPartition(new_part), environment_context);
  return mapping.transformOutboundPartition(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions(List<Partition> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (Partition partition : new_parts) {
      mapping.checkWritePermissions(partition.getDbName());
    }
    return mapping.getClient().add_partitions(mapping.transformInboundPartitions(new_parts));
  }
  return 0;
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions_pspec(List<PartitionSpec> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (PartitionSpec partitionSpec : new_parts) {
      mapping.checkWritePermissions(partitionSpec.getDbName());
    }
    return mapping.getClient().add_partitions_pspec(mapping.transformInboundPartitionSpecs(new_parts));
  }
  return 0;
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition(String db_name, String tbl_name, List<String> part_vals)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition result = mapping
      .getClient()
      .append_partition(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals);
  return mapping.transformOutboundPartition(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(request.getDbName());
  for (Partition partition : request.getParts()) {
    mapping.checkWritePermissions(partition.getDbName());
  }
  AddPartitionsResult result = mapping
      .getClient()
      .add_partitions_req(mapping.transformInboundAddPartitionsRequest(request));
  return mapping.transformOutboundAddPartitionsResult(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_with_environment_context(
    String db_name,
    String tbl_name,
    List<String> part_vals,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals,
          environment_context);
  return mapping.transformOutboundPartition(partition);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_by_name(String db_name, String tbl_name, String part_name)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_by_name(mapping.transformInboundDatabaseName(db_name), tbl_name, part_name);
  return mapping.transformOutboundPartition(partition);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_by_name_with_environment_context(
    String db_name,
    String tbl_name,
    String part_name,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_by_name_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name,
          part_name, environment_context);
  return mapping.transformOutboundPartition(partition);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Index add_index(Index new_index, Table index_table)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_index.getDbName());
  mapping.checkWritePermissions(index_table.getDbName());
  Index result = mapping
      .getClient()
      .add_index(mapping.transformInboundIndex(new_index), mapping.transformInboundTable(index_table));
  return mapping.transformOutboundIndex(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_function(Function func)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(func.getDbName());
  mapping.getClient().create_function(mapping.transformInboundFunction(func));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_constraints(Table tbl, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_constraints(mapping.transformInboundTable(tbl), primaryKeys, foreignKeys);
}
 

/**
 * Adds and drops partitions in one transaction.
 *
 * @param databaseName database name
 * @param tableName    table name
 * @param addParts     list of partitions
 * @param dropParts    list of partition values
 * @param deleteData   if true, deletes the data
 * @return true if successful
 * @throws NoSuchObjectException Exception if table does not exists
 * @throws MetaException         Exception if
 * @throws TException            any internal exception
 */
@SuppressWarnings({"checkstyle:methodname"})
public boolean add_drop_partitions(final String databaseName,
                                   final String tableName, final List<Partition> addParts,
                                   final List<List<String>> dropParts, final boolean deleteData)
        throws NoSuchObjectException, MetaException, TException {
    startFunction("add_drop_partitions : db=" + databaseName + " tbl=" + tableName);
    if (addParts.size() == 0 && dropParts.size() == 0) {
        return true;
    }
    for (List<String> partVals : dropParts) {
        LOG.info("Drop Partition values:" + partVals);
    }
    for (Partition part : addParts) {
        LOG.info("Add Partition values:" + part);
    }

    boolean ret = false;
    Exception ex = null;
    try {
        ret = addDropPartitionsCore(getMS(), databaseName, tableName, addParts, dropParts, false, null);
    } catch (Exception e) {
        ex = e;
        if (e instanceof MetaException) {
            throw (MetaException) e;
        } else if (e instanceof InvalidObjectException) {
            throw (InvalidObjectException) e;
        } else if (e instanceof AlreadyExistsException) {
            throw (AlreadyExistsException) e;
        } else if (e instanceof NoSuchObjectException) {
            throw (NoSuchObjectException) e;
        } else {
            throw newMetaException(e);
        }
    } finally {
        endFunction("drop_partitions", ret, ex, tableName);
    }
    return ret;

}
 

private boolean startAddPartition(
        final RawStore ms, final Partition part, final boolean ifNotExists) throws MetaException, TException {
    MetaStoreUtils.validatePartitionNameCharacters(part.getValues(),
            partitionValidationPattern);
    final boolean doesExist = ms.doesPartitionExist(
            part.getDbName(), part.getTableName(), part.getValues());
    if (doesExist && !ifNotExists) {
        throw new AlreadyExistsException("Partition already exists: " + part);
    }
    return !doesExist;
}
 

private List<Partition> addPartitionsCoreNoTxn(
        final RawStore ms, final Table tbl, final List<Partition> parts, final boolean ifNotExists,
        final Map<PartValEqWrapper, Boolean> addedPartitions, final List<Partition> existingParts)
        throws MetaException, InvalidObjectException, AlreadyExistsException, TException {
    logInfo("add_partitions");
    final String dbName = tbl.getDbName();
    final String tblName = tbl.getTableName();
    final List<Partition> result = new ArrayList<Partition>();
    for (Partition part : parts) {
        if (!part.getTableName().equals(tblName) || !part.getDbName().equals(dbName)) {
            throw new MetaException("Partition does not belong to target table "
                    + dbName + "." + tblName + ": " + part);
        }
        final boolean shouldAdd = startAddPartition(ms, part, ifNotExists);
        if (!shouldAdd) {
            existingParts.add(part);
            LOG.info("Not adding partition " + part + " as it already exists");
            continue;
        }
        final boolean madeDir = createLocationForAddedPartition(tbl, part);
        if (addedPartitions.put(new PartValEqWrapper(part), madeDir) != null) {
            // Technically, for ifNotExists case, we could insert one and discard the other
            // because the first one now "exists", but it seems better to report the problem
            // upstream as such a command doesn't make sense.
            throw new MetaException("Duplicate partitions in the list: " + part);
        }
        initializeAddedPartition(tbl, part, madeDir);
        result.add(part);
    }
    return result;
}
 

private void add_partition(HiveMetaStoreClient client, Table table,
	      List<String> vals, String location) throws InvalidObjectException,
	        AlreadyExistsException, MetaException, TException {

    Partition part = new Partition();
    part.setDbName(table.getDbName());
    part.setTableName(table.getTableName());
    part.setValues(vals);
    part.setParameters(new HashMap<String, String>());
    part.setSd(table.getSd());
    part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
    part.getSd().setLocation(table.getSd().getLocation() + location);

    client.add_partition(part);
}
 

public void createTestDb(String dbName) throws Exception {
  Database db = new Database(dbName, "Some description", "/tmp/" + dbName, new HashMap<String, String>());
  try {
    this.localMetastoreClient.createDatabase(db);
  } catch (AlreadyExistsException e) {
    log.warn(dbName + " already exits");
  }
}
 

@Test
public void addPartition_shouldIgnoreAlreadyExistsException() throws Exception {
  doThrow(AlreadyExistsException.class).when(metaStoreClient).add_partition(any());
  doReturn(URI.create("resolved/location")).when(locationResolver).resolveLocation(LOCATION, false);
  doReturn(Instant.ofEpochSecond(1526462225L)).when(clock).instant();

  Optional<Partition> result = underTest.addPartition(TABLE, PARTITION_VALUES, LOCATION);

  assertThat(result.isPresent(), is(false));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_database(Database database)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.primaryDatabaseMapping();
  mapping.createDatabase(mapping.transformInboundDatabase(database));
}
 

@Override
public void createDatabase(Database database)
  throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  if (accessControlHandler.hasCreatePermission()) {
    getClient().create_database(database);
    accessControlHandler.databaseCreatedNotification(database.getName());
  } else {
    throw new NotAllowedException("You cannot create the database '" + database.getName() + "'.");
  }
}
 

public void createTableWithConstraints(
    org.apache.hadoop.hive.metastore.api.Table table,
    List<SQLPrimaryKey> primaryKeys,
    List<SQLForeignKey> foreignKeys
) throws AlreadyExistsException, TException {
  throw new UnsupportedOperationException("createTableWithConstraints is not supported");
}
 

private void addPartitionsToSource(Table sourceTable, URI sourceTableLocation)
  throws IOException, InvalidObjectException, AlreadyExistsException, MetaException, TException {

  final File dataFileUk = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileUk, "1\tadam\tlondon\n2\tsusan\tglasgow\n");
  String partitionEurope = "/continent=Europe";
  String partitionUk = partitionEurope + "/country=UK";
  String partitionFileLocationUk = SOURCE_MANAGED_PARTITIONED_TABLE + partitionUk;
  String fileKeyUk = String.format("%s/%s/%s", DATABASE, partitionFileLocationUk, PART_00000);
  s3Client.putObject("source", fileKeyUk, dataFileUk);

  final File dataFileChina = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileChina, "1\tchun\tbeijing\n2\tshanghai\tmilan\n");
  String partitionAsia = "/continent=Asia";
  String partitionChina = partitionAsia + "/country=China";
  String partitionFileLocation = SOURCE_MANAGED_PARTITIONED_TABLE + partitionChina;
  String fileKeyChina = String.format("%s/%s/%s", DATABASE, partitionFileLocation, PART_00000);
  s3Client.putObject("source", fileKeyChina, dataFileChina);

  URI ukPartitionURI = URI.create(sourceTableLocation + partitionUk);
  URI chinaPartitionURI = URI.create(sourceTableLocation + partitionChina);
  sourceCatalog
      .client()
      .add_partitions(Arrays
          .asList(newTablePartition(sourceTable, Arrays.asList("Europe", "UK"), ukPartitionURI),
              newTablePartition(sourceTable, Arrays.asList("Asia", "China"), chinaPartitionURI)));
}
 

@Test (expected = AlreadyExistsException.class)
public void createDuplicateIndex() throws TException {
  assertFalse(indexExist(hiveDB.getName(), hiveTable.getTableName(), hiveIndex.getIndexName()));
  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  assertTrue(indexExist(hiveDB.getName(), hiveTable.getTableName(), hiveIndex.getIndexName()));

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  hiveIndex.setIndexTableName(hiveIndexTable2.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable2);
}
 

public TException get(String msg) {
  return new AlreadyExistsException(msg);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean create_type(Type type)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  return getPrimaryClient().create_type(type);
}