org.apache.hadoop.hive.metastore.api.Database Java Examples

@Override
public void alterDatabase(String databaseName, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
		throws DatabaseNotExistException, CatalogException {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName), "databaseName cannot be null or empty");
	checkNotNull(newDatabase, "newDatabase cannot be null");

	// client.alterDatabase doesn't throw any exception if there is no existing database
	if (!databaseExists(databaseName)) {
		if (!ignoreIfNotExists) {
			throw new DatabaseNotExistException(getName(), databaseName);
		}

		return;
	}

	Database newHiveDatabase = instantiateHiveDatabase(databaseName, newDatabase);

	try {
		client.alterDatabase(databaseName, newHiveDatabase);
	} catch (TException e) {
		throw new CatalogException(String.format("Failed to alter database %s", databaseName), e);
	}
}
 

@Override
public synchronized void alterDatabase(HiveIdentity identity, String databaseName, Database newDatabase)
{
    String newDatabaseName = newDatabase.getName();

    if (databaseName.equals(newDatabaseName)) {
        if (databases.replace(databaseName, newDatabase) == null) {
            throw new SchemaNotFoundException(databaseName);
        }
        return;
    }

    Database database = databases.get(databaseName);
    if (database == null) {
        throw new SchemaNotFoundException(databaseName);
    }
    if (databases.putIfAbsent(newDatabaseName, database) != null) {
        throw new SchemaAlreadyExistsException(newDatabaseName);
    }
    databases.remove(databaseName);

    rewriteKeys(relations, name -> new SchemaTableName(newDatabaseName, name.getTableName()));
    rewriteKeys(views, name -> new SchemaTableName(newDatabaseName, name.getTableName()));
    rewriteKeys(partitions, name -> name.withSchemaName(newDatabaseName));
    rewriteKeys(tablePrivileges, name -> name.withDatabase(newDatabaseName));
}
 

@Test
public void testSetNamespaceProperties() throws TException {
  Namespace namespace = Namespace.of("dbname_set");

  catalog.createNamespace(namespace, meta);
  catalog.setProperties(namespace,
      ImmutableMap.of(
          "owner", "alter_apache",
          "test", "test",
          "location", "file:/data/tmp",
          "comment", "iceberg test")
  );

  Database database = metastoreClient.getDatabase(namespace.level(0));
  Assert.assertEquals(database.getParameters().get("owner"), "alter_apache");
  Assert.assertEquals(database.getParameters().get("test"), "test");
  Assert.assertEquals(database.getParameters().get("group"), "iceberg");
  AssertHelpers.assertThrows("Should fail to namespace not exist" + namespace,
      NoSuchNamespaceException.class, "Namespace does not exist: ", () -> {
        catalog.setProperties(Namespace.of("db2", "db2", "ns2"), meta);
      });
}
 

@Test
public void testCreate() throws TException {
  HiveConf hiveConf = new HiveConf();
  HiveMetaStoreClientFactory factory = new HiveMetaStoreClientFactory(hiveConf);

  // Since we havE a specified hive-site in the classpath, so have to null it out here to proceed the test
  // The original value it will get if no local hive-site is placed, will be an empty string. 
  hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "");
  hiveConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, "");
  IMetaStoreClient msc = factory.create();

  String dbName = "test_db";
  String description = "test database";
  String location = "file:/tmp/" + dbName;
  Database db = new Database(dbName, description, location, null);

  msc.dropDatabase(dbName, true, true);
  msc.createDatabase(db);
  db = msc.getDatabase(dbName);
  Assert.assertEquals(db.getName(), dbName);
  Assert.assertEquals(db.getDescription(), description);
  Assert.assertEquals(db.getLocationUri(), location);
}
 

@BeforeClass
public static void startMetastoreAndSpark() throws Exception {
  TestIcebergSourceHiveTables.metastore = new TestHiveMetastore();
  metastore.start();
  TestIcebergSourceHiveTables.hiveConf = metastore.hiveConf();
  String dbPath = metastore.getDatabasePath("db");
  Database db = new Database("db", "desc", dbPath, new HashMap<>());
  TestIcebergSourceHiveTables.clients = new HiveClientPool(1, hiveConf);
  clients.run(client -> {
    client.createDatabase(db);
    return null;
  });

  TestIcebergSourceHiveTables.spark = SparkSession.builder()
      .master("local[2]")
      .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname))
      .getOrCreate();

  TestIcebergSourceHiveTables.catalog = new HiveCatalog(hiveConf);
}
 

@Override
public void doTask() throws Exception {
  Database db = hmsHandler.get_database(dbName);
  List<String> dbPath = PathsUpdate.parsePath(db.getLocationUri());
  if (dbPath != null) {
    synchronized (update) {
      Preconditions.checkArgument(dbName.equalsIgnoreCase(db.getName()));
      update.newPathChange(dbName).addToAddPaths(dbPath);
    }
  }
  List<String> allTblStr = hmsHandler.get_all_tables(dbName);
  for (int i = 0; i < allTblStr.size(); i += maxTablesPerCall) {
    List<String> tablesToFetch =
            allTblStr.subList(i, Math.min(
                    i + maxTablesPerCall, allTblStr.size()));
    Callable<CallResult> tableTask =
            new TableTask(db, tablesToFetch, update);
    synchronized (results) {
      results.add(threadPool.submit(tableTask));
    }
  }
}
 

@Test
public void testImportThatUpdatesRegisteredDatabase() throws Exception {
    // setup database
    when(hiveClient.getAllDatabases()).thenReturn(Arrays.asList(new String[]{TEST_DB_NAME}));
    String description = "This is a default database";
    Database db = new Database(TEST_DB_NAME, description, "/user/hive/default", null);
    when(hiveClient.getDatabase(TEST_DB_NAME)).thenReturn(db);
    when(hiveClient.getAllTables(TEST_DB_NAME)).thenReturn(Arrays.asList(new String[]{}));

    returnExistingDatabase(TEST_DB_NAME, atlasClientV2, METADATA_NAMESPACE);

    when(atlasEntityWithExtInfo.getEntity("72e06b34-9151-4023-aa9d-b82103a50e76"))
            .thenReturn((new AtlasEntity.AtlasEntityWithExtInfo(
                    getEntity(HiveDataTypes.HIVE_DB.getName(), AtlasClient.GUID, "72e06b34-9151-4023-aa9d-b82103a50e76"))).getEntity());

    HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(METADATA_NAMESPACE, hiveClient, atlasClientV2);
    bridge.importHiveMetadata(null, null, true);

    // verify update is called
    verify(atlasClientV2).updateEntity(anyObject());
}
 

@Test
public void createDatabaseUsingManualAndWhitelistingUpdatesConfig() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 

@Test
public void createDatabaseDatabaseUsingPrefixAndWhitelistingUpdates() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 

/**
 * Checks if db is already registered, else creates and registers db entity
 * @param databaseName
 * @return
 * @throws Exception
 */
private Referenceable registerDatabase(String databaseName) throws Exception {
    Referenceable dbRef = getDatabaseReference(clusterName, databaseName);
    Database db = hiveClient.getDatabase(databaseName);

    if (db != null) {
        if (dbRef == null) {
            dbRef = createDBInstance(db);
            dbRef = registerInstance(dbRef);
        } else {
            LOG.info("Database {} is already registered with id {}. Updating it.", databaseName, dbRef.getId().id);
            dbRef = createOrUpdateDBInstance(db, dbRef);
            updateInstance(dbRef);
        }
    }
    return dbRef;
}
 

@Before
public void setup() throws IOException,
        TException,
        InvocationTargetException,
        NoSuchMethodException,
        IllegalAccessException,
        NoSuchFieldException, SQLException {
  this.executorService = Executors.newSingleThreadExecutor();
  hiveLocalDir = createTempDirectory("hive", asFileAttribute(fromString("rwxrwxrwx"))).toFile();
  setupDB("jdbc:derby:" + getDerbyPath() + ";create=true");

  this.server = thriftServer();
  executorService.submit(() -> server.serve());

  this.metastoreClient = new HiveMetaStoreClient(this.hiveConf);
  createIfNotExistsCatalog("hive");
  this.metastoreClient.createDatabase(new Database(DB_NAME, "description", getDBPath(), new HashMap<>()));
  new HiveTables(this.hiveConf).create(schema, partitionSpec, DB_NAME, TABLE_NAME);
}
 

@Test
public void readWriteCreateAllowed() throws Exception {
  String writableDatabase = "writable_db";

  localServer.createDatabase(writableDatabase);

  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_AND_CREATE)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  // create rights
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // read rights
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // write rights
  proxy.dropTable(LOCAL_DATABASE, LOCAL_TABLE);
  try {
    proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
    fail("Should get NoSuchObjectException");
  } catch (NoSuchObjectException e) {
    // Local table should be allowed to drop, so it now no longer exists and we get an appropriate exception
  }
}
 

@Test
public void primaryMappedDatabasesPrefixed() throws Exception {
  localServer.createDatabase("random_primary");
  remoteServer.createDatabase("random_federated");

  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .withPrimaryMappedDatabases(new String[] { LOCAL_DATABASE })
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  List<String> allDatabases = proxy.getAllDatabases();
  assertThat(allDatabases.size(), is(2));
  assertThat(allDatabases.get(0), is(LOCAL_DATABASE));
  assertThat(allDatabases.get(1), is(PREFIXED_REMOTE_DATABASE));

  // Ensure that the database is added to mapped-databases
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore primaryMetaStore = federations.getPrimaryMetaStore();
  assertThat(primaryMetaStore.getMappedDatabases().contains("newDB"), is(true));
}
 

public static String getDatabaseName(Database hiveDB) {
    String dbName      = hiveDB.getName().toLowerCase();
    String catalogName = hiveDB.getCatalogName() != null ? hiveDB.getCatalogName().toLowerCase() : null;

    if (StringUtils.isNotEmpty(catalogName) && !StringUtils.equals(catalogName, DEFAULT_METASTORE_CATALOG)) {
        dbName = catalogName + SEP + dbName;
    }

    return dbName;
}
 

/**
 * {@inheritDoc}.
 */
@Override
public void createDatabase(final Database database) throws TException {
    callWrap(HiveMetrics.TagCreateDatabase.getMetricName(), () -> {
        handler.create_database(database);
        return null;
    });
}
 

@Override
public Database getDatabase(String name) throws NoSuchObjectException {
  Database db = super.getDatabase(name);
  try {
    if (filterDatabases(Lists.newArrayList(name)).isEmpty()) {
      throw new NoSuchObjectException(getNoAccessMessageForDB(name));
    }
  } catch (MetaException e) {
    throw new NoSuchObjectException("Failed to authorized access to " + name
        + " : " + e.getMessage());
  }
  return db;
}
 

@Test
public void listValidDatabases() throws TException {
  Database database2 = CatalogToHiveConverter.convertDatabase(getTestDatabase());
  additionalDbForCleanup.add(database2.getName());
  metastoreClient.createDatabase(hiveDB);
  metastoreClient.createDatabase(database2);
  List<String> databaseName = metastoreClient.getAllDatabases();
  assertTrue(databaseName.contains(hiveDB.getName()));
  assertTrue(databaseName.contains(database2.getName()));
}
 

@Test
public void doesNotOverwriteConfigOnShutdownManualMode() throws Exception {
  // Note a similar test for PREFIX is not required
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.MANUAL)
      .overwriteConfigOnShutdown(false)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  Federations federations = stopServerAndGetConfiguration();

  PrimaryMetaStore primaryMetastore = federations.getPrimaryMetaStore();
  List<String> writableDatabases = primaryMetastore.getWritableDatabaseWhiteList();
  assertThat(writableDatabases.size(), is(0));

  // Double check federated metastores
  List<FederatedMetaStore> federatedMetastores = federations.getFederatedMetaStores();
  assertThat(federatedMetastores.size(), is(1));

  List<String> mappedDatabases = federatedMetastores.get(0).getMappedDatabases();
  assertThat(mappedDatabases.size(), is(1));
  assertThat(mappedDatabases.get(0), is(REMOTE_DATABASE));
}
 

@BeforeClass
public static void setup() throws MetaException {
  conf = mock(HiveConf.class);
  wh = mock(Warehouse.class);
  tmpPath = new Path("/db");
  when(wh.getDefaultDatabasePath(anyString())).thenReturn(tmpPath);
  when(wh.getDnsPath(any(Path.class))).thenReturn(tmpPath);
  when(wh.isDir(any(Path.class))).thenReturn(true);
  when(wh.getDatabasePath(any(Database.class))).thenReturn(tmpPath);
  when(conf.get(HiveConf.ConfVars.USERS_IN_ADMIN_ROLE.varname,"")).thenReturn("");

  glueClient = new GlueTestClientFactory().newClient();
  GlueClientFactory clientFactory = mock(GlueClientFactory.class);
  when(clientFactory.newClient()).thenReturn(glueClient);

  metastoreClient = new AWSCatalogMetastoreClient.Builder().withHiveConf(conf).withWarehouse(wh)
          .withClientFactory(clientFactory).build();
  catalogDB = getTestDatabase();
  catalogTable = getTestTable();
  hiveDB = CatalogToHiveConverter.convertDatabase(catalogDB);
  hiveTable = CatalogToHiveConverter.convertTable(catalogTable, catalogDB.getName());

  glueClient.createDatabase(new CreateDatabaseRequest()
    .withDatabaseInput(GlueInputConverter.convertToDatabaseInput(catalogDB)));
  glueClient.createTable(new CreateTableRequest()
    .withDatabaseName(catalogDB.getName())
    .withTableInput(GlueInputConverter.convertToTableInput(catalogTable)));
}
 

public static DatabaseInput convertToDatabaseInput(com.amazonaws.services.glue.model.Database database) {
  DatabaseInput input = new DatabaseInput();

  input.setName(database.getName());
  input.setDescription(database.getDescription());
  input.setLocationUri(database.getLocationUri());
  input.setParameters(database.getParameters());

  return input;
}
 

/**
 * {@inheritDoc}
 */
@Override
public void create_database(final Database database) throws TException {
    requestWrapper("create_database", new Object[]{database}, () -> {
        final String dbName = normalizeIdentifier(database.getName());
        v1.createDatabase(catalogName, dbName,
            DatabaseCreateRequestDto.builder().metadata(database.getParameters()).uri(database.getLocationUri())
                .build());
        return null;
    });
}
 

/**
 * {@inheritDoc}.
 */
@Override
public void createDatabase(final Database database) throws TException {
    try (HiveMetastoreClient client = createMetastoreClient()) {
        client.create_database(database);
    }
}
 

@BeforeClass
public static void startMetastore() throws Exception {
  HiveMetastoreTest.metastore = new TestHiveMetastore();
  metastore.start();
  HiveMetastoreTest.hiveConf = metastore.hiveConf();
  HiveMetastoreTest.metastoreClient = new HiveMetaStoreClient(hiveConf);
  String dbPath = metastore.getDatabasePath(DB_NAME);
  Database db = new Database(DB_NAME, "description", dbPath, new HashMap<>());
  metastoreClient.createDatabase(db);
  HiveMetastoreTest.catalog = new HiveCatalog(hiveConf);
}
 

/**
 * {@inheritDoc}
 */
@Override
public Database get_database(final String name) throws TException {
    return requestWrapper("get_database", new Object[]{name}, () -> {
        final String databaseName = normalizeIdentifier(name);
        final DatabaseDto dto = v1.getDatabase(catalogName, databaseName, true, false);
        return hiveConverters.metacatToHiveDatabase(dto);
    });
}
 

/**
 * Converts from DatabaseDto to the connector database.
 *
 * @param databaseInfo Metacat database Info
 * @return connector database
 */
@Override
public Database fromDatabaseInfo(final DatabaseInfo databaseInfo) {
    final QualifiedName databaseName = databaseInfo.getName();
    final String name = (databaseName == null) ? "" : databaseName.getDatabaseName();
    //this is a temp hack to resolve the uri = null issue
    // final String dbUri = Strings.isNullOrEmpty(databaseInfo.getUri()) ? "file://temp/" : databaseInfo.getUri();
    final Map<String, String> metadata
        = (databaseInfo.getMetadata() != null) ? databaseInfo.getMetadata() : Collections.EMPTY_MAP;
    return new Database(name, name, databaseInfo.getUri(), metadata);
}
 

@Override
public boolean removeProperties(Namespace namespace,  Set<String> properties) {
  Map<String, String> parameter = Maps.newHashMap();

  parameter.putAll(loadNamespaceMetadata(namespace));
  properties.forEach(key -> parameter.put(key, null));
  Database database = convertToDatabase(namespace, parameter);

  return alterHiveDataBase(namespace, database);
}
 

@Test
public void alter_database() throws TException {
  Database database = new Database();
  database.setName(DB_P);
  Database inboundDB = new Database();
  inboundDB.setName("inbound");
  when(primaryMapping.transformInboundDatabase(database)).thenReturn(inboundDB);
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");

  handler.alter_database(DB_P, database);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_database("inbound", inboundDB);
}
 

@Override
public boolean setProperties(Namespace namespace,  Map<String, String> properties) {
  Map<String, String> parameter = Maps.newHashMap();

  parameter.putAll(loadNamespaceMetadata(namespace));
  parameter.putAll(properties);
  Database database = convertToDatabase(namespace, parameter);

  return alterHiveDataBase(namespace, database);
}
 

@Test
public void testCreateDatabase() throws Exception {
	tableEnv.executeSql("create database db1 comment 'db1 comment'");
	Database db = hiveCatalog.getHiveDatabase("db1");
	assertEquals("db1 comment", db.getDescription());
	assertFalse(Boolean.parseBoolean(db.getParameters().get(CatalogConfig.IS_GENERIC)));

	String db2Location = warehouse + "/db2_location";
	tableEnv.executeSql(String.format("create database db2 location '%s' with dbproperties('k1'='v1')", db2Location));
	db = hiveCatalog.getHiveDatabase("db2");
	assertEquals(db2Location, locationPath(db.getLocationUri()));
	assertEquals("v1", db.getParameters().get("k1"));
}
 

protected AtlasEntity toTableEntity(Table table, AtlasEntityExtInfo entityExtInfo) throws Exception {
    Database    db       = getDatabases(table.getDbName());
    AtlasEntity dbEntity = toDbEntity(db);

    if (entityExtInfo != null) {
        if (dbEntity != null) {
            entityExtInfo.addReferredEntity(dbEntity);
        }
    }

    AtlasEntity ret = toTableEntity(AtlasTypeUtil.getObjectId(dbEntity), table, entityExtInfo);

    return ret;
}