org.apache.hadoop.hive.metastore.HiveMetaStoreClient Java Examples
The following examples show how to use
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HivePurgerPublisher.java From incubator-gobblin with Apache License 2.0 | 7 votes |
public void initHiveMetastoreClient() throws Exception { if (this.state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION)) { String superUser = this.state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER); String realm = this.state.getProp(ConfigurationKeys.KERBEROS_REALM); String keytabLocation = this.state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION); log.info("Establishing MetastoreClient connection using " + keytabLocation); UserGroupInformation.loginUserFromKeytab(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation); UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); loginUser.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws TException { HivePurgerPublisher.this.client = new HiveMetaStoreClient(new HiveConf()); return null; } }); } else { HivePurgerPublisher.this.client = new HiveMetaStoreClient(new HiveConf()); } }
Example #2
Source File: HiveMetaStoreClientFactory.java From incubator-gobblin with Apache License 2.0 | 7 votes |
private IMetaStoreClient createMetaStoreClient() throws MetaException { HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { @Override public HiveMetaHook getHook(Table tbl) throws MetaException { if (tbl == null) { return null; } try { HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(hiveConf, tbl.getParameters().get(META_TABLE_STORAGE)); return storageHandler == null ? null : storageHandler.getMetaHook(); } catch (HiveException e) { LOG.error(e.toString()); throw new MetaException("Failed to get storage handler: " + e); } } }; return RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName()); }
Example #3
Source File: HiveMetadataService.java From streamline with Apache License 2.0 | 6 votes |
/** * Creates secure {@link HiveMetadataService}, which delegates to {@link HiveMetaStoreClient} * instantiated with the {@link HiveConf} provided using the first parameter */ public static HiveMetadataService newInstance(HiveConf hiveConf, SecurityContext securityContext, Subject subject, Component hiveMetastore, Collection<ComponentProcess> hiveMetastoreProcesses) throws MetaException, IOException, EntityNotFoundException, PrivilegedActionException { if (SecurityUtil.isKerberosAuthenticated(securityContext)) { UserGroupInformation.setConfiguration(hiveConf); // Sets Kerberos rules UserGroupInformation.getUGIFromSubject(subject); // Adds User principal to this subject return new HiveMetadataService( SecurityUtil.execute(() -> new HiveMetaStoreClient(hiveConf), securityContext, subject), hiveConf, securityContext, subject, hiveMetastore, hiveMetastoreProcesses); } else { return new HiveMetadataService(new HiveMetaStoreClient(hiveConf), hiveConf, securityContext, subject, hiveMetastore, hiveMetastoreProcesses); } }
Example #4
Source File: HiveMetaStoreProxy.java From griffin with Apache License 2.0 | 6 votes |
@Bean public IMetaStoreClient initHiveMetastoreClient() { HiveConf hiveConf = new HiveConf(); hiveConf.set("hive.metastore.local", "false"); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris); hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts); hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval); try { client = HiveMetaStoreClient.newSynchronizedClient(new HiveMetaStoreClient(hiveConf)); } catch (Exception e) { LOGGER.error("Failed to connect hive metastore. {}", e); } return client; }
Example #5
Source File: HiveClientPool.java From iceberg with Apache License 2.0 | 6 votes |
@Override protected HiveMetaStoreClient newClient() { try { return new HiveMetaStoreClient(hiveConf); } catch (MetaException e) { throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore"); } catch (Throwable t) { if (t.getMessage().contains("Another instance of Derby may have already booted")) { throw new RuntimeMetaException(t, "Failed to start an embedded metastore because embedded " + "Derby supports only one client at a time. To fix this, use a metastore that supports " + "multiple clients."); } throw new RuntimeMetaException(t, "Failed to connect to Hive Metastore"); } }
Example #6
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 6 votes |
@Test public void dropTable() throws Exception { HiveServer2Core server = setupServer(); String tableName = "my_table"; createUnpartitionedTable(DATABASE, tableName, server); try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl()); Statement statement = connection.createStatement()) { String dropHql = String.format("DROP TABLE %s.%s", DATABASE, tableName); statement.execute(dropHql); } HiveMetaStoreClient client = server.getCore().newClient(); try { client.getTable(DATABASE, tableName); fail(String.format("Table %s.%s was not deleted", DATABASE, tableName)); } catch (NoSuchObjectException e) { // expected } finally { client.close(); } server.shutdown(); }
Example #7
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 6 votes |
@Test public void dropDatabase() throws Exception { HiveServer2Core server = setupServer(); String databaseName = "Another_DB"; server.getCore().createDatabase(databaseName); try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl()); Statement statement = connection.createStatement()) { String dropHql = String.format("DROP DATABASE %s", databaseName); statement.execute(dropHql); } HiveMetaStoreClient client = server.getCore().newClient(); try { client.getDatabase(databaseName); fail(String.format("Database %s was not deleted", databaseName)); } catch (NoSuchObjectException e) { // expected } finally { client.close(); } server.shutdown(); }
Example #8
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 6 votes |
@Test public void addPartition() throws Exception { HiveServer2Core server = setupServer(); String tableName = "my_table"; createPartitionedTable(DATABASE, tableName, server); try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl()); Statement statement = connection.createStatement()) { String addPartitionHql = String.format("ALTER TABLE %s.%s ADD PARTITION (partcol=1)", DATABASE, tableName); statement.execute(addPartitionHql); } HiveMetaStoreClient client = server.getCore().newClient(); try { List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1); assertThat(partitions.size(), is(1)); assertThat(partitions.get(0).getDbName(), is(DATABASE)); assertThat(partitions.get(0).getTableName(), is(tableName)); assertThat(partitions.get(0).getValues(), is(Arrays.asList("1"))); assertThat(partitions.get(0).getSd().getLocation(), is(String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName))); } finally { client.close(); } server.shutdown(); }
Example #9
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 6 votes |
private Table createUnpartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception { Table table = new Table(); table.setDbName(databaseName); table.setTableName(tableName); table.setSd(new StorageDescriptor()); table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null))); table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"); table.getSd().setSerdeInfo(new SerDeInfo()); table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"); HiveMetaStoreClient client = server.getCore().newClient(); client.createTable(table); client.close(); return table; }
Example #10
Source File: HiveServer2CoreTest.java From beeju with Apache License 2.0 | 6 votes |
private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception { Table table = new Table(); table.setDbName(DATABASE); table.setTableName(tableName); table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null))); table.setSd(new StorageDescriptor()); table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null))); table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"); table.getSd().setSerdeInfo(new SerDeInfo()); table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"); HiveMetaStoreClient client = server.getCore().newClient(); client.createTable(table); client.close(); return table; }
Example #11
Source File: ComparisonToolIntegrationTest.java From circus-train with Apache License 2.0 | 6 votes |
@Before public void init() throws Exception { sourceTempUri = temporaryFolder.newFolder("source-temp"); sourceWarehouseUri = temporaryFolder.newFolder("source-warehouse"); sourceTableUri = new File(sourceWarehouseUri, DATABASE + "/" + SOURCE_TABLE); createSourceTable(); replicaWarehouseUri = temporaryFolder.newFolder("replica-warehouse"); replicaTableUri = new File(replicaWarehouseUri, DATABASE + "/" + REPLICA_TABLE); createReplicaTable(); ymlFile = temporaryFolder.newFile("diff.yml"); HiveMetaStoreClient client = catalog.client(); LOG.info(">>>> Source Table = {}", client.getTable(DATABASE, SOURCE_TABLE)); LOG.info(">>>> Replica Table = {}", client.getTable(DATABASE, REPLICA_TABLE)); // Create hive config testHiveSiteXml = new HiveXml(catalog.getThriftConnectionUri(), catalog.connectionURL(), catalog.driverClassName()) .create() .getCanonicalPath(); outputFile = temporaryFolder.newFile("diff.out"); }
Example #12
Source File: HiveTableBaseTest.java From iceberg with Apache License 2.0 | 6 votes |
@Before public void setup() throws IOException, TException, InvocationTargetException, NoSuchMethodException, IllegalAccessException, NoSuchFieldException, SQLException { this.executorService = Executors.newSingleThreadExecutor(); hiveLocalDir = createTempDirectory("hive", asFileAttribute(fromString("rwxrwxrwx"))).toFile(); setupDB("jdbc:derby:" + getDerbyPath() + ";create=true"); this.server = thriftServer(); executorService.submit(() -> server.serve()); this.metastoreClient = new HiveMetaStoreClient(this.hiveConf); createIfNotExistsCatalog("hive"); this.metastoreClient.createDatabase(new Database(DB_NAME, "description", getDBPath(), new HashMap<>())); new HiveTables(this.hiveConf).create(schema, partitionSpec, DB_NAME, TABLE_NAME); }
Example #13
Source File: HiveDifferencesIntegrationTest.java From circus-train with Apache License 2.0 | 6 votes |
@Before public void init() throws Exception { // We mock the checksum function because LocalFileSystem doesn't compute checksums when(checksumFunction.apply(any(Path.class))).thenAnswer(new Answer<String>() { @Override public String answer(InvocationOnMock invocation) throws Throwable { Path path = (Path) invocation.getArguments()[0]; return path.getName(); } }); sourceWarehouseUri = temporaryFolder.newFolder("source-warehouse"); sourceTableUri = new File(sourceWarehouseUri, DATABASE + "/" + SOURCE_TABLE); createTable(DATABASE, SOURCE_TABLE, sourceTableUri, null, null, false); replicaWarehouseUri = temporaryFolder.newFolder("replica-warehouse"); replicaTableUri = new File(replicaWarehouseUri, DATABASE + "/" + REPLICA_TABLE); createTable(DATABASE, REPLICA_TABLE, replicaTableUri, DATABASE + "." + SOURCE_TABLE, sourceTableUri.toURI().toString(), true); HiveMetaStoreClient client = catalog.client(); LOG.info(">>>> Source Table = {}", client.getTable(DATABASE, SOURCE_TABLE)); LOG.info(">>>> Replica Table = {}", client.getTable(DATABASE, REPLICA_TABLE)); }
Example #14
Source File: TestUtils.java From waggle-dance with Apache License 2.0 | 6 votes |
static Table createPartitionedTable(HiveMetaStoreClient metaStoreClient, String database, String table, File location) throws Exception { Table hiveTable = new Table(); hiveTable.setDbName(database); hiveTable.setTableName(table); hiveTable.setTableType(TableType.EXTERNAL_TABLE.name()); hiveTable.putToParameters("EXTERNAL", "TRUE"); hiveTable.setPartitionKeys(PARTITION_COLUMNS); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(DATA_COLUMNS); sd.setLocation(location.toURI().toString()); sd.setParameters(new HashMap<>()); sd.setSerdeInfo(new SerDeInfo()); hiveTable.setSd(sd); metaStoreClient.createTable(hiveTable); return hiveTable; }
Example #15
Source File: TestMetaStoreWithPigHCat.java From incubator-sentry with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME); FileOutputStream to = new FileOutputStream(dataFile); Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); to.close(); policyFile = setAdminOnServer1(ADMINGROUP); policyFile .addRolesToGroup(USERGROUP1, db_all_role) .addRolesToGroup(USERGROUP2, "read_db_role") .addPermissionsToRole(db_all_role, "server=server1->db=" + dbName) .addPermissionsToRole("read_db_role", "server=server1->db=" + dbName + "->table=" + tabName2 + "->action=SELECT") .setUserGroupMapping(StaticUserGroup.getStaticMapping()); writePolicyFile(policyFile); HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1); client.dropDatabase(dbName, true, true, true); createMetastoreDB(client, dbName); client.close(); }
Example #16
Source File: HiveAdapter.java From hadoop-etl-udfs with MIT License | 6 votes |
static SchemaMetadata readMetadata(SchemaMetadataInfo schemaMetaInfo, ExaConnectionInformation connection, List<String> tableNames, String newSchema) throws SQLException, MetadataException { String databaseName = newSchema; if (newSchema == null) { databaseName = HiveAdapterProperties.getSchema(schemaMetaInfo.getProperties()); } HiveMetaStoreClient hiveMetastoreClient = HiveMetastoreService.getHiveMetastoreClient(connection.getAddress(), HiveQueryGenerator.isKerberosAuth(connection.getPassword()), HiveAdapterProperties.getConnectionName(schemaMetaInfo.getProperties())); if (tableNames == null) { try { tableNames = hiveMetastoreClient.getAllTables(databaseName); } catch (MetaException e) { e.getMessage(); } } List<TableMetadata> tables = new ArrayList<>(); if (tableNames != null) { for (String tableName : tableNames) { Table table = HiveTableInformation.getHiveTable(hiveMetastoreClient, tableName, databaseName); tables.add(HiveTableInformation.getTableMetadataFromHCatTableMetadata(tableName.toUpperCase(), HiveTableInformation.getHCatTableMetadata(table))); } } return new SchemaMetadata("", tables); }
Example #17
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void getDatabaseFromPatternPrefixed() throws Exception { runner = WaggleDanceRunner .builder(configLocation) .databaseResolution(DatabaseResolution.PREFIXED) .overwriteConfigOnShutdown(false) .primary("primary", localServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST) .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), "remote.?database") .federate("third", remoteServer.getThriftConnectionUri(), "no_match.*") .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); List<String> allDatabases = proxy.getAllDatabases(); List<String> expected = Lists.newArrayList("default", LOCAL_DATABASE, PREFIXED_REMOTE_DATABASE); assertThat(allDatabases, is(expected)); }
Example #18
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void alterTableOnFederatedIsNotAllowedUsingManual() throws Exception { String[] mappableDatabases = new String[] { REMOTE_DATABASE }; String[] writableDatabaseWhitelist = new String[] { REMOTE_DATABASE }; runner = WaggleDanceRunner .builder(configLocation) .databaseResolution(DatabaseResolution.MANUAL) .primary("primary", localServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST) .federate("doesNotMatterManualMode", remoteServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_ON_DATABASE_WHITELIST, mappableDatabases, writableDatabaseWhitelist) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); Table table = proxy.getTable(REMOTE_DATABASE, REMOTE_TABLE); Table newTable = new Table(table); newTable.setTableName("new_remote_table"); proxy.alter_table_with_environmentContext(REMOTE_DATABASE, REMOTE_TABLE, newTable, null); assertThat(proxy.tableExists(REMOTE_DATABASE, "new_remote_table"), is(true)); }
Example #19
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void createDatabaseDatabaseUsingPrefixAndWhitelistingUpdates() throws Exception { runner = WaggleDanceRunner .builder(configLocation) .databaseResolution(DatabaseResolution.PREFIXED) .primary("primary", localServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null)); Database newDB = proxy.getDatabase("newDB"); assertNotNull(newDB); // Should be allowed due to newly loaded write privileges proxy.alterDatabase("newDB", new Database(newDB)); Federations federations = stopServerAndGetConfiguration(); PrimaryMetaStore metaStore = federations.getPrimaryMetaStore(); assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1)); assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb")); // Mapped databases should still map everything assertThat(metaStore.getMappedDatabases(), is(nullValue())); }
Example #20
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void createDatabaseUsingManualAndWhitelistingUpdatesConfig() throws Exception { runner = WaggleDanceRunner .builder(configLocation) .primary("primary", localServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null)); Database newDB = proxy.getDatabase("newDB"); assertNotNull(newDB); // Should be allowed due to newly loaded write privileges proxy.alterDatabase("newDB", new Database(newDB)); Federations federations = stopServerAndGetConfiguration(); PrimaryMetaStore metaStore = federations.getPrimaryMetaStore(); assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1)); assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb")); // Mapped databases should still map everything assertThat(metaStore.getMappedDatabases(), is(nullValue())); }
Example #21
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void usePrefix() throws Exception { runner = WaggleDanceRunner .builder(configLocation) .databaseResolution(DatabaseResolution.PREFIXED) .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY) .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri()) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); // Local table Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE); Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE); assertThat(waggledLocalTable, is(localTable)); // Remote table String waggledRemoteDbName = PREFIXED_REMOTE_DATABASE; assertTypicalRemoteTable(proxy, waggledRemoteDbName); }
Example #22
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
@Test public void typical() throws Exception { runner = WaggleDanceRunner .builder(configLocation) .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY) .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); // Local table Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE); Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE); assertThat(waggledLocalTable, is(localTable)); // Remote table String waggledRemoteDbName = REMOTE_DATABASE; assertTypicalRemoteTable(proxy, waggledRemoteDbName); }
Example #23
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 6 votes |
private void createRemoteTable(File tableUri) throws Exception { HiveMetaStoreClient client = remoteServer.client(); Table hiveTable = createPartitionedTable(client, REMOTE_DATABASE, REMOTE_TABLE, tableUri); File partitionEurope = new File(tableUri, "continent=Europe"); File partitionUk = new File(partitionEurope, "country=UK"); File partitionAsia = new File(tableUri, "continent=Asia"); File partitionChina = new File(partitionAsia, "country=China"); LOG .info(">>>> Partitions added: {}", client .add_partitions(Arrays .asList(newPartition(hiveTable, Arrays.asList("Europe", "UK"), partitionUk), newPartition(hiveTable, Arrays.asList("Asia", "China"), partitionChina)))); }
Example #24
Source File: AbstractMetastoreTestWithStaticConfiguration.java From incubator-sentry with Apache License 2.0 | 6 votes |
public Table makeMetastoreTableObject(HiveMetaStoreClient client, String dbName, String tabName, List<FieldSchema> cols) throws Exception { Table tbl = new Table(); tbl.setDbName(dbName); tbl.setTableName(tabName); StorageDescriptor sd = new StorageDescriptor(); tbl.setSd(sd); tbl.setParameters(new HashMap<String, String>()); sd.setCols(cols); sd.setCompressed(false); sd.setParameters(new HashMap<String, String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters() .put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setSortCols(new ArrayList<Order>()); return tbl; }
Example #25
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Test public void usePrimaryPrefix() throws Exception { String primaryPrefix = "primary_"; runner = WaggleDanceRunner .builder(configLocation) .databaseResolution(DatabaseResolution.PREFIXED) .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY) .withPrimaryPrefix(primaryPrefix) .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri()) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); // Local table String prefixedLocalDbName = primaryPrefix + LOCAL_DATABASE; Table localTable = proxy.getTable(prefixedLocalDbName, LOCAL_TABLE); assertThat(localTable.getDbName(), is(prefixedLocalDbName)); // fetch without prefix works and result is prefixed Table localTable2 = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE); assertThat(localTable2.getDbName(), is(prefixedLocalDbName)); // Remote table String prefixedRemoteDbName = PREFIXED_REMOTE_DATABASE; assertTypicalRemoteTable(proxy, prefixedRemoteDbName); }
Example #26
Source File: HiveClientPool.java From iceberg with Apache License 2.0 | 5 votes |
@Override protected HiveMetaStoreClient reconnect(HiveMetaStoreClient client) { try { client.close(); client.reconnect(); } catch (MetaException e) { throw new RuntimeMetaException(e, "Failed to reconnect to Hive Metastore"); } return client; }
Example #27
Source File: WaggleDanceIntegrationTest.java From waggle-dance with Apache License 2.0 | 5 votes |
@Test public void readWriteCreateAllowed() throws Exception { String writableDatabase = "writable_db"; localServer.createDatabase(writableDatabase); runner = WaggleDanceRunner .builder(configLocation) .primary("primary", localServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_AND_CREATE) .build(); runWaggleDance(runner); HiveMetaStoreClient proxy = getWaggleDanceClient(); // create rights proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null)); Database newDB = proxy.getDatabase("newDB"); assertNotNull(newDB); // read rights Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE); Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE); assertThat(waggledLocalTable, is(localTable)); // write rights proxy.dropTable(LOCAL_DATABASE, LOCAL_TABLE); try { proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE); fail("Should get NoSuchObjectException"); } catch (NoSuchObjectException e) { // Local table should be allowed to drop, so it now no longer exists and we get an appropriate exception } }
Example #28
Source File: HiveMetastoreService.java From hadoop-etl-udfs with MIT License | 5 votes |
public static HiveMetaStoreClient checkHiveMetaStoreClient(String hiveMetastoreUrl,boolean useKerberos, String hcatUserOrServicePrincipal) throws MetaException { HiveConf hiveConf = new HiveConf(new Configuration(), HiveConf.class); hiveConf.set("hive.metastore.local", "false"); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUrl); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); if (useKerberos) { System.out.println("Add hive.metastore.kerberos.principal: " + hcatUserOrServicePrincipal); hiveConf.set("hive.metastore.kerberos.principal", hcatUserOrServicePrincipal); hiveConf.set("hive.metastore.sasl.enabled", "true"); } return new HiveMetaStoreClient(hiveConf); }
Example #29
Source File: Context.java From incubator-sentry with Apache License 2.0 | 5 votes |
public HiveMetaStoreClient getMetaStoreClient(String userName) throws Exception { UserGroupInformation clientUgi = UserGroupInformation.createRemoteUser(userName); HiveMetaStoreClient client = (HiveMetaStoreClient) clientUgi. doAs(new PrivilegedExceptionAction<Object> () { @Override public HiveMetaStoreClient run() throws Exception { return new HiveMetaStoreClient(new HiveConf()); } }); return client; }
Example #30
Source File: HiveMetastoreTest.java From iceberg with Apache License 2.0 | 5 votes |
@BeforeClass public static void startMetastore() throws Exception { HiveMetastoreTest.metastore = new TestHiveMetastore(); metastore.start(); HiveMetastoreTest.hiveConf = metastore.hiveConf(); HiveMetastoreTest.metastoreClient = new HiveMetaStoreClient(hiveConf); String dbPath = metastore.getDatabasePath(DB_NAME); Database db = new Database(DB_NAME, "description", dbPath, new HashMap<>()); metastoreClient.createDatabase(db); HiveMetastoreTest.catalog = new HiveCatalog(hiveConf); }