org.apache.hadoop.hive.metastore.api.TableMeta Java Examples

@Test
public void panopticOperationsHandlerGetTableMeta() throws Exception {
  String pattern = "pattern";
  TableMeta primaryTableMeta = new TableMeta(PRIMARY_DB, "tbl", null);
  TableMeta federatedTableMeta = new TableMeta(FEDERATED_DB, "tbl", null);
  TableMeta ignoredTableMeta = new TableMeta("non_mapped_db", "tbl", null);

  when(primaryDatabaseClient.get_table_meta(pattern, pattern, null))
      .thenReturn(Collections.singletonList(primaryTableMeta));
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta(pattern, pattern, null))
      .thenReturn(Arrays.asList(federatedTableMeta, ignoredTableMeta));

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> expected = Lists.newArrayList(primaryTableMeta, federatedTableMeta);
  List<TableMeta> result = handler.getTableMeta(pattern, pattern, null);
  assertThat(result, is(expected));
}
 

public List<TableMeta> getTableMeta(
    String dbPatterns,
    String tablePatterns,
    List<String> tableTypes
) throws TException  {
  List<TableMeta> tables = new ArrayList<>();
  List<String> databases = getDatabases(dbPatterns);
  for (String dbName : databases) {
    String nextToken = null;
    List<Table> dbTables = glueMetastore.getTables(dbName, tablePatterns);
    for (Table catalogTable : dbTables) {
      if (tableTypes == null ||
          tableTypes.isEmpty() ||
          tableTypes.contains(catalogTable.getTableType())) {
        tables.add(CatalogToHiveConverter.convertTableMeta(catalogTable, dbName));
      }
    }
  }
  return tables;
}
 

@Test
public void panopticOperationsHandlerGetTableMetaWithNonWhitelistedDb() throws TException {
  List<String> tblTypes = Lists.newArrayList();
  TableMeta tableMeta = new TableMeta("federated_db", "tbl", null);
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta("federated_*", "*", tblTypes))
      .thenReturn(Lists.newArrayList(tableMeta));

  // set metastore whitelist to be nonempty
  federatedMetastore.setMappedDatabases(Collections.singletonList("testName"));
  service = new PrefixBasedDatabaseMappingService(metaStoreMappingFactory,
      Arrays.asList(primaryMetastore, federatedMetastore), queryMapping);

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> tableMetas = handler.getTableMeta("name_federated_*", "*", tblTypes);

  assertThat(tableMetas.size(), is(0));
}
 

@Test
public void panopticOperationsHandlerGetTableMeta() throws Exception {
  TableMeta federatedTableMeta = new TableMeta("federated_db", "tbl", null);
  TableMeta primaryTableMeta = new TableMeta("primary_db", "tbl", null);

  when(primaryDatabaseClient.get_table_meta("*_db", "*", null))
      .thenReturn(Collections.singletonList(primaryTableMeta));
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta("*_db", "*", null))
      .thenReturn(Collections.singletonList(federatedTableMeta));
  when(metaStoreMappingFederated.transformOutboundDatabaseName("federated_db")).thenReturn("name_federated_db");

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> expected = Arrays.asList(primaryTableMeta, federatedTableMeta);
  List<TableMeta> result = handler.getTableMeta("*_db", "*", null);
  assertThat(result, is(expected));
}
 

@Test
public void testGetTableMetaNullEmptyTableType() throws Exception {
  List<Table> tables = Lists.newArrayList(testTbl);
  List<String> tableTypes = null;

  when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
    new GetDatabasesResult().withDatabaseList(testDb));
  when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
    new GetTablesResult().withTableList(tables));

  List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));

  tableTypes = Lists.newArrayList();
  tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
 

protected List<TableMeta> getTableMeta(
    String tablePatterns,
    List<String> tableTypes,
    Map<DatabaseMapping, String> databaseMappingsForPattern,
    BiFunction<TableMeta, DatabaseMapping, Boolean> filter) {
  List<GetTableMetaRequest> allRequests = new ArrayList<>();

  for (Entry<DatabaseMapping, String> mappingWithPattern : databaseMappingsForPattern.entrySet()) {
    DatabaseMapping mapping = mappingWithPattern.getKey();
    GetTableMetaRequest tableMetaRequest = new GetTableMetaRequest(mapping, mappingWithPattern.getValue(),
        tablePatterns, tableTypes, filter);
    allRequests.add(tableMetaRequest);
  }

  List<TableMeta> result = getPanopticOperationExecutor()
      .executeRequests(allRequests, GET_TABLE_META_TIMEOUT, "Got exception fetching get_table_meta: {}");
  return result;
}
 

public static TableMeta convertTableMeta(com.amazonaws.services.glue.model.Table catalogTable, String dbName) {
  TableMeta tableMeta = new TableMeta();
  tableMeta.setDbName(dbName);
  tableMeta.setTableName(catalogTable.getName());
  tableMeta.setTableType(catalogTable.getTableType());
  if (catalogTable.getParameters().containsKey("comment")) {
    tableMeta.setComments(catalogTable.getParameters().get("comment"));
  }
  return tableMeta;
}
 

@Test
public void transformOutboundTableMeta() throws Exception {
  TableMeta tableMeta = new TableMeta();
  tableMeta.setDbName(DB_NAME);
  TableMeta result = databaseMapping.transformOutboundTableMeta(tableMeta);
  assertThat(result, is(sameInstance(tableMeta)));
  assertThat(result.getDbName(), is(OUT_DB_NAME));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<TableMeta> get_table_meta(String db_patterns, String tbl_patterns, List<String> tbl_types)
    throws MetaException, TException {
  return databaseMappingService.getPanopticOperationHandler()
      .getTableMeta(db_patterns, tbl_patterns, tbl_types);
}
 

@Override
public List<TableMeta> call() throws TException {
  List<TableMeta> tables = mapping.getClient().get_table_meta(dbPattern, tablePattern, tableTypes);
  List<TableMeta> mappedTableMeta = new ArrayList<>();
  for (TableMeta tableMeta : tables) {
    if (filter.apply(tableMeta, mapping)) {
      mappedTableMeta.add(mapping.transformOutboundTableMeta(tableMeta));
    }
  }
  return mappedTableMeta;
}
 

public GetTableMetaRequest(
    DatabaseMapping mapping,
    String dbPattern,
    String tablePattern,
    List<String> tableTypes,
    BiFunction<TableMeta, DatabaseMapping, Boolean> filter) {
  this.mapping = mapping;
  this.dbPattern = dbPattern;
  this.tablePattern = tablePattern;
  this.tableTypes = tableTypes;
  this.filter = filter;
}
 

@Test
public void testGetTableMeta() throws Exception {
  List<Table> tables = Lists.newArrayList(testTbl);
  List<String> tableTypes = Lists.newArrayList(TableType.MANAGED_TABLE.name());

  when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
      new GetDatabasesResult().withDatabaseList(testDb));
  when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
      new GetTablesResult().withTableList(tables));

  List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
 

@Test
public void testTableMetaConversion() {
  Table catalogTable = TestObjects.getTestTable();
  TableMeta tableMeta = CatalogToHiveConverter.convertTableMeta(catalogTable, TEST_DB_NAME);
  assertEquals(catalogTable.getName(), tableMeta.getTableName());
  assertEquals(TEST_DB_NAME, tableMeta.getDbName());
  assertEquals(catalogTable.getTableType(), tableMeta.getTableType());
}
 

@Override
public TableMeta transformOutboundTableMeta(TableMeta tableMeta) {
  return tableMeta;
}
 

@Override
public TableMeta transformOutboundTableMeta(TableMeta tableMeta) {
  tableMeta.setDbName(metaStoreMapping.transformOutboundDatabaseName(tableMeta.getDbName()));
  return tableMeta;
}
 

@Test
public void transformOutboundTableMeta() throws Exception {
  TableMeta tableMeta = new TableMeta();
  TableMeta result = databaseMapping.transformOutboundTableMeta(tableMeta);
  assertThat(result, is(sameInstance(tableMeta)));
}
 

@Override
public List<TableMeta> getTableMeta(String dbPatterns, String tablePatterns, List<String> tableTypes)
    throws MetaException, TException, UnknownDBException {
  return glueMetastoreClientDelegate.getTableMeta(dbPatterns, tablePatterns, tableTypes);
}
 

public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
    throws MetaException {
  List<TableMeta> tableMetas = rs.getTableMeta(dbNames, tableNames, tableTypes);
  return tableMetas;
}
 

/**
 * Implements {@link HMSHandler#get_table_meta(String, String, List)} over multiple metastores
 *
 * @param databasePatterns database patterns to match
 * @param tablePatterns table patterns to match
 * @param tableTypes table types to match
 * @return list of table metadata
 */
abstract public List<TableMeta> getTableMeta(String databasePatterns, String tablePatterns, List<String> tableTypes);
 

TableMeta transformOutboundTableMeta(TableMeta tableMeta);