org.apache.hadoop.hive.metastore.api.Index Java Examples

@Test
public void testListIndexNames() throws Exception{
  Index testIndex2 = getTestHiveIndex(testDB.getName());
  List<String> expectedIndexNameList = ImmutableList.of(testIndex.getIndexName(), testIndex2.getIndexName());
  List<Index> indexList = ImmutableList.of(testIndex, testIndex2);
  for (Index index : indexList) {
    Table catalogIndex = HiveToCatalogConverter.convertIndexToTableObject(index);
    testTable.getParameters().put(INDEX_PREFIX + index.getIndexName(), catalogTableToString(catalogIndex));
  }

  when(glueClient.getTable(any(GetTableRequest.class)))
    .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  List<String> result = metastoreClient.listIndexNames(testTable.getDbName(), testTable.getTableName(), (short)2);

  verify(glueClient).getTable(any(GetTableRequest.class));
  assertEquals(expectedIndexNameList.size(), result.size());
  for (String indexName : expectedIndexNameList) {
    assertTrue(result.contains(indexName));
  }
}
 

@Test (expected = NoSuchObjectException.class)
public void listIndexesInvalidWithUnknownTable() throws TException {
  Index index2 = TestObjects.getTestHiveIndex(hiveDB.getName());
  Index index3 = TestObjects.getTestHiveIndex(hiveDB.getName());

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  Table hiveIndexTable3 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());

  index2.setOrigTableName(hiveTable.getTableName());
  index2.setIndexTableName(hiveIndexTable2.getTableName());
  index3.setOrigTableName(hiveTable.getTableName());
  index3.setIndexTableName(hiveIndexTable3.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  metastoreClient.createIndex(index2, hiveIndexTable2);
  metastoreClient.createIndex(index3, hiveIndexTable3);
  metastoreClient.listIndexes(hiveDB.getName(), invalidTable, (short) 2);
}
 

@Test
public void listIndexesValid() throws TException {
  Index index2 = TestObjects.getTestHiveIndex(hiveDB.getName());
  Index index3 = TestObjects.getTestHiveIndex(hiveDB.getName());

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  Table hiveIndexTable3 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());

  index2.setOrigTableName(hiveTable.getTableName());
  index2.setIndexTableName(hiveIndexTable2.getTableName());
  index3.setOrigTableName(hiveTable.getTableName());
  index3.setIndexTableName(hiveIndexTable3.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  metastoreClient.createIndex(index2, hiveIndexTable2);
  metastoreClient.createIndex(index3, hiveIndexTable3);
  List<Index> indexList = new ArrayList<>();
  indexList.add(hiveIndex);
  indexList.add(index3);
  indexList.add(index2);

  List<Index> result = metastoreClient.listIndexes(hiveDB.getName(), hiveTable.getTableName(), (short) 2);
  assertTrue(result.containsAll(indexList));
}
 

@Test (expected = NoSuchObjectException.class)
public void listIndexesInvalidWithUnknownDb() throws TException {
  Index index2 = TestObjects.getTestHiveIndex(hiveDB.getName());
  Index index3 = TestObjects.getTestHiveIndex(hiveDB.getName());

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  Table hiveIndexTable3 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());

  index2.setOrigTableName(hiveTable.getTableName());
  index2.setIndexTableName(hiveIndexTable2.getTableName());
  index3.setOrigTableName(hiveTable.getTableName());
  index3.setIndexTableName(hiveIndexTable3.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  metastoreClient.createIndex(index2, hiveIndexTable2);
  metastoreClient.createIndex(index3, hiveIndexTable3);
  metastoreClient.listIndexes(invalidDatabase, hiveTable.getTableName(), (short) 2);
}
 

@Test
public void listIndexNamesValid() throws TException {
  Index index2 = TestObjects.getTestHiveIndex(hiveDB.getName());
  Index index3 = TestObjects.getTestHiveIndex(hiveDB.getName());

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  Table hiveIndexTable3 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());

  index2.setOrigTableName(hiveTable.getTableName());
  index2.setIndexTableName(hiveIndexTable2.getTableName());
  index3.setOrigTableName(hiveTable.getTableName());
  index3.setIndexTableName(hiveIndexTable3.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  metastoreClient.createIndex(index2, hiveIndexTable2);
  metastoreClient.createIndex(index3, hiveIndexTable3);
  List<String> indexNameList = new ArrayList<>();
  indexNameList.add(hiveIndex.getIndexName());
  indexNameList.add(index2.getIndexName());
  indexNameList.add(index3.getIndexName());

  List<String> result = metastoreClient.listIndexNames(hiveDB.getName(), hiveTable.getTableName(), (short) 2);

  assertTrue(result.containsAll(indexNameList));
}
 

@Test (expected = NoSuchObjectException.class)
public void listIndexNameInvalidWithUnknownTable() throws TException {
  Index index2 = TestObjects.getTestHiveIndex(hiveDB.getName());
  Index index3 = TestObjects.getTestHiveIndex(hiveDB.getName());

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  Table hiveIndexTable3 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());

  index2.setOrigTableName(hiveTable.getTableName());
  index2.setIndexTableName(hiveIndexTable2.getTableName());
  index3.setOrigTableName(hiveTable.getTableName());
  index3.setIndexTableName(hiveIndexTable3.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  metastoreClient.createIndex(index2, hiveIndexTable2);
  metastoreClient.createIndex(index3, hiveIndexTable3);

  metastoreClient.listIndexNames(hiveDB.getName(), invalidTable, (short) 2);
}
 

@Test (expected = NoSuchObjectException.class)
public void listIndexNameInvalidWithUnknownDb() throws TException {
  Index index2 = TestObjects.getTestHiveIndex(hiveDB.getName());
  Index index3 = TestObjects.getTestHiveIndex(hiveDB.getName());

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  Table hiveIndexTable3 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());

  index2.setOrigTableName(hiveTable.getTableName());
  index2.setIndexTableName(hiveIndexTable2.getTableName());
  index3.setOrigTableName(hiveTable.getTableName());
  index3.setIndexTableName(hiveIndexTable3.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  metastoreClient.createIndex(index2, hiveIndexTable2);
  metastoreClient.createIndex(index3, hiveIndexTable3);

  metastoreClient.listIndexNames(invalidDatabase, hiveTable.getTableName(), (short) 2);
}
 

@Override
public boolean dropIndex(String dbName, String tblName, String name, boolean deleteData) throws NoSuchObjectException,
      MetaException, TException {
    Index indexToDrop = getIndex(dbName, tblName, name);
    String indexTableName = indexToDrop.getIndexTableName();

    // Drop the index metadata
    org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, tblName);
    Map<String, String> parameters = originTable.getParameters();
    String indexTableObjectName = INDEX_PREFIX + name;
    if (!parameters.containsKey(indexTableObjectName)) {
        throw new NoSuchObjectException("can not find Index: " + name);
    }
    parameters.remove(indexTableObjectName);

    alter_table(dbName, tblName, originTable);

    // Now drop the data associated with the table used to hold the index data
    if(indexTableName != null && indexTableName.length() > 0) {
        dropTable(dbName, indexTableName, deleteData, true);
    }

    return true;
}
 

public List<Index> listIndexes(String dbName, String tblName) throws TException {
  checkArgument(StringUtils.isNotEmpty(dbName), "dbName cannot be null or empty");
  checkArgument(StringUtils.isNotEmpty(tblName), "tblName cannot be null or empty");

  org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, tblName);
  Map<String, String> parameters = originTable.getParameters();
  List<Table> indexTableObjects = Lists.newArrayList();
  for(String key : parameters.keySet()) {
    if(key.startsWith(INDEX_PREFIX)) {
      String serialisedString = parameters.get(key);
      indexTableObjects.add(stringToCatalogTable(serialisedString));
    }
  }

  List<Index> hiveIndexList = Lists.newArrayList();
  for (Table catalogIndexTableObject : indexTableObjects) {
    hiveIndexList.add(CatalogToHiveConverter.convertTableObjectToIndex(catalogIndexTableObject));
  }
  return hiveIndexList;
}
 

@Test
public void add_index() throws TException {
  Index newIndex = new Index();
  newIndex.setDbName(DB_P);
  Index inboundIndex = new Index();
  Index outboundIndex = new Index();
  Table newTable = new Table();
  newTable.setDbName(DB_P);
  Table inboundTable = new Table();

  when(primaryMapping.transformInboundIndex(newIndex)).thenReturn(inboundIndex);
  when(primaryMapping.transformInboundTable(newTable)).thenReturn(inboundTable);
  when(primaryMapping.transformOutboundIndex(outboundIndex)).thenReturn(newIndex);
  when(primaryClient.add_index(inboundIndex, inboundTable)).thenReturn(outboundIndex);

  Index result = handler.add_index(newIndex, newTable);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  assertThat(result, is(newIndex));
}
 

public static com.amazonaws.services.glue.model.Table convertIndexToTableObject(Index hiveIndex) {
  // convert index object to a table object
  com.amazonaws.services.glue.model.Table catalogIndexTableObject = new com.amazonaws.services.glue.model.Table();
  catalogIndexTableObject.setName(hiveIndex.getIndexName());
  catalogIndexTableObject.setCreateTime(new Date((long) (hiveIndex.getCreateTime()) * 1000));
  catalogIndexTableObject.setLastAccessTime(new Date((long) (hiveIndex.getLastAccessTime()) * 1000));
  catalogIndexTableObject.setStorageDescriptor(convertStorageDescriptor(hiveIndex.getSd()));
  catalogIndexTableObject.setParameters(hiveIndex.getParameters());

  // store rest of fields in index to paramter map
  catalogIndexTableObject.getParameters().put(INDEX_DEFERRED_REBUILD, hiveIndex.isDeferredRebuild() ? "TRUE": "FALSE");
  catalogIndexTableObject.getParameters().put(INDEX_TABLE_NAME, hiveIndex.getIndexTableName());
  catalogIndexTableObject.getParameters().put(INDEX_HANDLER_CLASS, hiveIndex.getIndexHandlerClass());
  catalogIndexTableObject.getParameters().put(INDEX_DB_NAME, hiveIndex.getDbName());
  catalogIndexTableObject.getParameters().put(INDEX_ORIGIN_TABLE_NAME, hiveIndex.getOrigTableName());

  return catalogIndexTableObject;
}
 

public static Index convertTableObjectToIndex(com.amazonaws.services.glue.model.Table catalogTable) {
  Index hiveIndex = new Index();
  Map<String, String> parameters = catalogTable.getParameters();
  hiveIndex.setIndexName(catalogTable.getName());
  hiveIndex.setCreateTime((int) (catalogTable.getCreateTime().getTime() / 1000));
  hiveIndex.setLastAccessTime((int) (catalogTable.getLastAccessTime().getTime() / 1000));
  hiveIndex.setSd(convertStorageDescriptor(catalogTable.getStorageDescriptor()));
  hiveIndex.setParameters(catalogTable.getParameters());

  hiveIndex.setDeferredRebuild(parameters.get(INDEX_DEFERRED_REBUILD).equals("TRUE"));
  hiveIndex.setIndexHandlerClass(parameters.get(INDEX_HANDLER_CLASS));
  hiveIndex.setDbName(parameters.get(INDEX_DB_NAME));
  hiveIndex.setOrigTableName(parameters.get(INDEX_ORIGIN_TABLE_NAME));
  hiveIndex.setIndexTableName(parameters.get(INDEX_TABLE_NAME));

  return hiveIndex;
}
 

@Override
public boolean dropIndex(String dbName, String tblName, String name, boolean deleteData) throws NoSuchObjectException,
      MetaException, TException {
  Index indexToDrop = getIndex(dbName, tblName, name);
  String indexTableName = indexToDrop.getIndexTableName();

  // Drop the index metadata
  org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, tblName);
  Map<String, String> parameters = originTable.getParameters();
  String indexTableObjectName = INDEX_PREFIX + name;
  if (!parameters.containsKey(indexTableObjectName)) {
    throw new NoSuchObjectException("can not find Index: " + name);
  }
  parameters.remove(indexTableObjectName);

  alter_table(dbName, tblName, originTable);

  // Now drop the data associated with the table used to hold the index data
  if(indexTableName != null && indexTableName.length() > 0) {
    dropTable(dbName, indexTableName, deleteData, true);
  }

  return true;
}
 

@Override
public List<String> listIndexNames(String db_name, String tbl_name, short max) throws MetaException, TException {
  // In current hive implementation, it ignores fields "max"
  // https://github.com/apache/hive/blob/rel/release-2.3.0/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java#L3902-L3932
  List<Index> indexes = listIndexes(db_name, tbl_name, max);
  List<String> indexNames = Lists.newArrayList();
  for(Index index : indexes) {
    indexNames.add(index.getIndexName());
  }

  return indexNames;
}
 

@Override
public List<Index> transformOutboundIndexes(List<Index> indexes) {
  for (Index index : indexes) {
    transformOutboundIndex(index);
  }
  return indexes;
}
 

@Override
public Index getIndex(String dbName, String tblName, String indexName) throws MetaException, UnknownTableException,
      NoSuchObjectException, TException {
  org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, tblName);
  Map<String, String> map = originTable.getParameters();
  String indexTableName = INDEX_PREFIX + indexName;
  if(!map.containsKey(indexTableName)) {
    throw new NoSuchObjectException("can not find index: " + indexName);
  }
  Table indexTableObject = stringToCatalogTable(map.get(indexTableName));
  return CatalogToHiveConverter.convertTableObjectToIndex(indexTableObject);
}
 

@Override
public List<Index> listIndexes(String db_name, String tbl_name, short max) throws NoSuchObjectException, MetaException,
      TException {
  // In current hive implementation, it ignores fields "max"
  // https://github.com/apache/hive/blob/rel/release-2.3.0/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java#L3867-L3899
  return glueMetastoreClientDelegate.listIndexes(db_name, tbl_name);
}
 

@Override
public List<Index> getIndexes(String dbName, String origTableName, int max)
    throws MetaException {
  if (filterTables(dbName, Lists.newArrayList(origTableName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, origTableName));
  }
  return super.getIndexes(dbName, origTableName, max);
}
 

@Override
public List<Index> listIndexes(String db_name, String tbl_name, short max) throws NoSuchObjectException, MetaException,
      TException {
  // In current hive implementation, it ignores fields "max"
  // https://github.com/apache/hive/blob/rel/release-2.3.0/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java#L3867-L3899
  return glueMetastoreClientDelegate.listIndexes(db_name, tbl_name);
}
 

@Override
public Index getIndex(String dbName, String origTableName, String indexName)
    throws MetaException {
  if (filterTables(dbName, Lists.newArrayList(origTableName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, origTableName));
  }
  return super.getIndex(dbName, origTableName, indexName);
}
 

@Override
public List<Index> getIndexes(String dbName, String origTableName, int max)
    throws MetaException {
  if (filterTables(dbName, Lists.newArrayList(origTableName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, origTableName));
  }
  return super.getIndexes(dbName, origTableName, max);
}
 

@Test
public void get_indexes() throws TException {
  List<Index> indexList = Collections.singletonList(new Index());
  List<Index> outboundIndexList = Collections.singletonList(new Index());
  when(primaryMapping.transformOutboundIndexes(indexList)).thenReturn(outboundIndexList);
  when(primaryClient.get_indexes(DB_P, "table", (short) 2)).thenReturn(indexList);

  List<Index> result = handler.get_indexes(DB_P, "table", (short) 2);
  assertThat(result, is(outboundIndexList));
}
 

@Test
public void get_index_by_name() throws TException {
  Index index = new Index();
  Index outboundIndex = new Index();
  when(primaryClient.get_index_by_name(DB_P, "table", "index")).thenReturn(index);
  when(primaryMapping.transformOutboundIndex(index)).thenReturn(outboundIndex);
  Index result = handler.get_index_by_name(DB_P, "table", "index");
  assertThat(result, is(outboundIndex));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Index add_index(Index new_index, Table index_table)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_index.getDbName());
  mapping.checkWritePermissions(index_table.getDbName());
  Index result = mapping
      .getClient()
      .add_index(mapping.transformInboundIndex(new_index), mapping.transformInboundTable(index_table));
  return mapping.transformOutboundIndex(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void alter_index(String dbname, String base_tbl_name, String idx_name, Index new_idx)
    throws InvalidOperationException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(dbname);
  mapping.checkWritePermissions(new_idx.getDbName());
  mapping
      .getClient()
      .alter_index(mapping.transformInboundDatabaseName(dbname), base_tbl_name, idx_name,
          mapping.transformInboundIndex(new_idx));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Index get_index_by_name(String db_name, String tbl_name, String index_name)
    throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  return mapping
      .transformOutboundIndex(
          mapping.getClient().get_index_by_name(mapping.transformInboundDatabaseName(db_name), tbl_name, index_name));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<Index> get_indexes(String db_name, String tbl_name, short max_indexes)
    throws NoSuchObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  List<Index> indexes = mapping
      .getClient()
      .get_indexes(mapping.transformInboundDatabaseName(db_name), tbl_name, max_indexes);
  return mapping.transformOutboundIndexes(indexes);
}
 

@Test
public void alter_index() throws TException {
  Index newIndex = new Index();
  newIndex.setDbName(DB_P);
  Index inboundIndex = new Index();
  when(primaryMapping.transformInboundIndex(newIndex)).thenReturn(inboundIndex);

  handler.alter_index(DB_P, "table", "index", newIndex);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_index(DB_P, "table", "index", inboundIndex);
}
 

@Test
public void transformOutboundIndexes() throws Exception {
  List<Index> indexes = new ArrayList<>();
  indexes.add(index);
  List<Index> result = databaseMapping.transformOutboundIndexes(indexes);
  assertThat(result, is(sameInstance(indexes)));
  Index resultIndex = result.get(0);
  assertThat(resultIndex, is(sameInstance(index)));
  assertThat(resultIndex.getDbName(), is(OUT_DB_NAME));
}
 

@Before
public void setUp() {
  databaseMapping = new DatabaseMappingImpl(metastoreMapping, queryMapping);
  database = new Database();
  database.setName(DB_NAME);
  partition = new Partition();
  partition.setDbName(DB_NAME);
  partitions = Lists.newArrayList(partition);
  index = new Index();
  index.setDbName(DB_NAME);
  hiveObjectRef = new HiveObjectRef();
  hiveObjectRef.setDbName(DB_NAME);
  hiveObjectRef.setObjectType(HiveObjectType.DATABASE);
  hiveObjectRef.setObjectName(DB_NAME);
  hiveObjectPrivileges = new ArrayList<>();
  HiveObjectPrivilege hiveObjectPrivilege = new HiveObjectPrivilege();
  hiveObjectPrivilege.setHiveObject(hiveObjectRef);
  hiveObjectPrivileges.add(hiveObjectPrivilege);
  partitionSpec = new PartitionSpec();
  partitionSpec.setDbName(DB_NAME);
  when(metastoreMapping.transformInboundDatabaseName(anyString())).thenReturn(IN_DB_NAME);
  when(metastoreMapping.transformOutboundDatabaseName(anyString())).thenReturn(OUT_DB_NAME);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_EXPANDED_TEXT))
      .thenReturn(VIEW_EXPANDED_TEXT_TRANSFORMED);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_ORIGINAL_TEXT))
      .thenReturn(VIEW_ORIGINAL_TEXT_TRANSFORMED);
}