org.apache.hadoop.hive.metastore.api.ColumnStatistics Java Examples

@Test
public void transformInboundSetPartitionStatsRequest() throws Exception {
  SetPartitionsStatsRequest setPartitionsStatsRequest = new SetPartitionsStatsRequest();
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  setPartitionsStatsRequest.setColStats(Lists.newArrayList(columnStatistics));
  SetPartitionsStatsRequest result = databaseMapping
      .transformInboundSetPartitionStatsRequest(setPartitionsStatsRequest);
  assertThat(result, is(sameInstance(setPartitionsStatsRequest)));
  ColumnStatistics resultColStats = result.getColStats().get(0);
  assertThat(resultColStats, is(sameInstance(columnStatistics)));
  assertThat(resultColStats.getStatsDesc(), is(sameInstance(statsDesc)));
  assertThat(resultColStats.getStatsDesc().getDbName(), is(IN_DB_NAME));
}
 

private static ColumnStatistics createHiveColumnStatistics(
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		StorageDescriptor sd,
		ColumnStatisticsDesc desc,
		String hiveVersion) {
	List<ColumnStatisticsObj> colStatsList = new ArrayList<>();

	for (FieldSchema field : sd.getCols()) {
		String hiveColName = field.getName();
		String hiveColType = field.getType();
		CatalogColumnStatisticsDataBase flinkColStat = colStats.get(field.getName());
		if (null != flinkColStat) {
			ColumnStatisticsData statsData = getColumnStatisticsData(
					HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(hiveColType)),
					flinkColStat,
					hiveVersion);
			ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj(hiveColName, hiveColType, statsData);
			colStatsList.add(columnStatisticsObj);
		}
	}

	return new ColumnStatistics(desc, colStatsList);
}
 

private static ColumnStatistics createHiveColumnStatistics(
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		StorageDescriptor sd,
		ColumnStatisticsDesc desc) {
	List<ColumnStatisticsObj> colStatsList = new ArrayList<>();

	for (FieldSchema field : sd.getCols()) {
		String hiveColName = field.getName();
		String hiveColType = field.getType();
		CatalogColumnStatisticsDataBase flinkColStat = colStats.get(field.getName());
		if (null != flinkColStat) {
			ColumnStatisticsData statsData =
					getColumnStatisticsData(HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(hiveColType)), flinkColStat);
			ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj(hiveColName, hiveColType, statsData);
			colStatsList.add(columnStatisticsObj);
		}
	}

	return new ColumnStatistics(desc, colStatsList);
}
 

private PartitionsAndStatistics filterOnReplicatedPartitions(
    CloseableMetaStoreClient replicaClient,
    PartitionsAndStatistics sourcePartitionsAndStatistics,
    List<FieldSchema> partitionKeys)
  throws TException {
  Map<Partition, ColumnStatistics> statisticsByPartition = new LinkedHashMap<>();
  for (Partition partition : sourcePartitionsAndStatistics.getPartitions()) {
    try {
      replicaClient.getPartition(replicaDatabaseName, replicaTableName, partition.getValues());
      statisticsByPartition.put(partition, sourcePartitionsAndStatistics.getStatisticsForPartition(partition));
    } catch (NoSuchObjectException e) {
      LOG.debug("Partition {} doesn't exist, skipping it...", Warehouse.getQualifiedName(partition));
    }
  }
  return new PartitionsAndStatistics(partitionKeys, statisticsByPartition);
}
 

@Test
public void typical() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"), newFieldSchema("c"));
  Table table = newTable("t1", "db1", partitionKeys, newStorageDescriptor(new File("bla"), "col1"));
  List<Partition> partitions = Lists.newArrayList(newPartition(table, "b", "d"));
  statisticsPerPartitionName.put("a=b/c=d", columnStats);

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);
  List<String> expectedName = Lists.newArrayList("a=b/c=d");

  assertThat(partitionsAndStatistics.getPartitionNames(), is(expectedName));
  assertThat(partitionsAndStatistics.getPartitions(), is(partitions));
  assertThat(partitionsAndStatistics.getPartitionKeys(), is(partitionKeys));
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db1", "t1");
  statsDesc.setPartName("a=b/c=d");
  ColumnStatistics expectedStats = new ColumnStatistics(statsDesc, columnStats);
  assertThat(partitionsAndStatistics.getStatisticsForPartition(partitions.get(0)), is(expectedStats));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean update_table_column_statistics(ColumnStatistics stats_obj)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(stats_obj.getStatsDesc().getDbName());
  return mapping.getClient().update_table_column_statistics(mapping.transformInboundColumnStatistics(stats_obj));
}
 

@Test
public void transformOutboundColumnStatistics() throws Exception {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  ColumnStatistics result = databaseMapping.transformOutboundColumnStatistics(columnStatistics);
  assertThat(result, is(sameInstance(columnStatistics)));
  assertThat(result.getStatsDesc(), is(sameInstance(columnStatistics.getStatsDesc())));
  assertThat(result.getStatsDesc().getDbName(), is(OUT_DB_NAME));
}
 

@Test
public void transformInboundColumnStatistics() throws Exception {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  ColumnStatistics result = databaseMapping.transformInboundColumnStatistics(columnStatistics);
  assertThat(result, is(sameInstance(columnStatistics)));
  assertThat(result.getStatsDesc(), is(sameInstance(columnStatistics.getStatsDesc())));
  assertThat(result.getStatsDesc().getDbName(), is(IN_DB_NAME));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean set_aggr_stats_for(SetPartitionsStatsRequest request)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  if (!request.getColStats().isEmpty()) {
    DatabaseMapping mapping = databaseMappingService
        .databaseMapping(request.getColStats().get(0).getStatsDesc().getDbName());
    for (ColumnStatistics stats : request.getColStats()) {
      mapping.checkWritePermissions(stats.getStatsDesc().getDbName());
    }
    return mapping.getClient().set_aggr_stats_for(mapping.transformInboundSetPartitionStatsRequest(request));
  }
  return false;
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_partition_column_statistics(
    String db_name,
    String tbl_name,
    String part_name,
    String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_partition_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, part_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_table_column_statistics(String db_name, String tbl_name, String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_table_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean update_partition_column_statistics(ColumnStatistics stats_obj)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(stats_obj.getStatsDesc().getDbName());
  return mapping.getClient().update_partition_column_statistics(mapping.transformInboundColumnStatistics(stats_obj));
}
 

@Test
public void update_partition_column_statistics() throws TException {
  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(true, DB_P, "table");
  ColumnStatistics columnStatistics = new ColumnStatistics(columnStatisticsDesc, Collections.emptyList());
  ColumnStatistics inboundColumnStatistics = new ColumnStatistics();
  when(primaryMapping.transformInboundColumnStatistics(columnStatistics)).thenReturn(inboundColumnStatistics);
  when(primaryClient.update_partition_column_statistics(inboundColumnStatistics)).thenReturn(true);
  boolean result = handler.update_partition_column_statistics(columnStatistics);
  verify(primaryMapping).checkWritePermissions(DB_P);
  assertThat(result, is(true));
}
 

@Override
public SetPartitionsStatsRequest transformInboundSetPartitionStatsRequest(SetPartitionsStatsRequest request) {
  if (request.isSetColStats()) {
    for (ColumnStatistics stats : request.getColStats()) {
      transformInboundColumnStatistics(stats);
    }
  }
  return request;
}
 

@Override
public ColumnStatistics transformOutboundColumnStatistics(ColumnStatistics columnStatistics) {
  columnStatistics
      .getStatsDesc()
      .setDbName(metaStoreMapping.transformOutboundDatabaseName(columnStatistics.getStatsDesc().getDbName()));
  return columnStatistics;
}
 

@Override
public ColumnStatistics transform(ColumnStatistics columnStatistics) {
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "new_db", "new_table");
  statsDesc.setPartName("part=newPart");
  columnStatistics.setStatsDesc(statsDesc);
  return columnStatistics;
}
 

@Test
public void alteringExistingUnpartitionedReplicaTableWithNoStatsSucceeds() throws Exception {
  tableAndStatistics = new TableAndStatistics(sourceTable, null);
  existingReplicaTable.getParameters().put(REPLICATION_EVENT.parameterName(), "previousEventId");
  replica.updateMetadata(EVENT_ID, tableAndStatistics, DB_NAME, TABLE_NAME, mockReplicaLocationManager);
  verify(alterTableService).alterTable(eq(mockMetaStoreClient), eq(existingReplicaTable), any(Table.class));
  verify(mockMetaStoreClient, never()).updateTableColumnStatistics(any(ColumnStatistics.class));
  verify(mockReplicaLocationManager, never()).addCleanUpLocation(anyString(), any(Path.class));
}
 

@Test
public void update_table_column_statistics() throws TException {
  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(true, DB_P, "table");
  ColumnStatistics columnStatistics = new ColumnStatistics(columnStatisticsDesc, Collections.emptyList());
  ColumnStatistics inboundColumnStatistics = new ColumnStatistics();
  when(primaryMapping.transformInboundColumnStatistics(columnStatistics)).thenReturn(inboundColumnStatistics);
  when(primaryClient.update_table_column_statistics(inboundColumnStatistics)).thenReturn(true);
  boolean result = handler.update_table_column_statistics(columnStatistics);
  verify(primaryMapping).checkWritePermissions(DB_P);
  assertThat(result, is(true));
}
 

@Override
public void setPartitionColumnStatistics(String databaseName, String tableName, String partitionName, List<ColumnStatisticsObj> statistics)
        throws TException
{
    ColumnStatisticsDesc statisticsDescription = new ColumnStatisticsDesc(false, databaseName, tableName);
    statisticsDescription.setPartName(partitionName);
    ColumnStatistics request = new ColumnStatistics(statisticsDescription, statistics);
    client.update_partition_column_statistics(request);
}
 

@Test
public void get_table_column_statistics() throws TException {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatistics outboundColumnStatistics = new ColumnStatistics();
  when(primaryClient.get_table_column_statistics(DB_P, "table", "columnName")).thenReturn(columnStatistics);
  when(primaryMapping.transformOutboundColumnStatistics(columnStatistics)).thenReturn(outboundColumnStatistics);
  ColumnStatistics result = handler.get_table_column_statistics(DB_P, "table", "columnName");
  assertThat(result, is(outboundColumnStatistics));
}
 

@Test
public void get_partition_column_statistics() throws TException {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatistics outboundColumnStatistics = new ColumnStatistics();
  when(primaryClient.get_partition_column_statistics(DB_P, "table", "partitionName", "columnName"))
      .thenReturn(columnStatistics);
  when(primaryMapping.transformOutboundColumnStatistics(columnStatistics)).thenReturn(outboundColumnStatistics);
  ColumnStatistics result = handler.get_partition_column_statistics(DB_P, "table", "partitionName", "columnName");
  assertThat(result, is(outboundColumnStatistics));
}
 

@Test
public void set_aggr_stats_for() throws TException {
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, DB_P, "table");
  ColumnStatistics colStatistics = new ColumnStatistics(statsDesc, Collections.emptyList());
  List<ColumnStatistics> colStats = Collections.singletonList(colStatistics);
  SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats);
  SetPartitionsStatsRequest inboundRequest = new SetPartitionsStatsRequest();
  when(primaryMapping.transformInboundSetPartitionStatsRequest(request)).thenReturn(inboundRequest);
  when(primaryClient.set_aggr_stats_for(inboundRequest)).thenReturn(true);
  boolean result = handler.set_aggr_stats_for(request);
  assertThat(result, is(true));
}
 

/**
 * {@inheritDoc}
 */
@Override
public ColumnStatistics get_partition_column_statistics(
    final String dbName,
    final String tblName,
    final String partName,
    final String colName
) throws TException {
    throw unimplemented("get_partition_column_statistics", new Object[]{dbName, tblName, partName, colName});
}
 

/**
 * {@inheritDoc}
 */
@Override
public ColumnStatistics get_table_column_statistics(final String dbName, final String tblName,
                                                    final String colName)
    throws TException {
    throw unimplemented("get_table_column_statistics", new Object[]{dbName, tblName, colName});
}
 

/**
 * Create columnStatistics from the given Hive column stats of a hive table.
 */
public static ColumnStatistics createTableColumnStats(
		Table hiveTable,
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		String hiveVersion) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(true, hiveTable.getDbName(), hiveTable.getTableName());
	return createHiveColumnStatistics(colStats, hiveTable.getSd(), desc, hiveVersion);
}
 

/**
 * Create columnStatistics from the given Hive column stats of a hive partition.
 */
public static ColumnStatistics createPartitionColumnStats(
		Partition hivePartition,
		String partName,
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		String hiveVersion) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, hivePartition.getDbName(), hivePartition.getTableName());
	desc.setPartName(partName);
	return createHiveColumnStatistics(colStats, hivePartition.getSd(), desc, hiveVersion);
}
 

@Override
public ColumnStatistics getTableColumnStatistics(String dbName,
    String tableName, List<String> colNames) throws MetaException,
    NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tableName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tableName));
  }
  return super.getTableColumnStatistics(dbName, tableName, colNames);
}
 

@Override
public List<ColumnStatistics> getPartitionColumnStatistics(
    String dbName, String tblName, List<String> partNames,
    List<String> colNames) throws MetaException, NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tblName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tblName));
  }
  return super.getPartitionColumnStatistics(dbName, tblName, partNames,
      colNames);
}
 

@Override
public ColumnStatistics getTableColumnStatistics(String dbName,
    String tableName, List<String> colNames) throws MetaException,
    NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tableName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tableName));
  }
  return super.getTableColumnStatistics(dbName, tableName, colNames);
}
 

@Override
public List<ColumnStatistics> getPartitionColumnStatistics(
    String dbName, String tblName, List<String> partNames,
    List<String> colNames) throws MetaException, NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tblName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tblName));
  }
  return super.getPartitionColumnStatistics(dbName, tblName, partNames,
      colNames);
}