org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestObjects.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up 
public static org.apache.hadoop.hive.metastore.api.ColumnStatisticsData getHiveDecimalColumnStatsData() {
  DecimalColumnStatsData statsData = new DecimalColumnStatsData();
  org.apache.hadoop.hive.metastore.api.Decimal highValue = new org.apache.hadoop.hive.metastore.api.Decimal();
  highValue.setScale((short) 1);
  highValue.setUnscaled(BigInteger.valueOf(1234L).toByteArray());
  statsData.setHighValue(highValue);
  org.apache.hadoop.hive.metastore.api.Decimal lowValue = new org.apache.hadoop.hive.metastore.api.Decimal();
  lowValue.setScale((short) 4);
  lowValue.setUnscaled(BigInteger.valueOf(5678L).toByteArray());
  statsData.setLowValue(lowValue);
  statsData.setNumDVs(12L);
  statsData.setNumNulls(56L);
  org.apache.hadoop.hive.metastore.api.ColumnStatisticsData  statsWrapper =
          new org.apache.hadoop.hive.metastore.api.ColumnStatisticsData();
  statsWrapper.setDecimalStats(statsData);
  return statsWrapper;
}
Example #2
Source File: TestThriftMetastoreUtil.java    From presto with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDecimalStatsToColumnStatistics()
{
    DecimalColumnStatsData decimalColumnStatsData = new DecimalColumnStatsData();
    BigDecimal low = new BigDecimal("0");
    decimalColumnStatsData.setLowValue(toMetastoreDecimal(low));
    BigDecimal high = new BigDecimal("100");
    decimalColumnStatsData.setHighValue(toMetastoreDecimal(high));
    decimalColumnStatsData.setNumNulls(1);
    decimalColumnStatsData.setNumDVs(20);
    ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DECIMAL_TYPE_NAME, decimalStats(decimalColumnStatsData));
    HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000));

    assertEquals(actual.getIntegerStatistics(), Optional.empty());
    assertEquals(actual.getDoubleStatistics(), Optional.empty());
    assertEquals(actual.getDecimalStatistics(), Optional.of(new DecimalStatistics(Optional.of(low), Optional.of(high))));
    assertEquals(actual.getDateStatistics(), Optional.empty());
    assertEquals(actual.getBooleanStatistics(), Optional.empty());
    assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getNullsCount(), OptionalLong.of(1));
    assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19));
}
Example #3
Source File: TestThriftMetastoreUtil.java    From presto with Apache License 2.0 6 votes vote down vote up 
@Test
public void testEmptyDecimalStatsToColumnStatistics()
{
    DecimalColumnStatsData emptyDecimalColumnStatsData = new DecimalColumnStatsData();
    ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DECIMAL_TYPE_NAME, decimalStats(emptyDecimalColumnStatsData));
    HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.empty());

    assertEquals(actual.getIntegerStatistics(), Optional.empty());
    assertEquals(actual.getDoubleStatistics(), Optional.empty());
    assertEquals(actual.getDecimalStatistics(), Optional.of(new DecimalStatistics(Optional.empty(), Optional.empty())));
    assertEquals(actual.getDateStatistics(), Optional.empty());
    assertEquals(actual.getBooleanStatistics(), Optional.empty());
    assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getNullsCount(), OptionalLong.empty());
    assertEquals(actual.getDistinctValuesCount(), OptionalLong.empty());
}
Example #4
Source File: ThriftMetastoreUtil.java    From presto with Apache License 2.0 5 votes vote down vote up 
private static ColumnStatisticsObj createDecimalStatistics(String columnName, HiveType columnType, HiveColumnStatistics statistics)
{
    DecimalColumnStatsData data = new DecimalColumnStatsData();
    statistics.getDecimalStatistics().ifPresent(decimalStatistics -> {
        decimalStatistics.getMin().ifPresent(value -> data.setLowValue(toMetastoreDecimal(value)));
        decimalStatistics.getMax().ifPresent(value -> data.setHighValue(toMetastoreDecimal(value)));
    });
    statistics.getNullsCount().ifPresent(data::setNumNulls);
    toMetastoreDistinctValuesCount(statistics.getDistinctValuesCount(), statistics.getNullsCount()).ifPresent(data::setNumDVs);
    return new ColumnStatisticsObj(columnName, columnType.toString(), decimalStats(data));
}
Example #5
Source File: HiveStatsUtil.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Create Flink ColumnStats from Hive ColumnStatisticsData.
 */
private static CatalogColumnStatisticsDataBase createTableColumnStats(DataType colType, ColumnStatisticsData stats, String hiveVersion) {
	HiveShim hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	if (stats.isSetBinaryStats()) {
		BinaryColumnStatsData binaryStats = stats.getBinaryStats();
		return new CatalogColumnStatisticsDataBinary(
				binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null,
				binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null,
				binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null);
	} else if (stats.isSetBooleanStats()) {
		BooleanColumnStatsData booleanStats = stats.getBooleanStats();
		return new CatalogColumnStatisticsDataBoolean(
				booleanStats.isSetNumTrues() ? booleanStats.getNumTrues() : null,
				booleanStats.isSetNumFalses() ? booleanStats.getNumFalses() : null,
				booleanStats.isSetNumNulls() ? booleanStats.getNumNulls() : null);
	} else if (hiveShim.isDateStats(stats)) {
		return hiveShim.toFlinkDateColStats(stats);
	} else if (stats.isSetDoubleStats()) {
			DoubleColumnStatsData doubleStats = stats.getDoubleStats();
			return new CatalogColumnStatisticsDataDouble(
					doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null,
					doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null,
					doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null,
					doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null);
	} else if (stats.isSetLongStats()) {
			LongColumnStatsData longColStats = stats.getLongStats();
			return new CatalogColumnStatisticsDataLong(
					longColStats.isSetLowValue() ? longColStats.getLowValue() : null,
					longColStats.isSetHighValue() ? longColStats.getHighValue() : null,
					longColStats.isSetNumDVs() ? longColStats.getNumDVs() : null,
					longColStats.isSetNumNulls() ? longColStats.getNumNulls() : null);
	} else if (stats.isSetStringStats()) {
		StringColumnStatsData stringStats = stats.getStringStats();
		return new CatalogColumnStatisticsDataString(
				stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null,
				stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null,
				stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null,
				stringStats.isSetNumDVs() ? stringStats.getNumNulls() : null);
	} else if (stats.isSetDecimalStats()) {
		DecimalColumnStatsData decimalStats = stats.getDecimalStats();
		// for now, just return CatalogColumnStatisticsDataDouble for decimal columns
		Double max = null;
		if (decimalStats.isSetHighValue()) {
			max = toHiveDecimal(decimalStats.getHighValue()).doubleValue();
		}
		Double min = null;
		if (decimalStats.isSetLowValue()) {
			min = toHiveDecimal(decimalStats.getLowValue()).doubleValue();
		}
		Long ndv = decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null;
		Long nullCount = decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null;
		return new CatalogColumnStatisticsDataDouble(min, max, ndv, nullCount);
	} else {
		LOG.warn("Flink does not support converting ColumnStatisticsData '{}' for Hive column type '{}' yet.", stats, colType);
		return null;
	}
}