Java Code Examples for com.tdunning.math.stats.TDigest#asBytes()

The following examples show how to use com.tdunning.math.stats.TDigest#asBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SegmentGenerationWithBytesTypeTest.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up 
/**
 * Build Avro file containing serialized TDigest bytes.
 *
 * @param schema Schema of data (one fixed and one variable column)
 * @param _fixedExpected Serialized bytes of fixed length column are populated here
 * @param _varExpected Serialized bytes of variable length column are populated here
 * @throws IOException
 */
private void buildAvro(Schema schema, List<byte[]> _fixedExpected, List<byte[]> _varExpected)
    throws IOException {
  org.apache.avro.Schema avroSchema = AvroUtils.getAvroSchemaFromPinotSchema(schema);

  try (DataFileWriter<GenericData.Record> recordWriter = new DataFileWriter<>(new GenericDatumWriter<>(avroSchema))) {

    if (!new File(AVRO_DIR_NAME).mkdir()) {
      throw new RuntimeException("Unable to create test directory: " + AVRO_DIR_NAME);
    }

    recordWriter.create(avroSchema, new File(AVRO_DIR_NAME, AVRO_NAME));
    for (int i = 0; i < NUM_ROWS; i++) {
      GenericData.Record record = new GenericData.Record(avroSchema);

      TDigest tDigest = TDigest.createMergingDigest(PercentileTDigestAggregationFunction.DEFAULT_TDIGEST_COMPRESSION);
      tDigest.add(_random.nextDouble());

      ByteBuffer buffer = ByteBuffer.allocate(tDigest.byteSize());
      tDigest.asBytes(buffer);
      _fixedExpected.add(buffer.array());

      buffer.flip();
      record.put(FIXED_BYTES_UNSORTED_COLUMN, buffer);

      if (i % 2 == 0) {
        tDigest.add(_random.nextDouble());
      }

      buffer = ByteBuffer.allocate(tDigest.byteSize());
      tDigest.asBytes(buffer);
      _varExpected.add(buffer.array());

      buffer.flip();
      record.put(VARIABLE_BYTES_COLUMN, buffer);

      recordWriter.append(record);
    }
  }
}
Example 2
Source File: PercentileTDigestMVQueriesTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up 
@Override
protected void buildSegment()
    throws Exception {
  List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
  for (int i = 0; i < NUM_ROWS; i++) {
    HashMap<String, Object> valueMap = new HashMap<>();

    int numMultiValues = RANDOM.nextInt(MAX_NUM_MULTI_VALUES) + 1;
    Double[] values = new Double[numMultiValues];
    TDigest tDigest = TDigest.createMergingDigest(PercentileTDigestAggregationFunction.DEFAULT_TDIGEST_COMPRESSION);
    for (int j = 0; j < numMultiValues; j++) {
      double value = RANDOM.nextDouble() * VALUE_RANGE;
      values[j] = value;
      tDigest.add(value);
    }
    valueMap.put(DOUBLE_COLUMN, values);
    ByteBuffer byteBuffer = ByteBuffer.allocate(tDigest.byteSize());
    tDigest.asBytes(byteBuffer);
    valueMap.put(TDIGEST_COLUMN, byteBuffer.array());

    String group = GROUPS[RANDOM.nextInt(GROUPS.length)];
    valueMap.put(GROUP_BY_COLUMN, group);

    GenericRow genericRow = new GenericRow();
    genericRow.init(valueMap);
    rows.add(genericRow);
  }

  Schema schema = new Schema();
  schema.addField(new DimensionFieldSpec(DOUBLE_COLUMN, FieldSpec.DataType.DOUBLE, false));
  schema.addField(new MetricFieldSpec(TDIGEST_COLUMN, FieldSpec.DataType.BYTES));
  schema.addField(new DimensionFieldSpec(GROUP_BY_COLUMN, FieldSpec.DataType.STRING, true));
  TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();

  SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
  config.setOutDir(INDEX_DIR.getPath());
  config.setTableName(TABLE_NAME);
  config.setSegmentName(SEGMENT_NAME);
  config.setRawIndexCreationColumns(Collections.singletonList(TDIGEST_COLUMN));

  SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
  try (RecordReader recordReader = new GenericRowRecordReader(rows)) {
    driver.init(config, recordReader);
    driver.build();
  }
}
Example 3
Source File: PercentileTDigestQueriesTest.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up 
protected void buildSegment()
    throws Exception {
  List<GenericRow> rows = new ArrayList<>(NUM_ROWS);
  for (int i = 0; i < NUM_ROWS; i++) {
    HashMap<String, Object> valueMap = new HashMap<>();

    double value = RANDOM.nextDouble() * VALUE_RANGE;
    valueMap.put(DOUBLE_COLUMN, value);

    TDigest tDigest = TDigest.createMergingDigest(PercentileTDigestAggregationFunction.DEFAULT_TDIGEST_COMPRESSION);
    tDigest.add(value);
    ByteBuffer byteBuffer = ByteBuffer.allocate(tDigest.byteSize());
    tDigest.asBytes(byteBuffer);
    valueMap.put(TDIGEST_COLUMN, byteBuffer.array());

    String group = GROUPS[RANDOM.nextInt(GROUPS.length)];
    valueMap.put(GROUP_BY_COLUMN, group);

    GenericRow genericRow = new GenericRow();
    genericRow.init(valueMap);
    rows.add(genericRow);
  }

  Schema schema = new Schema();
  schema.addField(new MetricFieldSpec(DOUBLE_COLUMN, FieldSpec.DataType.DOUBLE));
  schema.addField(new MetricFieldSpec(TDIGEST_COLUMN, FieldSpec.DataType.BYTES));
  schema.addField(new DimensionFieldSpec(GROUP_BY_COLUMN, FieldSpec.DataType.STRING, true));
  TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME).build();

  SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema);
  config.setOutDir(INDEX_DIR.getPath());
  config.setTableName(TABLE_NAME);
  config.setSegmentName(SEGMENT_NAME);
  config.setRawIndexCreationColumns(Collections.singletonList(TDIGEST_COLUMN));

  SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
  try (RecordReader recordReader = new GenericRowRecordReader(rows)) {
    driver.init(config, recordReader);
    driver.build();
  }
}