Java Code Examples for org.HdrHistogram.Histogram#recordValue()

@Test
public void testEvalSoft90th() {
    SoftLatencyEvaluator latencyEvaluator = new SoftLatencyEvaluator(90, 90);

    Histogram histogram = new Histogram(1);

    for (int i = 0; i <= 100; i++) {
        histogram.recordValue(i);
        latencyEvaluator.record(histogram);

        /*
         It passes the threshold for the 90th percentile with 90 as the max value when the counter
         reaches 97. Subsequent calls to the evaluator should always be false after that
         */
        if (i <= 96) {
            assertTrue("Failed at record " + i, latencyEvaluator.eval());
        }
        else {
            assertFalse("Failed at record " + i, latencyEvaluator.eval());
        }
    }

    assertEquals(90.0, latencyEvaluator.getMaxValue(), 0.1);
}
 

@Test
public void testConvertHistogramToString() {
  Histogram histogram = new Histogram(1, 100, 3);
  histogram.recordValue(3);
  histogram.recordValue(25);
  histogram.recordValue(25);
  histogram.recordValue(92);
  String histogramString = KafkaExtractorStatsTracker.convertHistogramToString(histogram);

  HistogramLogReader logReader = new HistogramLogReader(new ByteArrayInputStream(histogramString.getBytes(
      Charsets.UTF_8)));
  Histogram histogram1 = (Histogram) logReader.nextIntervalHistogram();
  Assert.assertEquals(histogram1.getTotalCount(), 4);
  Assert.assertEquals(histogram1.getMaxValue(), 92);
  Assert.assertEquals(histogram1.getCountAtValue(25), 2);
  Assert.assertEquals(histogram1.getCountAtValue(3), 1);
  Assert.assertEquals(histogram1.getCountAtValue(92), 1);
}
 

@Test
void saveToFileCreatesNewFileByIncrementExistingMaxIndex(final @TempDir Path tempDir) throws IOException
{
    Files.createFile(tempDir.resolve("another_one-13.hdr"));
    Files.createFile(tempDir.resolve("another_one" + AGGREGATE_FILE_SUFFIX));

    final Histogram histogram = new Histogram(2);
    histogram.recordValue(2);
    histogram.recordValue(4);

    final PersistedHistogram persistedHistogram = new PersistedHistogram(histogram.copy());

    final Path file = persistedHistogram.saveToFile(tempDir, "another_one");

    assertNotNull(file);
    assertTrue(Files.exists(file));
    assertEquals("another_one-14.hdr", file.getFileName().toString());
    final Histogram savedHistogram = readHistogram(file);
    assertEquals(histogram, savedHistogram);
}
 

@Test
void saveToFileCreatesNewFileWithIndexZero(final @TempDir Path tempDir) throws IOException
{
    Files.createFile(tempDir.resolve("another-one-13.hdr"));

    final Histogram histogram = new Histogram(3);
    histogram.setStartTimeStamp(123456789);
    histogram.setEndTimeStamp(987654321);
    histogram.recordValue(100);
    histogram.recordValue(1000);
    histogram.recordValue(250);

    final PersistedHistogram persistedHistogram = new PersistedHistogram(histogram.copy());

    final Path file = persistedHistogram.saveToFile(tempDir, "test-histogram");

    assertNotNull(file);
    assertTrue(Files.exists(file));
    assertEquals("test-histogram-0.hdr", file.getFileName().toString());
    final Histogram savedHistogram = readHistogram(file);
    assertEquals(histogram, savedHistogram);
    assertEquals(histogram.getStartTimeStamp(), savedHistogram.getStartTimeStamp());
    assertEquals(histogram.getEndTimeStamp(), savedHistogram.getEndTimeStamp());
}
 

private void exchangeMessage(
    final SocketChannel socketChannel,
    final TestRequestEncoder testRequest,
    final HeaderEncoder header,
    final int index,
    final Histogram histogram)
    throws IOException
{
    header.msgSeqNum(index + 2);
    timestampEncoder.encode(System.currentTimeMillis());

    final long result = testRequest.encode(writeFlyweight, 0);

    final long sendingTime = System.nanoTime();
    write(socketChannel, result);

    read(socketChannel);
    final long returnTime = System.nanoTime();
    histogram.recordValue(returnTime - sendingTime);
}
 

private Histogram createHistogram(final long startTimeMs, final long endTimeMs, final long... values)
{
    final Histogram histogram = new Histogram(3);
    histogram.setStartTimeStamp(startTimeMs);
    histogram.setEndTimeStamp(endTimeMs);

    for (final long value : values)
    {
        histogram.recordValue(value);
    }

    return histogram;
}
 

@Test
void saveToFileThrowsIOExceptionIfSaveFails(final @TempDir Path tempDir) throws IOException
{
    final Path rootFile = Files.createFile(tempDir.resolve("my.txt"));

    final Histogram histogram = new Histogram(2);
    histogram.recordValue(2);
    histogram.recordValue(4);

    final PersistedHistogram persistedHistogram = new PersistedHistogram(histogram.copy());

    assertThrows(IOException.class, () -> persistedHistogram.saveToFile(rootFile, "ignore"));
}
 

private Histogram createEquivalentHistogram() {
    Histogram histogram = new Histogram(2);
    for (int i = 1; i <= 100000; i++) {
        histogram.recordValue(i);
    }
    return histogram;
}
 

public void testEvalHard(LatencyEvaluator latencyEvaluator) {
    Histogram histogram = new Histogram(1);

    for (int i = 0; i <= 99; i++) {
        histogram.recordValue(i);
        latencyEvaluator.record(histogram);
        assertTrue(latencyEvaluator.eval());
    }

    histogram.recordValue(100);
    latencyEvaluator.record(histogram);
    assertFalse(latencyEvaluator.eval());

    assertEquals(100.0d, latencyEvaluator.getMaxValue(), 0.1);
}
 

@EnsuresNonNull("histogram")
private void convertValuesToHistogram() {
    // tracking nanoseconds, but only at microsecond precision (to save histogram space)
    histogram = new Histogram(1000, 2000, HISTOGRAM_SIGNIFICANT_DIGITS);
    histogram.setAutoResize(true);
    for (int i = 0; i < size; i++) {
        histogram.recordValue(values[i]);
    }
    values = new long[0];
}
 

private static void runLatencyTest(final int samples, final IPerfTestRunner logger, final Histogram serviceTmHist,
        final Histogram responseTmHist, final Pacer pacer) {

    for (int i = 0; i < samples; i++) {
        final long expectedStartTimeNanos = pacer.expectedNextOperationNanoTime();
        pacer.acquire(1);
        final long actualStartTime = System.nanoTime();
        logger.log(LATENCY_MSG);
        final long doneTime = System.nanoTime();
        serviceTmHist.recordValue(doneTime - actualStartTime);
        responseTmHist.recordValue(doneTime - expectedStartTimeNanos);
    }
}
 

public void run()
{
    final Histogram histogram = new Histogram(3);
    final long scaleToMicros = TimeUnit.MICROSECONDS.toNanos(1);
    final SocketChannel socketChannel = this.socketChannel;
    final MutableAsciiBuffer readFlyweight = LatencyUnderLoadBenchmarkClient.this.readFlyweight;
    final long[] sendTimes = LatencyUnderLoadBenchmarkClient.this.sendTimes;

    while (true)
    {
        final long startTime = System.currentTimeMillis();
        int lastMessagesReceived = 0;
        while (lastMessagesReceived < MESSAGES_EXCHANGED)
        {
            try
            {
                final int length = read(socketChannel);
                final long time = System.nanoTime();
                final int received = scanForReceivesMessages(readFlyweight, length);
                for (int j = 0; j < received; j++)
                {
                    final long duration = time - sendTimes[lastMessagesReceived + j];
                    histogram.recordValue(duration);
                }
                lastMessagesReceived += received;
            }
            catch (final IOException ex)
            {
                ex.printStackTrace();
                System.exit(-1);
            }
        }

        printThroughput(startTime, MESSAGES_EXCHANGED);
        HistogramLogReader.prettyPrint(
            System.currentTimeMillis(), histogram, "Benchmark", scaleToMicros);

        histogram.reset();
        await();
    }
}
 

public static void main(String[] args) throws IOException {
  // set up house-keeping
  ObjectMapper mapper = new ObjectMapper();
  Histogram stats = new Histogram(1, 10000000, 2);
  Histogram global = new Histogram(1, 10000000, 2);

  final String TOPIC_FAST_MESSAGES = "/sample-stream:fast-messages";
  final String TOPIC_SUMMARY_MARKERS = "/sample-stream:summary-markers";


  Table fastMessagesTable = getTable("/apps/fast-messages");

  // and the consumer
  KafkaConsumer<String, String> consumer;
  try (InputStream props = Resources.getResource("consumer.props").openStream()) {
    Properties properties = new Properties();
    properties.load(props);
    // use a new group id for the dbconsumer
    if (properties.getProperty("group.id") == null) {
      properties.setProperty("group.id", "group-" + new Random().nextInt(100000));
    } else {
      String groupId = properties.getProperty("group.id");
      properties.setProperty("group.id", "db-" + groupId);
    }

    consumer = new KafkaConsumer<>(properties);
  }
  consumer.subscribe(Arrays.asList(TOPIC_FAST_MESSAGES, TOPIC_SUMMARY_MARKERS));
  int timeouts = 0;

  //noinspection InfiniteLoopStatement
  while (true) {
    // read records with a short timeout. If we time out, we don't really care.
    ConsumerRecords<String, String> records = consumer.poll(200);
    if (records.count() == 0) {
      timeouts++;
    } else {
      System.out.printf("Got %d records after %d timeouts\n", records.count(), timeouts);
      timeouts = 0;
    }
    for (ConsumerRecord<String, String> record : records) {
      switch (record.topic()) {
        case TOPIC_FAST_MESSAGES:
          // the send time is encoded inside the message
          JsonNode msg = mapper.readTree(record.value());
          switch (msg.get("type").asText()) {
            case "test":
              // create a Document and set an _id, in this case the message number (document will be updated each time)
              Document messageDocument = MapRDB.newDocument(msg);
              messageDocument.setId( Integer.toString(messageDocument.getInt("k")));
              fastMessagesTable.insertOrReplace( messageDocument );

              long latency = (long) ((System.nanoTime() * 1e-9 - msg.get("t").asDouble()) * 1000);
              stats.recordValue(latency);
              global.recordValue(latency);
              break;
            case "marker":
              // whenever we get a marker message, we should dump out the stats
              // note that the number of fast messages won't necessarily be quite constant
              System.out.printf("%d messages received in period, latency(min, max, avg, 99%%) = %d, %d, %.1f, %d (ms)\n",
                      stats.getTotalCount(),
                      stats.getValueAtPercentile(0), stats.getValueAtPercentile(100),
                      stats.getMean(), stats.getValueAtPercentile(99));
              System.out.printf("%d messages received overall, latency(min, max, avg, 99%%) = %d, %d, %.1f, %d (ms)\n",
                      global.getTotalCount(),
                      global.getValueAtPercentile(0), global.getValueAtPercentile(100),
                      global.getMean(), global.getValueAtPercentile(99));
              stats.reset();
              break;
            default:
              throw new IllegalArgumentException("Illegal message type: " + msg.get("type"));
          }
          break;
        case TOPIC_SUMMARY_MARKERS:
          break;
        default:
          throw new IllegalStateException("Shouldn't be possible to get message on topic " + record.topic());
      }
    }
  }
}
 

public static void main(String[] args) throws IOException {
    // set up house-keeping
    ObjectMapper mapper = new ObjectMapper();
    Histogram stats = new Histogram(1, 10000000, 2);
    Histogram global = new Histogram(1, 10000000, 2);

    final String TOPIC_FAST_MESSAGES = "/sample-stream:fast-messages";
    final String TOPIC_SUMMARY_MARKERS = "/sample-stream:summary-markers";

    // and the consumer
    KafkaConsumer<String, String> consumer;
    try (InputStream props = Resources.getResource("consumer.props").openStream()) {
        Properties properties = new Properties();
        properties.load(props);
        if (properties.getProperty("group.id") == null) {
            properties.setProperty("group.id", "group-" + new Random().nextInt(100000));
        }

        consumer = new KafkaConsumer<>(properties);
    }
    consumer.subscribe(Arrays.asList(TOPIC_FAST_MESSAGES, TOPIC_SUMMARY_MARKERS));
    int timeouts = 0;
    //noinspection InfiniteLoopStatement
    while (true) {
        // read records with a short timeout. If we time out, we don't really care.
        ConsumerRecords<String, String> records = consumer.poll(200);
        if (records.count() == 0) {
            timeouts++;
        } else {
            System.out.printf("Got %d records after %d timeouts\n", records.count(), timeouts);
            timeouts = 0;
        }
        for (ConsumerRecord<String, String> record : records) {
            switch (record.topic()) {
                case TOPIC_FAST_MESSAGES:
                    // the send time is encoded inside the message
                    JsonNode msg = mapper.readTree(record.value());
                    switch (msg.get("type").asText()) {
                        case "test":
                            long latency = (long) ((System.nanoTime() * 1e-9 - msg.get("t").asDouble()) * 1000);
                            stats.recordValue(latency);
                            global.recordValue(latency);
                            break;
                        case "marker":
                            // whenever we get a marker message, we should dump out the stats
                            // note that the number of fast messages won't necessarily be quite constant
                            System.out.printf("%d messages received in period, latency(min, max, avg, 99%%) = %d, %d, %.1f, %d (ms)\n",
                                    stats.getTotalCount(),
                                    stats.getValueAtPercentile(0), stats.getValueAtPercentile(100),
                                    stats.getMean(), stats.getValueAtPercentile(99));
                            System.out.printf("%d messages received overall, latency(min, max, avg, 99%%) = %d, %d, %.1f, %d (ms)\n",
                                    global.getTotalCount(),
                                    global.getValueAtPercentile(0), global.getValueAtPercentile(100),
                                    global.getMean(), global.getValueAtPercentile(99));

                            stats.reset();
                            break;
                        default:
                            throw new IllegalArgumentException("Illegal message type: " + msg.get("type"));
                    }
                    break;
                case TOPIC_SUMMARY_MARKERS:
                    break;
                default:
                    throw new IllegalStateException("Shouldn't be possible to get message on topic " + record.topic());
            }
        }
    }
}
 

public static void main(String[] args) throws IOException {
    // set up house-keeping
    ObjectMapper mapper = new ObjectMapper();
    Histogram stats = new Histogram(1, 10000000, 2);
    Histogram global = new Histogram(1, 10000000, 2);

    // and the consumer
    KafkaConsumer<String, String> consumer;
    try (InputStream props = Resources.getResource("consumer.props").openStream()) {
        Properties properties = new Properties();
        properties.load(props);
        if (properties.getProperty("group.id") == null) {
            properties.setProperty("group.id", "group-" + new Random().nextInt(100000));
        }
        consumer = new KafkaConsumer<>(properties);
    }
    consumer.subscribe(Arrays.asList("fast-messages", "summary-markers"));
    int timeouts = 0;
    //noinspection InfiniteLoopStatement
    while (true) {
        // read records with a short timeout. If we time out, we don't really care.
        ConsumerRecords<String, String> records = consumer.poll(200);
        if (records.count() == 0) {
            timeouts++;
        } else {
            System.out.printf("Got %d records after %d timeouts\n", records.count(), timeouts);
            timeouts = 0;
        }
        for (ConsumerRecord<String, String> record : records) {
            switch (record.topic()) {
                case "fast-messages":
                    // the send time is encoded inside the message
                    JsonNode msg = mapper.readTree(record.value());
                    switch (msg.get("type").asText()) {
                        case "test":
                            long latency = (long) ((System.nanoTime() * 1e-9 - msg.get("t").asDouble()) * 1000);
                            stats.recordValue(latency);
                            global.recordValue(latency);
                            break;
                        case "marker":
                            // whenever we get a marker message, we should dump out the stats
                            // note that the number of fast messages won't necessarily be quite constant
                            System.out.printf("%d messages received in period, latency(min, max, avg, 99%%) = %d, %d, %.1f, %d (ms)\n",
                                    stats.getTotalCount(),
                                    stats.getValueAtPercentile(0), stats.getValueAtPercentile(100),
                                    stats.getMean(), stats.getValueAtPercentile(99));
                            System.out.printf("%d messages received overall, latency(min, max, avg, 99%%) = %d, %d, %.1f, %d (ms)\n",
                                    global.getTotalCount(),
                                    global.getValueAtPercentile(0), global.getValueAtPercentile(100),
                                    global.getMean(), global.getValueAtPercentile(99));

                            stats.reset();
                            break;
                        default:
                            throw new IllegalArgumentException("Illegal message type: " + msg.get("type"));
                    }
                    break;
                case "summary-markers":
                    break;
                default:
                    throw new IllegalStateException("Shouldn't be possible to get message on topic " + record.topic());
            }
        }
    }
}
 

private static void measureRoundTrip(
    final Histogram histogram,
    final ByteBuffer buffer,
    final DatagramChannel[] receiveChannels,
    final DatagramChannel writeChannel,
    final AtomicBoolean running)
    throws IOException
{
    for (int sequenceNumber = 0; sequenceNumber < Common.NUM_MESSAGES; sequenceNumber++)
    {
        final long timestampNs = System.nanoTime();

        buffer.clear();
        buffer.putLong(sequenceNumber);
        buffer.putLong(timestampNs);
        buffer.flip();

        writeChannel.write(buffer);

        buffer.clear();
        boolean available = false;
        while (!available)
        {
            if (!running.get())
            {
                return;
            }

            for (int i = receiveChannels.length - 1; i >= 0; i--)
            {
                if (null != receiveChannels[i].receive(buffer))
                {
                    available = true;
                    break;
                }
            }
        }

        final long receivedSequenceNumber = buffer.getLong(0);
        if (receivedSequenceNumber != sequenceNumber)
        {
            throw new IllegalStateException("Data Loss:" + sequenceNumber + " to " + receivedSequenceNumber);
        }

        final long durationNs = System.nanoTime() - buffer.getLong(SIZE_OF_LONG);
        histogram.recordValue(durationNs);
    }

    histogram.outputPercentileDistribution(System.out, 1000.0);
}
 

private void run() throws IOException
{
    final Histogram histogram = new Histogram(TimeUnit.SECONDS.toNanos(10), 3);
    final ByteBuffer buffer = ByteBuffer.allocateDirect(Configuration.MTU_LENGTH_DEFAULT);

    final DatagramChannel receiveChannel = DatagramChannel.open();
    Common.init(receiveChannel);
    receiveChannel.bind(new InetSocketAddress("localhost", Common.PONG_PORT));

    final DatagramChannel sendChannel = DatagramChannel.open();
    Common.init(sendChannel);

    final Selector selector = Selector.open();

    final IntSupplier handler =
        () ->
        {
            try
            {
                buffer.clear();
                receiveChannel.receive(buffer);

                final long receivedSequenceNumber = buffer.getLong(0);
                final long timestampNs = buffer.getLong(SIZE_OF_LONG);

                if (receivedSequenceNumber != sequenceNumber)
                {
                    throw new IllegalStateException(
                        "data Loss:" + sequenceNumber + " to " + receivedSequenceNumber);
                }

                final long durationNs = System.nanoTime() - timestampNs;
                histogram.recordValue(durationNs);
            }
            catch (final IOException ex)
            {
                ex.printStackTrace();
            }

            return 1;
        };

    receiveChannel.register(selector, OP_READ, handler);

    final AtomicBoolean running = new AtomicBoolean(true);
    SigInt.register(() -> running.set(false));

    while (running.get())
    {
        measureRoundTrip(histogram, SEND_ADDRESS, buffer, sendChannel, selector, running);

        histogram.reset();
        System.gc();
        LockSupport.parkNanos(1000 * 1000 * 1000);
    }
}
 

private static void measureRoundTrip(
    final Histogram histogram,
    final InetSocketAddress sendAddress,
    final ByteBuffer buffer,
    final DatagramChannel[] receiveChannels,
    final DatagramChannel sendChannel,
    final AtomicBoolean running)
    throws IOException
{
    for (int sequenceNumber = 0; sequenceNumber < Common.NUM_MESSAGES; sequenceNumber++)
    {
        final long timestampNs = System.nanoTime();

        buffer.clear();
        buffer.putLong(sequenceNumber);
        buffer.putLong(timestampNs);
        buffer.flip();

        sendChannel.send(buffer, sendAddress);

        buffer.clear();
        boolean available = false;
        while (!available)
        {
            if (!running.get())
            {
                return;
            }

            for (int i = receiveChannels.length - 1; i >= 0; i--)
            {
                if (null != receiveChannels[i].receive(buffer))
                {
                    available = true;
                    break;
                }
            }
        }

        final long receivedSequenceNumber = buffer.getLong(0);
        if (receivedSequenceNumber != sequenceNumber)
        {
            throw new IllegalStateException("Data Loss:" + sequenceNumber + " to " + receivedSequenceNumber);
        }

        final long durationNs = System.nanoTime() - buffer.getLong(BitUtil.SIZE_OF_LONG);
        histogram.recordValue(durationNs);
    }

    histogram.outputPercentileDistribution(System.out, 1000.0);
}
 

private void populateHistogram(Histogram original) {
    for (int i = 0; i < LATENCY_RECORD_COUNT; i++) {
        original.recordValue(random.nextInt(MAX_LATENCY));
    }
}