org.openjdk.jmh.results.RunResult Java Examples

/**
     * Run without overriding any parameters
     */
    public Collection<RunResult> run() throws RunnerException {
        Options opt = new OptionsBuilder()
                .include(".*" + benchmarkClass.getSimpleName() + ".*")
                .jvmArgsPrepend("-Xms2g", "-Xmx2g")
                .jvmArgsAppend(properties.toArray(new String[properties.size()]))
                .verbosity(VerboseMode.NORMAL)
                .warmupIterations(1)
                .warmupTime(TimeValue.seconds(10))
                .measurementIterations(2)
                .measurementTime(TimeValue.seconds(10))
                .shouldDoGC(true)
                .forks(2)
                .threads(1)
//              .mode(Mode.AverageTime)
                .addProfiler(StackProfiler.class)
                .build();
        return new Runner(opt).run();
    }
 

/**
     * Run and override the 'exampleName' param with the passed fileName
     */
    public Collection<RunResult> run(String fileName) throws RunnerException {
        Options opt = new OptionsBuilder()
                .include(".*" + benchmarkClass.getSimpleName() + ".*")
                .jvmArgsPrepend("-Xms4g", "-Xmx4g")
                .jvmArgsAppend(properties.toArray(new String[properties.size()]))
                .verbosity(VerboseMode.NORMAL)
                .warmupIterations(1)
                .warmupTime(TimeValue.seconds(10))
                .measurementIterations(2)
                .measurementTime(TimeValue.seconds(10))
                .shouldDoGC(false)
                .forks(1)
//              .mode(Mode.AverageTime)
                .addProfiler(StackProfiler.class)
                .param("exampleName", fileName)
                .build();
        return new Runner(opt).run();
    }
 

/**
 * Run the benchmark with all the examples in BenchmarkUtil.SPEC_EXAMPLE_NAMES
 */
public Collection<RunResult> runAll() throws RunnerException {
    Options opt = new OptionsBuilder()
            .include(".*" + benchmarkClass.getSimpleName() + ".*")
            .jvmArgsPrepend("-Xms4g", "-Xmx4g")
            .jvmArgsAppend(properties.toArray(new String[properties.size()]))
            .verbosity(VerboseMode.NORMAL)
            .warmupIterations(1)
            .warmupTime(TimeValue.seconds(10))
            .measurementIterations(1)
            .measurementTime(TimeValue.seconds(10))
            .shouldDoGC(true)
            .forks(1)
            .output("results.txt")
            .mode(Mode.SingleShotTime)
            .param("exampleName", BenchmarkUtil.SPEC_EXAMPLE_NAMES.toArray(new String[0])) // https://stackoverflow.com/a/4042464/161022
            .build();
    return new Runner(opt).run();
}
 

@Test
@Ignore("Ignoring this test in favor of the existing jMeter tests, but keep it for local tests of future refactorings to action matching")
public void launchBenchmark() throws Exception {
  final double expectedMinUsPerOp = 350;

  Options opt = new OptionsBuilder()
    .include(this.getClass().getSimpleName() + "\\.benchmark.*")
    .mode(Mode.AverageTime)
    .timeUnit(TimeUnit.MICROSECONDS)
    .warmupTime(TimeValue.seconds(1))
    .warmupIterations(2)
    .measurementTime(TimeValue.seconds(1))
    .measurementIterations(2)
    .threads(1)
    .forks(1)
    .shouldFailOnError(true)
    .shouldDoGC(true)
    .build();

  RunResult result = new Runner(opt).runSingle();
  double usPerOp = result.getPrimaryResult().getScore();
  assertTrue("µs/matching operation = " + usPerOp + " is higher than " + expectedMinUsPerOp,
    usPerOp < expectedMinUsPerOp);
}
 

private void testScenario(String scenarioName, double threshold) throws RunnerException, IOException {
    String scenario = Resources.toString(Resources.getResource(scenarioName), Charsets.UTF_8);
    scenario = scenario.replaceAll("DOCUMENTPLACEHOLDER", tempFolder.getRoot().getAbsolutePath());

    String json = ConfigFactory.parseString(scenario).root().render(ConfigRenderOptions.concise());
    Options opts = JsonUtils.fromJsonString(SawmillBenchmarkOptions.class, json).toJmhOptions();
    Iterator<RunResult> results = new Runner(opts).run().iterator();

    // Copy the artifacts
    File sourceResults = new File(tempFolder.getRoot(), "result.json");
    Files.copy(sourceResults, new File(targetDir(), "result.json"));

    logger.info("Test Output:");
    logger.info(FileUtils.readFileToString(sourceResults, Charsets.UTF_8));

    while ( results.hasNext()) {
        RunResult runResults = results.next();
        assertThat(runResults.getPrimaryResult().getScore()).isGreaterThan(threshold);
    }
}
 

Collection<RunResult> run() throws RunnerException {
    Options opt = new OptionsBuilder()
            .include(getSimpleClassName())
            .forks(1)
            .build();
    return new Runner(opt).run();
}
 

public static void main(String[] args)
        throws Exception
{
    Options opt = new OptionsBuilder()
            .include(".*\\." + BenchmarkHiveFileFormat.class.getSimpleName() + ".*")
            .jvmArgsAppend("-Xmx4g", "-Xms4g", "-XX:+UseG1GC")
            .build();

    Collection<RunResult> results = new Runner(opt).run();

    for (RunResult result : results) {
        Statistics inputSizeStats = result.getSecondaryResults().get("inputSize").getStatistics();
        Statistics outputSizeStats = result.getSecondaryResults().get("outputSize").getStatistics();
        double compressionRatio = 1.0 * inputSizeStats.getSum() / outputSizeStats.getSum();
        String compression = result.getParams().getParam("compression");
        String fileFormat = result.getParams().getParam("fileFormat");
        String dataSet = result.getParams().getParam("dataSet");
        System.out.printf("  %-10s  %-30s  %-10s  %-25s  %2.2f  %10s ± %11s (%5.2f%%) (N = %d, \u03B1 = 99.9%%)\n",
                result.getPrimaryResult().getLabel(),
                dataSet,
                compression,
                fileFormat,
                compressionRatio,
                toHumanReadableSpeed((long) inputSizeStats.getMean()),
                toHumanReadableSpeed((long) inputSizeStats.getMeanErrorAt(0.999)),
                inputSizeStats.getMeanErrorAt(0.999) * 100 / inputSizeStats.getMean(),
                inputSizeStats.getN());
    }
    System.out.println();
}
 

/**
 * Benchmark runner
 */
public static void main(String[] args) throws RunnerException {
    List<String> policies = Arrays.asList("inc", "dec", "r25", "r50", "r75");
    int[] threads = {2, 4, 8, 16, 32};

    List<RunResult> results = new ArrayList<>();

    for (String policy : policies) {
        for (int thread : threads) {
            ChainedOptionsBuilder builder = new OptionsBuilder()
                .jvmArgs()
                .timeUnit(TimeUnit.MILLISECONDS)
                .measurementIterations(10)
                .measurementTime(TimeValue.seconds(20))
                .warmupIterations(5)
                .warmupTime(TimeValue.seconds(10))
                .jvmArgs("-Dbench.exp.policy=" + policy)
                .forks(1)
                .threads(thread)
                .mode(Mode.Throughput)
                .include(JmhWaitStategyBenchmark.class.getSimpleName());

            results.addAll(new Runner(builder.build()).run());
        }
    }

    for (RunResult result : results) {
        BenchmarkParams params = result.getParams();
        Collection<String> args1 = params.getJvmArgs();
        for (String s : args1) {
            System.out.print(s.substring(s.length() - 3, s.length()));
            System.out.print(" x ");
        }
        System.out.print(params.getThreads());
        System.out.print("\t\t");
        System.out.println(result.getPrimaryResult().toString());
    }
}
 

private void doWrite(Collection<RunResult> results) throws IOException {

		StandardEnvironment env = new StandardEnvironment();

		String projectVersion = env.getProperty("project.version", "unknown");
		String gitBranch = env.getProperty("git.branch", "unknown");
		String gitDirty = env.getProperty("git.dirty", "no");
		String gitCommitId = env.getProperty("git.commit.id", "unknown");

		HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
		connection.setConnectTimeout((int) Duration.ofSeconds(1).toMillis());
		connection.setReadTimeout((int) Duration.ofSeconds(1).toMillis());
		connection.setDoOutput(true);
		connection.setRequestMethod("POST");

		connection.setRequestProperty("Content-Type", "application/json");
		connection.addRequestProperty("X-Project-Version", projectVersion);
		connection.addRequestProperty("X-Git-Branch", gitBranch);
		connection.addRequestProperty("X-Git-Dirty", gitDirty);
		connection.addRequestProperty("X-Git-Commit-Id", gitCommitId);

		try (OutputStream output = connection.getOutputStream()) {
			output.write(ResultsWriter.jsonifyResults(results).getBytes(StandardCharsets.UTF_8));
		}

		if (connection.getResponseCode() >= 400) {
			throw new IllegalStateException(
					String.format("Status %d %s", connection.getResponseCode(), connection.getResponseMessage()));
		}
	}
 

@Override
public void write(OutputFormat output, Collection<RunResult> results) {

	if (CollectionUtils.isEmpty(results)) {
		return;
	}

	try {
		doWrite(results);
	} catch (IOException e) {
		output.println("Failed to write results: " + e.toString());
	}
}
 

private void doWrite(Collection<RunResult> results) throws ParseException {

		Date now = new Date();
		StandardEnvironment env = new StandardEnvironment();

		String projectVersion = env.getProperty("project.version", "unknown");
		String gitBranch = env.getProperty("git.branch", "unknown");
		String gitDirty = env.getProperty("git.dirty", "no");
		String gitCommitId = env.getProperty("git.commit.id", "unknown");

		ConnectionString uri = new ConnectionString(this.uri);
		MongoClient client = MongoClients.create();

		String dbName = StringUtils.hasText(uri.getDatabase()) ? uri.getDatabase() : "spring-data-mongodb-benchmarks";
		MongoDatabase db = client.getDatabase(dbName);

		String resultsJson = ResultsWriter.jsonifyResults(results).trim();
		JSONArray array = (JSONArray) new JSONParser(JSONParser.MODE_PERMISSIVE).parse(resultsJson);
		for (Object object : array) {
			JSONObject dbo = (JSONObject) object;

			String collectionName = extractClass(dbo.get("benchmark").toString());

			Document sink = new Document();
			sink.append("_version", projectVersion);
			sink.append("_branch", gitBranch);
			sink.append("_commit", gitCommitId);
			sink.append("_dirty", gitDirty);
			sink.append("_method", extractBenchmarkName(dbo.get("benchmark").toString()));
			sink.append("_date", now);
			sink.append("_snapshot", projectVersion.toLowerCase().contains("snapshot"));

			sink.putAll(dbo);

			db.getCollection(collectionName).insertOne(fixDocumentKeys(sink));
		}

		client.close();
	}
 

@Override
public void write(OutputFormat output, Collection<RunResult> results) {

	if (CollectionUtils.isEmpty(results)) {
		return;
	}

	try {
		doWrite(results);
	} catch (ParseException | RuntimeException e) {
		output.println("Failed to write results: " + e.toString());
	}
}
 

public static void main(String... args) throws RunnerException, CommandLineOptionException {
    Options opt = new OptionsBuilder()
        .parent(new CommandLineOptions())
        .include(V2DefaultClientCreationBenchmark.class.getSimpleName())
        .addProfiler(StackProfiler.class)
        .build();
    Collection<RunResult> run = new Runner(opt).run();
}
 

public static void main(String... args) throws RunnerException, CommandLineOptionException {
    Options opt = new OptionsBuilder()
        .parent(new CommandLineOptions())
        .include(V1ClientCreationBenchmark.class.getSimpleName())
        .addProfiler(StackProfiler.class)
        .build();
    Collection<RunResult> run = new Runner(opt).run();
}
 

public static void main(String... args) throws RunnerException, CommandLineOptionException {
    Options opt = new OptionsBuilder()
        .parent(new CommandLineOptions())
        .include(V2OptimizedClientCreationBenchmark.class.getSimpleName())
        .addProfiler(StackProfiler.class)
        .build();
    Collection<RunResult> run = new Runner(opt).run();
}
 

public static void main(String... args) throws Exception {
    Options opt = new OptionsBuilder()
        .include(NettyHttpClientH1Benchmark.class.getSimpleName())
        .addProfiler(StackProfiler.class)
        .build();
    Collection<RunResult> run = new Runner(opt).run();
}
 

public static void main(String... args) throws Exception {
    Options opt = new OptionsBuilder()
        .include(NettyClientH1NonTlsBenchmark.class.getSimpleName())
        .addProfiler(StackProfiler.class)
        .build();
    Collection<RunResult> run = new Runner(opt).run();
}
 

public static void main(String... args) throws Exception {

        Options opt = new OptionsBuilder()
            .include(UrlConnectionHttpClientBenchmark.class.getSimpleName())
            .addProfiler(StackProfiler.class)
            .build();
        Collection<RunResult> run = new Runner(opt).run();
    }
 

public static void main(String... args) throws Exception {

        Options opt = new OptionsBuilder()
            .include(ApacheHttpClientBenchmark.class.getSimpleName() + ".concurrentApiCall")
            .addProfiler(StackProfiler.class)
            .build();
        Collection<RunResult> run = new Runner(opt).run();
    }
 

private SdkBenchmarkResult constructSdkBenchmarkResult(RunResult runResult) {
    Statistics statistics = runResult.getPrimaryResult().getStatistics();

    SdkBenchmarkStatistics sdkBenchmarkStatistics = new SdkBenchmarkStatistics(statistics);
    SdkBenchmarkParams sdkBenchmarkParams = new SdkBenchmarkParams(runResult.getParams());

    return new SdkBenchmarkResult(getBenchmarkId(runResult.getParams()),
                                  sdkBenchmarkParams,
                                  sdkBenchmarkStatistics);
}
 

/**
 * Process benchmark results
 *
 * @param results the results of the benchmark
 * @return the benchmark Id that failed the regression
 */
List<String> processBenchmarkResult(Collection<RunResult> results) {
    List<SdkBenchmarkResult> currentData = new ArrayList<>();
    for (RunResult result : results) {
        String benchmarkId = getBenchmarkId(result.getParams());

        SdkBenchmarkResult baselineResult = baseline.get(benchmarkId);
        SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result);

        if (baselineResult == null) {
            log.warn(() -> {
                String benchmarkResultJson = null;
                try {
                    benchmarkResultJson = OBJECT_MAPPER.writeValueAsString(sdkBenchmarkData);
                } catch (IOException e) {
                    log.error(() -> "Unable to serialize result data to JSON");
                }
                return String.format("Unable to find the baseline for %s. Skipping regression validation. " +
                        "Results were: %s", benchmarkId, benchmarkResultJson);
            });
            continue;
        }

        currentData.add(sdkBenchmarkData);

        if (!validateBenchmarkResult(sdkBenchmarkData, baselineResult)) {
            failedBenchmarkIds.add(benchmarkId);
        }
    }

    log.info(() -> "Current result: " + serializeResult(currentData));
    return failedBenchmarkIds;
}
 

public static void exportResult(String titleStr, Collection<RunResult> results,
                                String paramKey, String xunit) throws Exception {
    // 几个测试对象
    List<String> objects = new ArrayList<>();
    // 测试维度，输入值n
    List<String> dimensions = new ArrayList<>();
    // 有几个测试对象，就有几组测试数据，每组测试数据中对应几个维度的结果
    List<List<Double>> allData = new ArrayList<>();
    List<Double> temp = new ArrayList<>();
    for (RunResult runResult : results) {
        BenchmarkResult benchmarkResult = runResult.getAggregatedResult();
        Result r = benchmarkResult.getPrimaryResult();
        BenchmarkParams params = runResult.getParams();
        if (!objects.contains(r.getLabel())) {
            objects.add(r.getLabel());
            if (!temp.isEmpty()) {
                allData.add(temp);
                temp = new ArrayList<>();
            }
        }
        temp.add(Double.parseDouble(String.format("%.2f", r.getScore())));
        // 测试维度
        if (!dimensions.contains("n=" + params.getParam(paramKey))) {
            dimensions.add("n=" + params.getParam(paramKey));
        }
    }
    // 最后一组测试数据别忘记加进去了
    allData.add(temp);

    String optionStr = generateOption(titleStr, objects, dimensions, allData, xunit);
    // POST到接口上
    postOption(optionStr, "http://localhost:9075/api/v1/data");
}
 

public static void main(String[] args) throws Exception {
    Options opt = new OptionsBuilder()
            .include(SecondBenchmark.class.getSimpleName())
            .forks(1)
            .warmupIterations(5)
            .measurementIterations(2)
            .build();
    Collection<RunResult> results =  new Runner(opt).run();
    ResultExporter.exportResult("单线程与多线程求和性能", results, "length", "微秒");
}
 

public static void main(String[] args) throws Exception {
    Options opt = new OptionsBuilder()
            .include(JsonSerializeBenchmark.class.getSimpleName())
            .forks(1)
            .warmupIterations(0)
            .build();
    Collection<RunResult> results =  new Runner(opt).run();
    ResultExporter.exportResult("JSON序列化性能", results, "count", "秒");
}
 

public static void main(String[] args) throws Exception {
    Options opt = new OptionsBuilder()
            .include(JsonDeserializeBenchmark.class.getSimpleName())
            .forks(1)
            .warmupIterations(0)
            .build();
    Collection<RunResult> results =  new Runner(opt).run();
    ResultExporter.exportResult("JSON反序列化性能", results, "count", "秒");
}
 

public double score() {
    if (score != Double.NEGATIVE_INFINITY) {
        // Already got the score, shortcutting
        return score;
    }

    try {
        // Add the options encoded by this solution:
        //  a) Mix in base options.
        //  b) Add JVM arguments: we opt to parse the
        //     stringly representation to make the example
        //     shorter. There are, of course, cleaner ways
        //     to do this.
        Options theseOpts = new OptionsBuilder()
                .parent(baseOpts)
                .jvmArgs(toString().split("[ ]"))
                .build();

        // Run through JMH and get the result back.
        RunResult runResult = new Runner(theseOpts).runSingle();
        score = runResult.getPrimaryResult().getScore();
    } catch (RunnerException e) {
        // Something went wrong, the solution is defective
        score = Double.MIN_VALUE;
    }

    return score;
}
 

public static void main(String... args) throws Exception {
    Options opt = new OptionsBuilder()
        .include(NettyHttpClientH2Benchmark.class.getSimpleName())
        .build();
    Collection<RunResult> run = new Runner(opt).run();
}
 

private void runBenchmark() throws RunnerException {
    ChainedOptionsBuilder optionsBuilder = new OptionsBuilder();

    benchmarksToRun.forEach(optionsBuilder::include);

    log.info(() -> "Starting to run: " + benchmarksToRun);

    Collection<RunResult> results = new Runner(optionsBuilder.build()).run();

    List<String> failedResult = resultProcessor.processBenchmarkResult(results);

    if (!failedResult.isEmpty()) {
        log.info(() -> "Failed perf regression tests: " + failedResult);
        throw new RuntimeException("Perf regression tests failed: " + failedResult);
    }
}