Java Code Examples for htsjdk.samtools.metrics.MetricsFile#read()

@Test(dataProvider = "shardDataProvider")
public void testMerge(final List<File> inputs) throws IOException {
    final File mergedFilePrefix = new File(TEST_DATA_DIR + "mergeTest");
    final File mergedSummaryFile = new File(mergedFilePrefix.getAbsolutePath() + ".variant_calling_summary_metrics");
    final File mergedDetailFile = new File(mergedFilePrefix.getAbsolutePath() + ".variant_calling_detail_metrics");
    mergedSummaryFile.deleteOnExit();
    mergedDetailFile.deleteOnExit();

    final AccumulateVariantCallingMetrics program = new AccumulateVariantCallingMetrics();
    program.INPUT = inputs;
    program.OUTPUT = mergedFilePrefix;

    Assert.assertEquals(program.doWork(), 0);

    final MetricsFile<CollectVariantCallingMetrics.VariantCallingDetailMetrics, Comparable<?>> detail = new MetricsFile<>();
    detail.read(new FileReader(mergedDetailFile));

    final MetricsFile<CollectVariantCallingMetrics.VariantCallingSummaryMetrics, Comparable<?>> summary = new MetricsFile<>();
    summary.read(new FileReader(mergedSummaryFile));

    checkResults(detail, summary);
}
 

@Test(dataProvider = "sumOfGaussiansDataProvider")
public void testDrawSumOfQScores(final File metricsFile, final int altDepth, final double tolerance) throws Exception {
    final MetricsFile<TheoreticalSensitivityMetrics, Integer> metrics = new MetricsFile<>();
    try (final FileReader metricsFileReader = new FileReader(metricsFile)) {
        metrics.read(metricsFileReader);
    }

    final List<Histogram<Integer>> histograms = metrics.getAllHistograms();

    final Histogram<Integer> qualityHistogram = histograms.get(1);
    final TheoreticalSensitivity.RouletteWheel qualityRW = new TheoreticalSensitivity.RouletteWheel(TheoreticalSensitivity.trimDistribution(TheoreticalSensitivity.normalizeHistogram(qualityHistogram)));

    final Random randomNumberGenerator = new Random(51);

    // Calculate mean and deviation of quality score distribution to enable Gaussian sampling below
    final double averageQuality = qualityHistogram.getMean();
    final double standardDeviationQuality = qualityHistogram.getStandardDeviation();

    for (int k = 0; k < 1; k++) {
        int sumOfQualitiesFull = IntStream.range(0, altDepth).map(n -> qualityRW.draw()).sum();
        int sumOfQualities = TheoreticalSensitivity.drawSumOfQScores(altDepth, averageQuality, standardDeviationQuality, randomNumberGenerator.nextGaussian());

        Assert.assertEquals(sumOfQualitiesFull, sumOfQualities, sumOfQualitiesFull * tolerance);
    }
}
 

@Test(dataProvider = "simpleTests")
public void simpleTest(final String vcf, final String bam, final Map<String, Object> fieldValueMap) throws IOException, NoSuchFieldException, IllegalAccessException {

    final CollectIndependentReplicateMetrics est = new CollectIndependentReplicateMetrics();
    est.INPUT = bams.get(bam);
    est.VCF = new File(testdir, vcf);
    est.OUTPUT = IOUtil.newTempFile("singleHet", ".duplication_metric", new File[]{bamOutDir});
    est.MATRIX_OUTPUT = IOUtil.newTempFile("singleHet", ".duplication_matrix", new File[]{bamOutDir});
    est.SAMPLE = "SAMPLE1";

    est.OUTPUT.deleteOnExit();
    est.MATRIX_OUTPUT.deleteOnExit();

    est.doWork();

    final MetricsFile<IndependentReplicateMetric, Integer> retval = new MetricsFile<>();
    retval.read(new FileReader(est.OUTPUT));

    for (final Map.Entry<String, Object> fieldValue : fieldValueMap.entrySet()) {
        final String field = fieldValue.getKey();
        final Object expectedValue = fieldValue.getValue();
        final Field o = IndependentReplicateMetric.class.getField(field);
        assertEquals(o.get(retval.getMetrics().get(0)), expectedValue, field);
    }
}
 

@Test(dataProvider = "equivalanceHetVsArbitrary")
public void testHetVsArbitrary(final File metricsFile, final double tolerance, final int sampleSize) throws Exception {
    // This test compares Theoretical Sensitivity for arbitrary allele fractions with the theoretical het sensitivity
    // model.  Since allele fraction of 0.5 is equivalent to a het, these should provide the same answer.
    final MetricsFile<?, Integer> metrics = new MetricsFile<>();
    try (final FileReader metricsFileReader = new FileReader(metricsFile)) {
        metrics.read(metricsFileReader);
    }

    final List<Histogram<Integer>> histograms = metrics.getAllHistograms();
    final Histogram<Integer> depthHistogram = histograms.get(0);
    final Histogram<Integer> qualityHistogram = histograms.get(1);

    final double[] qualityDistribution = TheoreticalSensitivity.normalizeHistogram(qualityHistogram);
    final double[] depthDistribution = TheoreticalSensitivity.normalizeHistogram(depthHistogram);

    final double resultFromTS = TheoreticalSensitivity.theoreticalSensitivity(depthHistogram, qualityHistogram, sampleSize, 3, 0.5);
    final double resultFromTHS = TheoreticalSensitivity.hetSNPSensitivity(depthDistribution, qualityDistribution, sampleSize, 3);

    Assert.assertEquals(resultFromTS, resultFromTHS, tolerance);
}
 

/** Read back the first metrics record in an hs metrics file. */
private HsMetrics readMetrics(final File f) {
    try {
        final MetricsFile<HsMetrics, Comparable<?>> mFile = new MetricsFile<HsMetrics, Comparable<?>>();
        mFile.read(new FileReader(f));
        return mFile.getMetrics().get(0);
    }
    catch (IOException ioe) {
         throw new RuntimeIOException(ioe);
    }
}
 

private void assertMetricsFileEqual(final File actualMetricsFile, final File expectedMetricsFile) throws FileNotFoundException {
    // Actual metrics file
    final MetricsFile<GenotypeConcordanceSummaryMetrics, Comparable<?>> actual = new MetricsFile<GenotypeConcordanceSummaryMetrics, Comparable<?>>();
    actual.read(new FileReader(actualMetricsFile));

    // Expected metrics file
    final MetricsFile<GenotypeConcordanceSummaryMetrics, Comparable<?>> expected = new MetricsFile<GenotypeConcordanceSummaryMetrics, Comparable<?>>();
    expected.read(new FileReader(expectedMetricsFile));

    // Note - cannot use .equals as it calls .areHeadersEqual and they are not since the timestamp (at a minimum is different)
    Assert.assertTrue(expected.areMetricsEqual(actual));
    Assert.assertTrue(expected.areHistogramsEqual(actual));
}
 

@Test(dataProvider="filenames", groups = {"R", "spark"})
public void test(final String inputFile, final String referenceName) throws IOException {
    final File input = new File(TEST_DATA_DIR, inputFile);
    final File expectedFile = new File(TEST_DATA_DIR, "meanqualbycycle.txt");
    final File outfile = GATKBaseTest.createTempFile("testMeanQualityByCycle", ".metrics");
    final File pdf = GATKBaseTest.createTempFile("testMeanQualityByCycle", ".pdf");
    outfile.deleteOnExit();
    pdf.deleteOnExit();
    final ArgumentsBuilder args = new ArgumentsBuilder();
    args.addRaw("--input");
    args.addRaw(input.getAbsolutePath());
    args.addRaw("--output");
    args.addRaw(outfile.getAbsolutePath());
    args.addRaw("--chart");
    args.addRaw(pdf.getAbsolutePath());
    if (null != referenceName) {
        final File REF = new File(referenceName);
        args.addRaw("-R");
        args.addRaw(REF.getAbsolutePath());
    }

    runCommandLine(args.getArgsArray());

    try (final FileReader actualReader = new FileReader(outfile);) {
        final MetricsFile<?,Integer> output = new MetricsFile<>();
        output.read(actualReader);
        Assert.assertEquals(output.getAllHistograms().size(), 1);
        Assert.assertEquals(output.getHistogram().size(), 202);
    }
    Assert.assertTrue(pdf.exists(), "exists");
    Assert.assertTrue(pdf.length() > 0, "length");
    IntegrationTestSpec.assertEqualTextFiles(outfile, expectedFile, "#");
}
 

/**
 * Compares metric's results by summary files without duplicates.
 * @throws IOException
 */
@Test
public void runNonDupsComparisonTest() throws IOException {
    final File inputFileWithDuplicates = new File("testdata/picard/metrics/chrMReads.sam");
    final File detailsOutfile = File.createTempFile("test", ".gc_bias_detail_metrics");
    final File summaryOutfile = File.createTempFile("test", ".gc_bias_summary_metrics");
    detailsOutfile.deleteOnExit();
    summaryOutfile.deleteOnExit();

    runGcBias(inputFileWithDuplicates, CHR_M_REFERENCE.getAbsolutePath(), summaryOutfile, detailsOutfile, true);

    final MetricsFile<GcBiasSummaryMetrics, Comparable<?>> outputSummary = new MetricsFile<>();
    outputSummary.read(new FileReader(summaryOutfile));

    for (final GcBiasSummaryMetrics summary : outputSummary.getMetrics()) {
        if (summary.ACCUMULATION_LEVEL.equals(ACCUMULATION_LEVEL_ALL_READS) && summary.READS_USED.equals(READS_USED_UNIQUE)) { //ALL_READS level for case without duplicates
            Assert.assertEquals(summary.TOTAL_CLUSTERS, 3);
            Assert.assertEquals(summary.ALIGNED_READS, 3);
            Assert.assertEquals(summary.AT_DROPOUT, 79.180328);
            Assert.assertEquals(summary.GC_DROPOUT, 12.28901);
            Assert.assertEquals(summary.GC_NC_0_19, 0.0);
            Assert.assertEquals(summary.GC_NC_20_39, 0.0);
            Assert.assertEquals(summary.GC_NC_40_59, 1.246783);
            Assert.assertEquals(summary.GC_NC_60_79, 0.0);
            Assert.assertEquals(summary.GC_NC_80_100, 0.0);
        }
        if (summary.ACCUMULATION_LEVEL.equals(ACCUMULATION_LEVEL_ALL_READS) && summary.READS_USED.equals(READS_USED_ALL)) { //ALL_READS level
            Assert.assertEquals(summary.TOTAL_CLUSTERS, 5);
            Assert.assertEquals(summary.ALIGNED_READS, 5);
            Assert.assertEquals(summary.AT_DROPOUT, 79.180328);
            Assert.assertEquals(summary.GC_DROPOUT, 10.37037);
            Assert.assertEquals(summary.GC_NC_0_19, 0.0);
            Assert.assertEquals(summary.GC_NC_20_39, 0.0);
            Assert.assertEquals(summary.GC_NC_40_59, 1.246783);
            Assert.assertEquals(summary.GC_NC_60_79, 0.0);
            Assert.assertEquals(summary.GC_NC_80_100, 0.0);
        }
    }
}
 

@Test(dataProvider = "compareSams")
public void testComparisons(final String f1, final String f2, final ArrayList<String> args, final boolean areEqual) throws IOException {
    final Path tmpOutput = Files.createTempFile("compareSam", ".tsv");
    final String in1 = new File(TEST_FILES_DIR, f1).getAbsolutePath();
    final String in2 = new File(TEST_FILES_DIR, f2).getAbsolutePath();
    ArrayList<String> commandArgs = new ArrayList<>(
            Arrays.asList(
                    in1,
                    in2,
                    "O=" + tmpOutput
            )
    );
    if (args != null) {
        commandArgs.addAll(args);
    }
    Assert.assertEquals(runPicardCommandLine(commandArgs) == 0, areEqual);
    final MetricsFile<SamComparisonMetric, Comparable<?>> metricsOutput = new MetricsFile<>();
    metricsOutput.read(new FileReader(tmpOutput.toFile()));

    //swap order of input files
    commandArgs = new ArrayList<>(
            Arrays.asList(
                    in2,
                    in1,
                    "O=" + tmpOutput
            )
    );
    if (args != null) {
        commandArgs.addAll(args);
    }
    Assert.assertEquals(runPicardCommandLine(commandArgs) == 0, areEqual);
    metricsOutput.read(new FileReader(tmpOutput.toFile()));

    Assert.assertEquals(metricsOutput.getMetrics().get(0).LEFT_FILE, in1);
    Assert.assertEquals(metricsOutput.getMetrics().get(0).RIGHT_FILE, in2);

    Assert.assertEquals(metricsOutput.getMetrics().get(1).LEFT_FILE, in2);
    Assert.assertEquals(metricsOutput.getMetrics().get(1).RIGHT_FILE, in1);
}
 

private MetricsFile<ExtractIlluminaBarcodes.BarcodeMetric, Integer> runIt(final List<String> args, final File metricsFile) throws Exception {
    // Generate _barcode.txt files and metrics file.
    Assert.assertEquals(runPicardCommandLine(args), 0);

    final MetricsFile<ExtractIlluminaBarcodes.BarcodeMetric, Integer> retval = new MetricsFile<>();
    retval.read(new FileReader(metricsFile));
    return retval;
}
 

@Test
public void testChimeras() throws IOException {
    final File input = new File(TEST_DATA_DIR, "summary_alignment_stats_test_chimeras.sam");
    final File reference = new File(TEST_DATA_DIR, "summary_alignment_stats_test.fasta");
    final File outfile = File.createTempFile("alignmentMetrics", ".txt");
    outfile.deleteOnExit();
    final String[] args = new String[]{
            "INPUT=" + input.getAbsolutePath(),
            "OUTPUT=" + outfile.getAbsolutePath(),
            "MAX_INSERT_SIZE=20",
            "REFERENCE_SEQUENCE=" + reference.getAbsolutePath(),
    };
    Assert.assertEquals(runPicardCommandLine(args), 0);

    final MetricsFile<AlignmentSummaryMetrics, Comparable<?>> output = new MetricsFile<>();
    try (FileReader reader = new FileReader(outfile)) {
        output.read(reader);
    }

    for (final AlignmentSummaryMetrics metrics : output.getMetrics()) {
        if (metrics.CATEGORY == AlignmentSummaryMetrics.Category.FIRST_OF_PAIR) {
            TestNGUtil.compareDoubleWithAccuracy(metrics.PCT_CHIMERAS, 5D / 6, 0.0001);
        }
        if (metrics.CATEGORY == AlignmentSummaryMetrics.Category.SECOND_OF_PAIR) {
            TestNGUtil.compareDoubleWithAccuracy(metrics.PCT_CHIMERAS, 3D / 6, 0.0001);
        }
    }
}
 

@Test
public void testAdapterReads() throws IOException {
    final File input = new File(TEST_DATA_DIR, "summary_alignment_stats_test_adapter_reads.sam");
    final File outfile = File.createTempFile("alignmentMetrics", ".txt");
    outfile.deleteOnExit();
    final String[] args = new String[]{
            "INPUT=" + input.getAbsolutePath(),
            "OUTPUT=" + outfile.getAbsolutePath(),
            "MAX_INSERT_SIZE=200",
            "REFERENCE_SEQUENCE=" + CHR_M_REFERENCE.getAbsolutePath(),
    };
    Assert.assertEquals(runPicardCommandLine(args), 0);

    final MetricsFile<AlignmentSummaryMetrics, Comparable<?>> output = new MetricsFile<>();
    try (FileReader reader = new FileReader(outfile)) {
        output.read(reader);
    }

    for (final AlignmentSummaryMetrics metrics : output.getMetrics()) {
        if (metrics.CATEGORY == AlignmentSummaryMetrics.Category.FIRST_OF_PAIR) {
            TestNGUtil.compareDoubleWithAccuracy(metrics.PCT_ADAPTER, 0D, 0.0001);
            TestNGUtil.compareDoubleWithAccuracy(metrics.PCT_PF_READS_ALIGNED, 1D, 0.0001);
        }
        if (metrics.CATEGORY == AlignmentSummaryMetrics.Category.SECOND_OF_PAIR) {
            TestNGUtil.compareDoubleWithAccuracy(metrics.PCT_ADAPTER, 1D, 0.0001);
            TestNGUtil.compareDoubleWithAccuracy(metrics.PCT_PF_READS_ALIGNED, 0D, 0.0001);
        }
    }
}
 

@Test
public void testCreateVerifyIDIntensityContaminationMetricsFile() throws IOException {
    final File outputBase = File.createTempFile("testCreateVerifyIDIntensityContaminationMetricsFile", "");
    final File output = new File(outputBase.getAbsolutePath() + "." + CreateVerifyIDIntensityContaminationMetricsFile.FILE_EXTENSION);
    output.deleteOnExit();
    System.out.println(outputBase.getAbsolutePath());
    System.out.println(output.getAbsolutePath());

    final CreateVerifyIDIntensityContaminationMetricsFile createVerifyIDIntensityContaminationMetricsFile = new CreateVerifyIDIntensityContaminationMetricsFile();
    createVerifyIDIntensityContaminationMetricsFile.INPUT = TEST_INPUT_FILE;
    createVerifyIDIntensityContaminationMetricsFile.OUTPUT = outputBase;

    Assert.assertEquals(createVerifyIDIntensityContaminationMetricsFile.instanceMain(new String[0]), 0);

    final MetricsFile<VerifyIDIntensityContaminationMetrics, Comparable<?>> metrics = new MetricsFile<>();
    metrics.read(new FileReader(output));

    Assert.assertEquals(metrics.getMetrics().size(), 4);
    Assert.assertEquals(metrics.getMetrics().get(0).ID, 0);
    Assert.assertEquals(metrics.getMetrics().get(0).PCT_MIX, 0.214766, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(0).LLK, 157575.0, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(0).LLK0, 177169.0, 0.0001);

    Assert.assertEquals(metrics.getMetrics().get(1).ID, 1);
    Assert.assertEquals(metrics.getMetrics().get(1).PCT_MIX, 0.214767, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(1).LLK, 157576.0, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(1).LLK0, 177170.0, 0.0001);

    Assert.assertEquals(metrics.getMetrics().get(2).ID, 2);
    Assert.assertEquals(metrics.getMetrics().get(2).PCT_MIX, 0.0994769, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(2).LLK, 90260.4, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(2).LLK0, 91166.7, 0.0001);

    Assert.assertEquals(metrics.getMetrics().get(3).ID, 3);
    Assert.assertEquals(metrics.getMetrics().get(3).PCT_MIX, 0.234567, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(3).LLK, -4703.97, 0.0001);
    Assert.assertEquals(metrics.getMetrics().get(3).LLK0, -5204.97, 0.0001);
}
 

@Test(dataProvider = "wgsAlgorithm")
public void testLargeIntervals(final String useFastAlgorithm) throws IOException {
    final File input = new File(TEST_DIR, "forMetrics.sam");
    final File outfile = File.createTempFile("test", ".wgs_metrics");
    outfile.deleteOnExit();
    final File ref = new File(TEST_DIR, "merger.fasta");
    final File intervals = new File(TEST_DIR, "largeIntervals.interval_list");
    final int sampleSize = 1000;
    final String[] args = new String[]{
            "INPUT=" + input.getAbsolutePath(),
            "OUTPUT=" + outfile.getAbsolutePath(),
            "REFERENCE_SEQUENCE=" + ref.getAbsolutePath(),
            "INTERVALS=" + intervals.getAbsolutePath(),
            "SAMPLE_SIZE=" + sampleSize,
            "USE_FAST_ALGORITHM=" + useFastAlgorithm
    };
    Assert.assertEquals(runPicardCommandLine(args), 0);

    final MetricsFile<WgsMetrics, Comparable<?>> output = new MetricsFile<>();
    try (FileReader reader = new FileReader(outfile)) {
        output.read(reader);
    }
    for (final WgsMetrics metrics : output.getMetrics()) {
        Assert.assertEquals(metrics.GENOME_TERRITORY, 404);
        Assert.assertEquals(metrics.PCT_EXC_ADAPTER, 0D);
        Assert.assertEquals(metrics.PCT_EXC_MAPQ, 0.271403);
        Assert.assertEquals(metrics.PCT_EXC_DUPE, 0.182149);
        Assert.assertEquals(metrics.PCT_EXC_UNPAIRED, 0.091075);
    }
}
 

@Test
public void testHistograms() throws IOException {
    final File outputTarGz = createTempFile("f1r2", ".tar.gz");
    final String sample = "SAMPLE";
    final File sam = createSyntheticSam(30, 1, sample);

    final String[] args = {
            "-R", hg19_chr1_1M_Reference,
            "-I", sam.getAbsolutePath(),
            "-O", outputTarGz.getAbsolutePath()
    };

    runCommandLine(args);

    final File extractedDir = createTempDir("extracted");
    IOUtils.extractTarGz(outputTarGz.toPath(), extractedDir.toPath());

    final File altHistogramFile = F1R2CountsCollector.getAltHistogramsFromExtractedTar(extractedDir).stream().findFirst().get();

    final MetricsFile<?, Integer> referenceSiteMetrics = new MetricsFile<>();
    final Reader in = IOUtil.openFileForBufferedReading(altHistogramFile);
    referenceSiteMetrics.read(in);
    CloserUtil.close(in);

    final MetricsFile<?, Integer> altSiteMetrics = new MetricsFile<>();
    final Reader altMetricsReader = IOUtil.openFileForBufferedReading(altHistogramFile);
    altSiteMetrics.read(altMetricsReader);
    CloserUtil.close(altMetricsReader);

    final List<Histogram<Integer>> histograms = altSiteMetrics.getAllHistograms();

    // TODO: should there be 64*3*2 = 384 histograms or just the non-zero ones?
    final String[] expectedTransitions = new String[]{"CAC_T_F2R1", "TAA_G_F2R1", "AGC_C_F2R1", "ACA_A_F2R1"};
    // Assert.assertEquals(histograms.size(), expectedTransitions.length, "alt histogram must contain the expected number of contexts");

    for (String transition : expectedTransitions ){
        Optional<Histogram<Integer>> histogram = histograms.stream()
                .filter(h -> h.getValueLabel().equals(transition))
                .findFirst();
        Assert.assertTrue(histogram.isPresent(), "histogram must exist");
        Assert.assertEquals((int) histogram.get().getSumOfValues(), 1, "histogram must only contain one read");
    }
}
 

@Override
public void test() {
    try {
        updateExpectedDuplicationMetrics();
        // Read the output and check the duplicate flag
        int outputRecords = 0;
        int indexInFile = 0;
        final SamReader reader = SamReaderFactory.makeDefault().open(getOutput());
        for (final SAMRecord record : reader) {
            outputRecords++;

            final String key = samRecordToDuplicatesFlagsKey(record);
            Assert.assertTrue(this.duplicateFlags.containsKey(key),"DOES NOT CONTAIN KEY: " + key);
            final boolean value = this.duplicateFlags.get(key);
            this.duplicateFlags.remove(key);
            if (value != record.getDuplicateReadFlag()) {
                System.err.println("Mismatching read:");
                System.err.print(record.getSAMString());
            }
            Assert.assertEquals(record.getDuplicateReadFlag(), value);
            if (testRepresentativeReads) {
                if (expectedRepresentativeIndexMap.containsKey(indexInFile) && expectedSetSizeMap.containsKey(record.getReadName())){
                    Assert.assertEquals(record.getAttribute("DI"), expectedRepresentativeIndexMap.get(indexInFile));
                    Assert.assertEquals(record.getAttribute("DS"), expectedSetSizeMap.get(record.getReadName()));
                }
            }
            indexInFile+=1;
        }
        CloserUtil.close(reader);

        // Ensure the program output the same number of records as were read in
        Assert.assertEquals(outputRecords, this.getNumberOfRecords(), ("saw " + outputRecords + " output records, vs. " + this.getNumberOfRecords() + " input records"));

        // Check the values written to metrics.txt against our input expectations
        final MetricsFile<DuplicationMetrics, Comparable<?>> metricsOutput = new MetricsFile<>();
        try{
            metricsOutput.read(new FileReader(metricsFile));
        }
        catch (final FileNotFoundException ex) {
            throw new PicardException("Metrics file not found: " + ex);
        }
        final List<DuplicationMetrics> g = metricsOutput.getMetrics();
        // expect getMetrics to return a collection with a single duplicateMetrics object
        Assert.assertEquals(metricsOutput.getMetrics().size(), 1);
        final DuplicationMetrics observedMetrics = metricsOutput.getMetrics().get(0);
        Assert.assertEquals(observedMetrics.UNPAIRED_READS_EXAMINED, expectedMetrics.UNPAIRED_READS_EXAMINED, "UNPAIRED_READS_EXAMINED does not match expected");
        Assert.assertEquals(observedMetrics.READ_PAIRS_EXAMINED, expectedMetrics.READ_PAIRS_EXAMINED, "READ_PAIRS_EXAMINED does not match expected");
        Assert.assertEquals(observedMetrics.UNMAPPED_READS, expectedMetrics.UNMAPPED_READS, "UNMAPPED_READS does not match expected");
        Assert.assertEquals(observedMetrics.UNPAIRED_READ_DUPLICATES, expectedMetrics.UNPAIRED_READ_DUPLICATES, "UNPAIRED_READ_DUPLICATES does not match expected");
        Assert.assertEquals(observedMetrics.READ_PAIR_DUPLICATES, expectedMetrics.READ_PAIR_DUPLICATES, "READ_PAIR_DUPLICATES does not match expected");
        Assert.assertEquals(observedMetrics.READ_PAIR_OPTICAL_DUPLICATES, expectedMetrics.READ_PAIR_OPTICAL_DUPLICATES, "READ_PAIR_OPTICAL_DUPLICATES does not match expected");
        Assert.assertEquals(observedMetrics.PERCENT_DUPLICATION, expectedMetrics.PERCENT_DUPLICATION, "PERCENT_DUPLICATION does not match expected");
        Assert.assertEquals(observedMetrics.ESTIMATED_LIBRARY_SIZE, expectedMetrics.ESTIMATED_LIBRARY_SIZE, "ESTIMATED_LIBRARY_SIZE does not match expected");
        Assert.assertEquals(observedMetrics.SECONDARY_OR_SUPPLEMENTARY_RDS, expectedMetrics.SECONDARY_OR_SUPPLEMENTARY_RDS, "SECONDARY_OR_SUPPLEMENTARY_RDS does not match expected");
    } finally {
        IOUtil.recursiveDelete(getOutputDir().toPath());
    }
}
 

@Test
public void testPoorQualityBases() throws IOException {
    final File reference = CHR_M_REFERENCE;
    final File testSamFile = File.createTempFile("CollectWgsMetrics", ".bam", TEST_DIR);
    testSamFile.deleteOnExit();

    /**
     *  Our test SAM looks as follows:
     *
     *   ----------   <- reads with great base qualities (60) ->  ----------
     *   ----------                                               ----------
     *   ----------                                               ----------
     *   **********   <- reads with poor base qualities (10) ->   **********
     *   **********                                               **********
     *   **********                                               **********
     *
     *  We exclude half of the bases because they are low quality.
     *  We do not exceed the coverage cap (3), thus none of the bases is excluded as such.
     *
     */

    final SAMRecordSetBuilder setBuilder = CollectWgsMetricsTestUtils.createTestSAMBuilder(reference, READ_GROUP_ID, SAMPLE, PLATFORM, LIBRARY);
    setBuilder.setReadLength(10);
    for (int i = 0; i < 3; i++){
        setBuilder.addPair("GreatBQRead:" + i, 0, 1, 30, false, false, "10M", "10M", false, true, 60);
    }

    for (int i = 0; i < 3; i++){
        setBuilder.addPair("PoorBQRead:" + i, 0, 1, 30, false, false, "10M", "10M", false, true, 10);
    }

    final SAMFileWriter writer = new SAMFileWriterFactory().setCreateIndex(true).makeBAMWriter(setBuilder.getHeader(), false, testSamFile);
    setBuilder.forEach(writer::addAlignment);
    writer.close();

    final File outfile = File.createTempFile("testExcludedBases-metrics", ".txt");
    outfile.deleteOnExit();

    final File chartOutFile = File.createTempFile("testExcludedBases",".pdf");
    chartOutFile.deleteOnExit();


    final String[] args = new String[] {
            "INPUT="  + testSamFile.getAbsolutePath(),
            "OUTPUT=" + outfile.getAbsolutePath(),
            "REFERENCE_SEQUENCE=" + reference.getAbsolutePath(),
            "INCLUDE_BQ_HISTOGRAM=true",
            "COVERAGE_CAP=3",
            "CHART_OUTPUT=" + chartOutFile.getAbsolutePath()
    };

    Assert.assertEquals(runPicardCommandLine(args), 0);
    Assert.assertTrue(chartOutFile.exists());

    final MetricsFile<WgsMetrics, Integer> output = new MetricsFile<>();
    output.read(new FileReader(outfile));

    final WgsMetrics metrics = output.getMetrics().get(0);
    final WgsMetrics nonZeroMetrics = output.getMetrics().get(1);


    // Some metrics should not change between with and without zero
    Assert.assertEquals(nonZeroMetrics.PCT_EXC_BASEQ, metrics.PCT_EXC_BASEQ);
    Assert.assertEquals(nonZeroMetrics.PCT_EXC_CAPPED, metrics.PCT_EXC_CAPPED);

    // Other metrics change when we ignore the zero depth bin
    Assert.assertEquals(nonZeroMetrics.GENOME_TERRITORY, 20);
    Assert.assertEquals(nonZeroMetrics.MEAN_COVERAGE, 3.0);
}
 

private void doTest(final String[] args, final File metrics, final int expectedRetVal, final int expectedNMetrics, final CrosscheckMetric.DataType expectedType, final boolean expectAllMatch) throws IOException, NoSuchFieldException {

        final CrosscheckReadGroupFingerprints crossChecker = new CrosscheckReadGroupFingerprints();
        Assert.assertEquals(crossChecker.instanceMain(args), expectedRetVal);

        final MetricsFile<CrosscheckMetric, Comparable<?>> metricsOutput = new MetricsFile<>();
        metricsOutput.read(new FileReader(metrics));

        Assert.assertTrue(metricsOutput.getMetrics().stream()
                .allMatch(m -> m.DATA_TYPE == expectedType));

        Assert.assertTrue(metricsOutput.getMetrics().stream()
                .allMatch(m -> m.LOD_SCORE_NORMAL_TUMOR != null));
        Assert.assertTrue(metricsOutput.getMetrics().stream()
                .allMatch(m -> m.LOD_SCORE != null));
        Assert.assertTrue(metricsOutput.getMetrics().stream()
                .allMatch(m -> m.LOD_SCORE_TUMOR_NORMAL != null));

        if (expectAllMatch) {
            Assert.assertTrue(metricsOutput.getMetrics().stream()
                    .allMatch(m -> m.RESULT == CrosscheckMetric.FingerprintResult.INCONCLUSIVE ||
                            m.RESULT.isMatch() == m.LEFT_SAMPLE.equals(m.RIGHT_SAMPLE)));
        } else if (expectedRetVal == 0) {
            Assert.assertTrue(metricsOutput.getMetrics().stream()
                    .allMatch(m -> m.RESULT == CrosscheckMetric.FingerprintResult.INCONCLUSIVE ||
                            m.RESULT.isExpected()));
        } else {
            Assert.assertTrue(metricsOutput.getMetrics().stream()
                    .anyMatch(m -> !m.RESULT.isExpected()));
        }

        Assert.assertEquals(metricsOutput.getMetrics().size(), expectedNMetrics);

        // at the readgroup level things aren't always conclusive...
        if (!metricsOutput.getMetrics().isEmpty() && expectedType != CrosscheckMetric.DataType.READGROUP) {
            Assert.assertTrue(metricsOutput.getMetrics().stream()
                    .anyMatch(m -> m.RESULT != CrosscheckMetric.FingerprintResult.INCONCLUSIVE));
        }

        //check that fields that should have an actual value, indeed do
        for (final String fieldName : lookupMap.get(expectedType)) {
            final Field field = CrosscheckMetric.class.getField(fieldName);
            Assert.assertTrue(metricsOutput.getMetrics().stream().allMatch(m -> {
                try {
                    return field.get(m) != multipleValuesString && field.get(m) != null;
                } catch (IllegalAccessException e) {
                    e.printStackTrace();
                    return false;
                }
            }));
        }
    }
 

@Test( dataProvider = "md")
public void testMarkDuplicatesSparkDeletingOpticalDuplicateReads(
        final File[] inputFiles, final long totalExpected, final long dupsExpected,
        Map<String, List<String>> metricsExpected) throws IOException {

    ArgumentsBuilder args = new ArgumentsBuilder();
    for (File input : inputFiles) {
        args.add(StandardArgumentDefinitions.INPUT_LONG_NAME,input.getPath());
    }
    args.addRaw("--" + StandardArgumentDefinitions.OUTPUT_LONG_NAME);

    File outputFile = createTempFile("markdups", ".bam");
    outputFile.delete();
    args.addRaw(outputFile.getAbsolutePath());

    args.addRaw("--" + StandardArgumentDefinitions.METRICS_FILE_LONG_NAME);
    File metricsFile = createTempFile("markdups_metrics", ".txt");
    args.addRaw(metricsFile.getAbsolutePath());

    args.addRaw("--" + MarkDuplicatesSparkArgumentCollection.REMOVE_SEQUENCING_DUPLICATE_READS);

    runCommandLine(args.getArgsArray());

    Assert.assertTrue(outputFile.exists(), "Can't find expected MarkDuplicates output file at " + outputFile.getAbsolutePath());

    int totalReads = 0;
    int duplicateReads = 0;
    try (final ReadsDataSource outputReads = new ReadsPathDataSource(outputFile.toPath())) {
        for (GATKRead read : outputReads) {
            ++totalReads;

            if (read.isDuplicate()) {
                ++duplicateReads;
            }
        }
    }

    final MetricsFile<GATKDuplicationMetrics, Comparable<?>> metricsOutput = new MetricsFile<>();
    try {
        metricsOutput.read(new FileReader(metricsFile));
    } catch (final FileNotFoundException ex) {
        System.err.println("Metrics file not found: " + ex);
    }
    final List<GATKDuplicationMetrics> nonEmptyMetrics = getGatkDuplicationMetrics(metricsOutput);

    // Asserting
    int expectedOpticalDuplicatesGroups = 0;
    for (int i = 0; i < nonEmptyMetrics.size(); i++ ){
        final GATKDuplicationMetrics observedMetrics = nonEmptyMetrics.get(i);
        List<?> expectedList = metricsExpected.get(observedMetrics.LIBRARY);
        expectedOpticalDuplicatesGroups += (Long) expectedList.get(5);
    }
    // NOTE: this test will fail if we add a more comprehensive example set with optical duplicates containing secondary/supplementary reads because of how the test is counting optical duplicate reads.
    Assert.assertEquals(totalReads, totalExpected - expectedOpticalDuplicatesGroups*2, "Wrong number of reads in output BAM");
    Assert.assertEquals(duplicateReads, dupsExpected - expectedOpticalDuplicatesGroups*2, "Wrong number of duplicate reads in output BAM");
}
 

@Test
public void testBisulfite() throws IOException {
    final File input = new File(TEST_DATA_DIR, "summary_alignment_bisulfite_test.sam");
    final File reference = new File(TEST_DATA_DIR, "summary_alignment_stats_test.fasta");
    final File outfile = File.createTempFile("alignmentMetrics", ".txt");
    outfile.deleteOnExit();
    final String[] args = new String[]{
            "INPUT=" + input.getAbsolutePath(),
            "OUTPUT=" + outfile.getAbsolutePath(),
            "REFERENCE_SEQUENCE=" + reference.getAbsolutePath(),
            "IS_BISULFITE_SEQUENCED=true"
    };
    Assert.assertEquals(runPicardCommandLine(args), 0);

    final NumberFormat format = NumberFormat.getInstance();
    format.setMaximumFractionDigits(4);

    final MetricsFile<AlignmentSummaryMetrics, Comparable<?>> output = new MetricsFile<>();
    try (FileReader reader = new FileReader(outfile)) {
        output.read(reader);
    }

    for (final AlignmentSummaryMetrics metrics : output.getMetrics()) {
        Assert.assertEquals(metrics.MEAN_READ_LENGTH, 101.0);
        switch (metrics.CATEGORY) {
            case FIRST_OF_PAIR:
                // 19 no-calls, one potentially methylated base, one mismatch at a potentially methylated base
                Assert.assertEquals(metrics.TOTAL_READS, 1);
                Assert.assertEquals(metrics.PF_READS, 1);
                Assert.assertEquals(metrics.PF_HQ_ALIGNED_BASES, 101);
                Assert.assertEquals(metrics.PF_HQ_MEDIAN_MISMATCHES, 21.0);
                Assert.assertEquals(metrics.PF_ALIGNED_BASES, 101);
                Assert.assertEquals(metrics.PF_MISMATCH_RATE, 0.212121 /*21D/99D*/);
                Assert.assertEquals(metrics.BAD_CYCLES, 21);
                Assert.assertEquals(format.format(metrics.PF_HQ_ERROR_RATE), format.format(21 / (double) 99));
                break;
            case SECOND_OF_PAIR:
                // Three no-calls, two potentially methylated bases
                Assert.assertEquals(metrics.TOTAL_READS, 1);
                Assert.assertEquals(metrics.PF_READS, 1);
                Assert.assertEquals(metrics.PF_HQ_ALIGNED_BASES, 101);
                Assert.assertEquals(metrics.PF_HQ_MEDIAN_MISMATCHES, 4.0);
                Assert.assertEquals(metrics.PF_ALIGNED_BASES, 101);
                Assert.assertEquals(metrics.PF_MISMATCH_RATE, /*4D/99D*/0.040404);
                Assert.assertEquals(metrics.BAD_CYCLES, 4);
                Assert.assertEquals(format.format(metrics.PF_HQ_ERROR_RATE), format.format(4 / (double) 99));
                break;
            case PAIR:
                Assert.assertEquals(metrics.TOTAL_READS, 2);
                Assert.assertEquals(metrics.PF_READS, 2);
                Assert.assertEquals(metrics.PF_HQ_ALIGNED_BASES, 202);
                Assert.assertEquals(metrics.PF_HQ_MEDIAN_MISMATCHES, 12.5D);
                Assert.assertEquals(metrics.PF_ALIGNED_BASES, 202);
                Assert.assertEquals(metrics.PF_MISMATCH_RATE, 0.126263);// 25D/198D
                Assert.assertEquals(metrics.BAD_CYCLES, 25);
                Assert.assertEquals(format.format(metrics.PF_HQ_ERROR_RATE), format.format(25 / (double) 198));
                break;
            case UNPAIRED:
            default:
                Assert.fail("Data does not contain this category: " + metrics.CATEGORY);
        }
    }
}