Java Code Examples for org.apache.nifi.util.TestRunner#getCounterValue()

The following examples show how to use org.apache.nifi.util.TestRunner#getCounterValue() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestJSONToAvroProcessor.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertJSONToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertJSONToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(FAILURE_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 1 row", 3, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals("Should set an error message",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 2
Source File: TestCounter.java    From nifi-scripting-samples with Apache License 2.0 6 votes vote down vote up
/**
 * Demonstrates writing to counters
 * @throws Exception
 */
@Test
public void testCounterJavascript() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new ExecuteScript());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(SCRIPT_ENGINE, "ECMAScript");
    runner.setProperty(ScriptingComponentUtils.SCRIPT_FILE, "src/test/resources/executescript/counter/counter.js");
    runner.setProperty(ScriptingComponentUtils.MODULES, "src/test/resources/executescript");
    runner.assertValid();

    runner.enqueue("sample text".getBytes(StandardCharsets.UTF_8));
    runner.run();

    runner.assertAllFlowFilesTransferred("success", 1);
    double counterValue = runner.getCounterValue("SampleScriptCounter");
    Assert.assertEquals(1d, counterValue, 0.01d);
}
 
Example 3
Source File: TestCSVToAvroProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAlternateCharset() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(ConvertCSVToAvro.CHARSET, "utf16");
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT, Charset.forName("UTF-16")));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 4
Source File: TestCSVToAvroProcessor.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAlternateCharset() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(ConvertCSVToAvro.CHARSET, "utf16");
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT, Charset.forName("UTF-16")));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 5
Source File: TestJSONToAvroProcessor.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyContent() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertJSONToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertJSONToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(""));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 0 row", 0, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals("Should set an error message",
            "No incoming records", incompatible.getAttribute("errors"));
}
 
Example 6
Source File: TestCSVToAvroProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testBasicConversionNoErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor("1,green\n2,blue,\n3,grey,12.95"));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 3 rows", 3, converted);
    Assert.assertEquals("Should reject 0 row", 0, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 0);
}
 
Example 7
Source File: TestKiteStorageProcessor.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorruptedBlocks() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(StoreInKiteDataset.class);
    runner.setProperty(StoreInKiteDataset.KITE_DATASET_URI, datasetUri);
    runner.assertValid();

    List<Record> records = Lists.newArrayList();
    for (int i = 0; i < 10000; i += 1) {
        String num = String.valueOf(i);
        records.add(user(num, num + "@example.com"));
    }

    runner.enqueue(invalidStreamFor(records));
    runner.run();

    long stored = runner.getCounterValue("Stored records");
    Assert.assertTrue("Should store some readable values",
            0 < stored && stored < 10000);

    runner.assertAllFlowFilesTransferred("success", 1);
}
 
Example 8
Source File: TestKiteStorageProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorruptedBlocks() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(StoreInKiteDataset.class);
    runner.setProperty(StoreInKiteDataset.KITE_DATASET_URI, datasetUri);
    runner.assertValid();

    List<Record> records = Lists.newArrayList();
    for (int i = 0; i < 10000; i += 1) {
        String num = String.valueOf(i);
        records.add(user(num, num + "@example.com"));
    }

    runner.enqueue(invalidStreamFor(records));
    runner.run();

    long stored = runner.getCounterValue("Stored records");
    Assert.assertTrue("Should store some readable values",
            0 < stored && stored < 10000);

    runner.assertAllFlowFilesTransferred("success", 1);
}
 
Example 9
Source File: TestJSONToAvroProcessor.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testBasicConversionNoErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertJSONToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertJSONToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(
            "{\"id\": 1,\"color\": \"green\"}\n" +
                    "{\"id\": 2, \"color\": \"grey\", \"price\": 12.95 }"));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 0 row", 0, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 0);
}
 
Example 10
Source File: TestJSONToAvroProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertJSONToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertJSONToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(FAILURE_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 1 row", 3, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals("Should set an error message",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 11
Source File: TestJSONToAvroProcessor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testBasicConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertJSONToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertJSONToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(JSON_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 3 rows", 3, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent = new String(runner.getContentAsByteArray(incompatible),
            StandardCharsets.UTF_8);
    Assert.assertEquals("Should reject an invalid string and double",
            JSON_CONTENT, failureContent);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 12
Source File: TestCSVToAvroProcessor.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Basic test for tab separated files, similar to #test
 */
@Test
public void testTabSeparatedConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(ConvertCSVToAvro.DELIMITER, "\\t");
    runner.assertValid();

    runner.enqueue(streamFor(TSV_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent = new String(runner.getContentAsByteArray(incompatible),
            StandardCharsets.UTF_8);

    Assert.assertEquals("Should reject an invalid string and double",
            TSV_CONTENT, failureContent);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 13
Source File: TestCSVToAvroProcessor.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testBasicConversionWithCompression() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(AbstractKiteConvertProcessor.COMPRESSION_TYPE, CodecType.DEFLATE.toString());
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent = new String(runner.getContentAsByteArray(incompatible),
            StandardCharsets.UTF_8);
    Assert.assertEquals("Should reject an invalid string and double",
            CSV_CONTENT, failureContent);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 14
Source File: TestJSONToAvroProcessor.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testBasicConversionWithCompression() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertJSONToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertJSONToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(AbstractKiteConvertProcessor.COMPRESSION_TYPE, CodecType.NONE.toString());
    runner.assertValid();

    runner.enqueue(streamFor(JSON_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 3 rows", 3, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent = new String(runner.getContentAsByteArray(incompatible),
            StandardCharsets.UTF_8);
    Assert.assertEquals("Should reject an invalid string and double",
            JSON_CONTENT, failureContent);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 15
Source File: TestCSVToAvroProcessor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testBasicConversionWithCompression() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(AbstractKiteConvertProcessor.COMPRESSION_TYPE, CodecType.DEFLATE.toString());
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent = new String(runner.getContentAsByteArray(incompatible),
            StandardCharsets.UTF_8);
    Assert.assertEquals("Should reject an invalid string and double",
            CSV_CONTENT, failureContent);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 16
Source File: TestCSVToAvroProcessor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testBasicConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent = new String(runner.getContentAsByteArray(incompatible),
            StandardCharsets.UTF_8);
    Assert.assertEquals("Should reject an invalid string and double",
            CSV_CONTENT, failureContent);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 17
Source File: TestCSVToAvroProcessor.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Basic test for tab separated files, similar to #test
 */
@Test
public void testTabSeparatedConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(ConvertCSVToAvro.DELIMITER, "\\t");
    runner.assertValid();

    runner.enqueue(streamFor(TSV_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent = new String(runner.getContentAsByteArray(incompatible),
            StandardCharsets.UTF_8);

    Assert.assertEquals("Should reject an invalid string and double",
            TSV_CONTENT, failureContent);
    Assert.assertEquals("Should accumulate error messages",
            FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
 
Example 18
Source File: TestConvertAvroSchema.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testNestedConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertAvroSchema.class);
    runner.assertNotValid();
    runner.setProperty(ConvertAvroSchema.INPUT_SCHEMA,
            TestAvroRecordConverter.NESTED_RECORD_SCHEMA.toString());
    runner.setProperty(ConvertAvroSchema.OUTPUT_SCHEMA,
            TestAvroRecordConverter.UNNESTED_OUTPUT_SCHEMA.toString());
    runner.setProperty("parent.id", "parentId");
    runner.assertValid();

    // Two valid rows
    Record goodRecord1 = dataNested(1L, "200", null, null);
    Record goodRecord2 = dataNested(2L, "300", 5L, "ParentCompany");
    List<Record> input = Lists.newArrayList(goodRecord1, goodRecord2);

    runner.enqueue(streamFor(input));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 0 rows", 0, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);

    GenericDatumReader<Record> successReader = new GenericDatumReader<Record>(
            TestAvroRecordConverter.UNNESTED_OUTPUT_SCHEMA);
    DataFileStream<Record> successStream = new DataFileStream<Record>(
            new ByteArrayInputStream(runner.getContentAsByteArray(runner
                    .getFlowFilesForRelationship("success").get(0))),
            successReader);
    int count = 0;
    for (Record r : successStream) {
        if (count == 0) {
            Assert.assertEquals(convertNested(goodRecord1), r);
        } else {
            Assert.assertEquals(convertNested(goodRecord2), r);
        }
        count++;
    }
    successStream.close();
    Assert.assertEquals(2, count);
}
 
Example 19
Source File: TestCSVToAvroProcessor.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testExpressionLanguageBasedCSVProperties() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.setProperty(ConvertCSVToAvro.DELIMITER, "${csv.delimiter}");
    runner.setProperty(ConvertCSVToAvro.QUOTE, "${csv.quote}");

    HashMap<String, String> flowFileAttributes = new HashMap<String,String>();
    flowFileAttributes.put("csv.delimiter", "|");
    flowFileAttributes.put("csv.quote", "~");

    runner.enqueue(streamFor("1|green\n2|~blue|field~|\n3|grey|12.95"), flowFileAttributes);
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 3 rows", 3, converted);
    Assert.assertEquals("Should reject 0 row", 0, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 0);

    final InputStream in = new ByteArrayInputStream(runner.getFlowFilesForRelationship("success").get(0).toByteArray());
    final DatumReader<GenericRecord> datumReader = new GenericDatumReader<>();
    try (DataFileStream<GenericRecord> dataFileReader = new DataFileStream<>(in, datumReader)) {
        assertTrue(dataFileReader.hasNext());
        GenericRecord record = dataFileReader.next();
        assertEquals(1L, record.get("id"));
        assertEquals("green", record.get("color").toString());
        assertNull(record.get("price"));

        assertTrue(dataFileReader.hasNext());
        record = dataFileReader.next();
        assertEquals(2L, record.get("id"));
        assertEquals("blue|field", record.get("color").toString());
        assertNull(record.get("price"));

        assertTrue(dataFileReader.hasNext());
        record = dataFileReader.next();
        assertEquals(3L, record.get("id"));
        assertEquals("grey", record.get("color").toString());
        assertEquals(12.95, record.get("price"));
    }
}
 
Example 20
Source File: TestConvertAvroSchema.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testNestedConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertAvroSchema.class);
    runner.assertNotValid();
    runner.setProperty(ConvertAvroSchema.INPUT_SCHEMA,
            TestAvroRecordConverter.NESTED_RECORD_SCHEMA.toString());
    runner.setProperty(ConvertAvroSchema.OUTPUT_SCHEMA,
            TestAvroRecordConverter.UNNESTED_OUTPUT_SCHEMA.toString());
    runner.setProperty("parent.id", "parentId");
    runner.assertValid();

    // Two valid rows
    Record goodRecord1 = dataNested(1L, "200", null, null);
    Record goodRecord2 = dataNested(2L, "300", 5L, "ParentCompany");
    List<Record> input = Lists.newArrayList(goodRecord1, goodRecord2);

    runner.enqueue(streamFor(input));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 0 rows", 0, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);

    GenericDatumReader<Record> successReader = new GenericDatumReader<Record>(
            TestAvroRecordConverter.UNNESTED_OUTPUT_SCHEMA);
    DataFileStream<Record> successStream = new DataFileStream<Record>(
            new ByteArrayInputStream(runner.getContentAsByteArray(runner
                    .getFlowFilesForRelationship("success").get(0))),
            successReader);
    int count = 0;
    for (Record r : successStream) {
        if (count == 0) {
            Assert.assertEquals(convertNested(goodRecord1), r);
        } else {
            Assert.assertEquals(convertNested(goodRecord2), r);
        }
        count++;
    }
    successStream.close();
    Assert.assertEquals(2, count);
}