Java Code Examples for org.apache.nifi.util.TestRunner#clearTransferState()

@Test
public void testBzip2Decompress() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(CompressContent.class);
    runner.setProperty(CompressContent.MODE, "decompress");
    runner.setProperty(CompressContent.COMPRESSION_FORMAT, "bzip2");
    runner.setProperty(CompressContent.UPDATE_FILENAME, "true");

    runner.enqueue(Paths.get("src/test/resources/CompressedData/SampleFile.txt.bz2"));
    runner.run();

    runner.assertAllFlowFilesTransferred(CompressContent.REL_SUCCESS, 1);
    MockFlowFile flowFile = runner.getFlowFilesForRelationship(CompressContent.REL_SUCCESS).get(0);
    flowFile.assertContentEquals(Paths.get("src/test/resources/CompressedData/SampleFile.txt"));
    flowFile.assertAttributeEquals("filename", "SampleFile.txt");

    runner.clearTransferState();
    runner.enqueue(Paths.get("src/test/resources/CompressedData/SampleFile1.txt.bz2"));
    runner.run();

    runner.assertAllFlowFilesTransferred(CompressContent.REL_SUCCESS, 1);
    flowFile = runner.getFlowFilesForRelationship(CompressContent.REL_SUCCESS).get(0);
    flowFile.assertContentEquals(Paths.get("src/test/resources/CompressedData/SampleFile.txt"));
    flowFile.assertAttributeEquals("filename", "SampleFile1.txt");
}
 

@Test
public void testClusterMonitorInactivity() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new TestableProcessor(10000));
    runner.setClustered(true);
    runner.setPrimaryNode(false);
    runner.setProperty(MonitorActivity.MONITORING_SCOPE, MonitorActivity.SCOPE_CLUSTER);
    runner.setProperty(MonitorActivity.THRESHOLD, "100 ms");
    runner.setProperty(MonitorActivity.COPY_ATTRIBUTES, "true");

    // Becomes inactive
    runner.run();
    runner.assertAllFlowFilesTransferred(MonitorActivity.REL_INACTIVE);
    final List<MockFlowFile> inactiveFiles = runner.getFlowFilesForRelationship(MonitorActivity.REL_INACTIVE);
    assertEquals(1, inactiveFiles.size());

    final MockFlowFile inactiveFile = inactiveFiles.get(0);
    assertNotNull(inactiveFile.getAttribute("inactivityStartMillis"));
    assertNotNull(inactiveFile.getAttribute("inactivityDurationMillis"));

    runner.clearTransferState();

}
 

@Test
public void testStorageClasses() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());

    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);

    Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());

    for (StorageClass storageClass : StorageClass.values()) {
        runner.setProperty(PutS3Object.STORAGE_CLASS, storageClass.name());

        final Map<String, String> attrs = new HashMap<>();
        attrs.put("filename", "testStorageClasses/small_" + storageClass.name() + ".txt");
        runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs);

        runner.run();

        runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
        FlowFile file = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS).get(0);
        Assert.assertEquals(storageClass.toString(), file.getAttribute(PutS3Object.S3_STORAGECLASS_ATTR_KEY));

        runner.clearTransferState();
    }
}
 

@Test
public void testClusterMonitorInactivityFallbackToNodeScope() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new TestableProcessor(10000));
    runner.setClustered(false);
    runner.setPrimaryNode(false);
    runner.setProperty(MonitorActivity.MONITORING_SCOPE, MonitorActivity.SCOPE_CLUSTER);
    runner.setProperty(MonitorActivity.THRESHOLD, "100 ms");
    runner.setProperty(MonitorActivity.COPY_ATTRIBUTES, "true");

    // Becomes inactive
    runner.run();
    runner.assertAllFlowFilesTransferred(MonitorActivity.REL_INACTIVE);
    final List<MockFlowFile> inactiveFiles = runner.getFlowFilesForRelationship(MonitorActivity.REL_INACTIVE);
    assertEquals(1, inactiveFiles.size());

    final MockFlowFile inactiveFile = inactiveFiles.get(0);
    assertNotNull(inactiveFile.getAttribute("inactivityStartMillis"));
    assertNotNull(inactiveFile.getAttribute("inactivityDurationMillis"));

    runner.clearTransferState();

}
 

@Test
public void testNiFi_4759_Regressions() {
    TestRunner runner = init(PutMongo.class);
    String[] upserts = new String[]{
            "{ \"_id\": \"12345\", \"$set\": { \"msg\": \"Hello, world\" } }",
            "{ \"_id\": \"5a5617b9c1f5de6d8276e87d\", \"$set\": { \"msg\": \"Hello, world\" } }",
            "{ \"updateKey\": \"12345\", \"$set\": { \"msg\": \"Hello, world\" } }"
    };

    String[] updateKeyProps = new String[] { "_id", "_id", "updateKey" };
    Object[] updateKeys = new Object[] { "12345", new ObjectId("5a5617b9c1f5de6d8276e87d"), "12345" };
    int index = 0;

    runner.setProperty(PutMongo.UPDATE_MODE, PutMongo.UPDATE_WITH_OPERATORS);
    runner.setProperty(PutMongo.MODE, "update");
    runner.setProperty(PutMongo.UPSERT, "true");

    final int LIMIT = 2;

    for (String upsert : upserts) {
        runner.setProperty(PutMongo.UPDATE_QUERY_KEY, updateKeyProps[index]);
        for (int x = 0; x < LIMIT; x++) {
            runner.enqueue(upsert);
        }
        runner.run(LIMIT, true, true);
        runner.assertTransferCount(PutMongo.REL_FAILURE, 0);
        runner.assertTransferCount(PutMongo.REL_SUCCESS, LIMIT);

        Document query = new Document(updateKeyProps[index], updateKeys[index]);
        Document result = collection.find(query).first();
        Assert.assertNotNull("Result was null", result);
        Assert.assertEquals("Count was wrong", 1, collection.count(query));
        runner.clearTransferState();
        index++;
    }
}
 

@Test(timeout=5000)
public void testFirstRunNoMessages() throws InterruptedException, IOException {
    // don't use the TestableProcessor, we want the real timestamp from @OnScheduled
    final TestRunner runner = TestRunners.newTestRunner(new MonitorActivity());
    runner.setProperty(MonitorActivity.CONTINUALLY_SEND_MESSAGES, "false");
    int threshold = 100;
    boolean rerun = false;
    do {
        rerun = false;
        runner.setProperty(MonitorActivity.THRESHOLD, threshold + " millis");

        Thread.sleep(1000L);

        // shouldn't generate inactivity b/c run() will reset the lastSuccessfulTransfer if @OnSchedule & onTrigger
        // does not  get called more than MonitorActivity.THRESHOLD apart
        runner.run();
        runner.assertTransferCount(MonitorActivity.REL_SUCCESS, 0);
        List<MockFlowFile> inactiveFlowFiles = runner.getFlowFilesForRelationship(MonitorActivity.REL_INACTIVE);
        if (inactiveFlowFiles.size() == 1) {
            // Seems Threshold was not sufficient, which has caused One inactive message.
            // Step-up and rerun the test until successful or jUnit times out
            threshold += threshold;
            rerun = true;
        } else {
            runner.assertTransferCount(MonitorActivity.REL_INACTIVE, 0);
        }
        runner.assertTransferCount(MonitorActivity.REL_ACTIVITY_RESTORED, 0);
        runner.clearTransferState();
    }
    while(rerun);
}
 

@Test
public void testHeaderAndSplit() {
    final TestRunner runner = TestRunners.newTestRunner(new ValidateCsv());
    runner.setProperty(ValidateCsv.DELIMITER_CHARACTER, ",");
    runner.setProperty(ValidateCsv.END_OF_LINE_CHARACTER, "\n");
    runner.setProperty(ValidateCsv.QUOTE_CHARACTER, "\"");
    runner.setProperty(ValidateCsv.HEADER, "true");
    runner.setProperty(ValidateCsv.VALIDATION_STRATEGY, ValidateCsv.VALIDATE_LINES_INDIVIDUALLY);

    runner.setProperty(ValidateCsv.SCHEMA, "Null, ParseDate(\"dd/MM/yyyy\"), Optional(ParseDouble())");

    runner.enqueue("Name,Birthdate,Weight\nJohn,22/11/1954,63.2\nBob,01/03/2004,45.0");
    runner.run();

    runner.assertTransferCount(ValidateCsv.REL_VALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_VALID).get(0).assertContentEquals("Name,Birthdate,Weight\nJohn,22/11/1954,63.2\nBob,01/03/2004,45.0");
    runner.assertTransferCount(ValidateCsv.REL_INVALID, 0);

    runner.clearTransferState();

    runner.enqueue("Name,Birthdate,Weight\nJohn,22/11/1954,63a2\nBob,01/032004,45.0");
    runner.run();

    runner.assertTransferCount(ValidateCsv.REL_VALID, 0);
    runner.assertTransferCount(ValidateCsv.REL_INVALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_INVALID).get(0).assertContentEquals("Name,Birthdate,Weight\nJohn,22/11/1954,63a2\nBob,01/032004,45.0");

    runner.clearTransferState();

    runner.enqueue("Name,Birthdate,Weight\nJohn,22/111954,63.2\nBob,01/03/2004,45.0");
    runner.run();

    runner.assertTransferCount(ValidateCsv.REL_VALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_VALID).get(0).assertContentEquals("Name,Birthdate,Weight\nBob,01/03/2004,45.0");
    runner.assertTransferCount(ValidateCsv.REL_INVALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_INVALID).get(0).assertContentEquals("Name,Birthdate,Weight\nJohn,22/111954,63.2");
}
 

@Test
public void testWithSplitThatStartsWithNewLine() {
    final TestRunner splitRunner = TestRunners.newTestRunner(new SplitText());
    splitRunner.setProperty(SplitText.HEADER_LINE_COUNT, "1");
    splitRunner.setProperty(SplitText.LINE_SPLIT_COUNT, "3");
    splitRunner.setProperty(SplitText.REMOVE_TRAILING_NEWLINES, "true");

    splitRunner.enqueue("H1\n1\n2\n3\n\n\n4\n");

    splitRunner.run();
    splitRunner.assertTransferCount(SplitText.REL_SPLITS, 2);
    splitRunner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    splitRunner.assertTransferCount(SplitText.REL_FAILURE, 0);

    final List<MockFlowFile> splits = splitRunner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splits.get(0).assertContentEquals("H1\n1\n2\n3");
    splits.get(1).assertContentEquals("H1\n\n\n4");

    splitRunner.clearTransferState();

    splitRunner.setProperty(SplitText.HEADER_LINE_COUNT, "0");
    splitRunner.enqueue("1\n2\n3\n\n\n4\n");

    splitRunner.run();
    splitRunner.assertTransferCount(SplitText.REL_SPLITS, 2);
    splitRunner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    splitRunner.assertTransferCount(SplitText.REL_FAILURE, 0);

    final List<MockFlowFile> splitsWithoutHeader = splitRunner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splitsWithoutHeader.get(0).assertContentEquals("1\n2\n3");
    splitsWithoutHeader.get(1).assertContentEquals("\n\n4");
}
 

@Test
public void testSplitWithCarriageReturnAndNewLines() {
    final TestRunner runner = TestRunners.newTestRunner(new SplitText());
    runner.setProperty(SplitText.HEADER_LINE_COUNT, "2");
    runner.setProperty(SplitText.LINE_SPLIT_COUNT, "3");

    runner.enqueue("H1\r\nH2\r\n1\r\n2\r\n3\r\n\r\n\r\n\r\n\r\n\r\n\r\n10\r\n11\r\n12\r\n");
    runner.run();

    runner.assertTransferCount(SplitText.REL_FAILURE, 0);
    runner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    runner.getFlowFilesForRelationship(SplitText.REL_ORIGINAL).get(0).assertAttributeEquals(SplitText.FRAGMENT_COUNT, "4");
    runner.assertTransferCount(SplitText.REL_SPLITS, 4);

    final List<MockFlowFile> splits =runner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splits.get(0).assertContentEquals("H1\r\nH2\r\n1\r\n2\r\n3");
    splits.get(1).assertContentEquals("H1\r\nH2");
    splits.get(2).assertContentEquals("H1\r\nH2");
    splits.get(3).assertContentEquals("H1\r\nH2\r\n10\r\n11\r\n12");

    runner.clearTransferState();
    runner.setProperty(SplitText.HEADER_LINE_COUNT, "0");
    runner.setProperty(SplitText.LINE_SPLIT_COUNT, "3");

    runner.enqueue("1\r\n2\r\n3\r\n\r\n\r\n\r\n\r\n\r\n\r\n10\r\n11\r\n12\r\n");
    runner.run();

    runner.assertTransferCount(SplitText.REL_FAILURE, 0);
    runner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    runner.getFlowFilesForRelationship(SplitText.REL_ORIGINAL).get(0).assertAttributeEquals(SplitText.FRAGMENT_COUNT, "2");
    runner.assertTransferCount(SplitText.REL_SPLITS, 2);

    final List<MockFlowFile> splitsWithNoHeader =runner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splitsWithNoHeader.get(0).assertContentEquals("1\r\n2\r\n3");
    splitsWithNoHeader.get(1).assertContentEquals("10\r\n11\r\n12");
}
 

@Test
public void testWithLotsOfBlankLinesAtEnd() {
    // verify with header lines
    final TestRunner splitRunner = TestRunners.newTestRunner(new SplitText());
    splitRunner.setProperty(SplitText.HEADER_LINE_COUNT, "2");
    splitRunner.setProperty(SplitText.LINE_SPLIT_COUNT, "10");
    splitRunner.setProperty(SplitText.REMOVE_TRAILING_NEWLINES, "true");

    splitRunner.enqueue("H1\nH2\n1\n\n\n");

    splitRunner.run();
    splitRunner.assertTransferCount(SplitText.REL_SPLITS, 1);
    splitRunner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    splitRunner.assertTransferCount(SplitText.REL_FAILURE, 0);

    final List<MockFlowFile> splits = splitRunner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splits.get(0).assertContentEquals("H1\nH2\n1");

    // verify without headers
    splitRunner.clearTransferState();
    splitRunner.setProperty(SplitText.HEADER_LINE_COUNT, "0");

    splitRunner.enqueue("1\n2\n\n\n\n");
    splitRunner.run();

    splitRunner.assertTransferCount(SplitText.REL_SPLITS, 1);
    splitRunner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    splitRunner.assertTransferCount(SplitText.REL_FAILURE, 0);

    final List<MockFlowFile> splitsWithNoHeader = splitRunner.getFlowFilesForRelationship(SplitText.REL_SPLITS);
    splitsWithNoHeader.get(0).assertContentEquals("1\n2");
}
 

@Test
public void testAllMatch() {
    final TestRunner runner = TestRunners.newTestRunner(new ScanAttribute());
    runner.setProperty(ScanAttribute.DICTIONARY_FILE, "src/test/resources/ScanAttribute/dictionary1");
    runner.setProperty(ScanAttribute.MATCHING_CRITERIA, ScanAttribute.MATCH_CRITERIA_ALL);
    runner.setProperty(ScanAttribute.ATTRIBUTE_PATTERN, "a.*");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "world");

    runner.enqueue(new byte[0], attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_MATCHED, 1);
    runner.clearTransferState();

    attributes.remove("abc");
    runner.enqueue(new byte[0], attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_MATCHED, 1);
    runner.clearTransferState();

    attributes.put("abc", "world");
    attributes.put("a world", "apart");
    runner.enqueue(new byte[0], attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_UNMATCHED, 1);
    runner.clearTransferState();

    attributes.put("abc", "world");
    attributes.put("a world", "hello");
    runner.enqueue(new byte[0], attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_MATCHED, 1);
}
 

@Test
public void testFileCountRate() throws InterruptedException {
    final TestRunner runner = TestRunners.newTestRunner(new ControlRate());
    runner.setProperty(ControlRate.RATE_CONTROL_CRITERIA, ControlRate.FLOWFILE_RATE);
    runner.setProperty(ControlRate.MAX_RATE, "3");
    runner.setProperty(ControlRate.TIME_PERIOD, "1 sec");

    runner.enqueue("test data 1");
    runner.enqueue("test data 2");
    runner.enqueue("test data 3");
    runner.enqueue("test data 4");

    runner.run(4, false);

    runner.assertAllFlowFilesTransferred(ControlRate.REL_SUCCESS, 3);
    runner.clearTransferState();

    runner.run(50, false);
    runner.assertTransferCount(ControlRate.REL_SUCCESS, 0);
    runner.assertTransferCount(ControlRate.REL_FAILURE, 0);
    runner.assertQueueNotEmpty();

    // we have sent 3 files and after 1 second, we should be able to send the 4th
    Thread.sleep(1100L);
    runner.run();
    runner.assertAllFlowFilesTransferred(ControlRate.REL_SUCCESS, 1);
    runner.assertQueueEmpty();
}
 

@Test
public void testHeaderAndSplit() {
    final TestRunner runner = TestRunners.newTestRunner(new ValidateCsv());
    runner.setProperty(ValidateCsv.DELIMITER_CHARACTER, ",");
    runner.setProperty(ValidateCsv.END_OF_LINE_CHARACTER, "\n");
    runner.setProperty(ValidateCsv.QUOTE_CHARACTER, "\"");
    runner.setProperty(ValidateCsv.HEADER, "true");
    runner.setProperty(ValidateCsv.VALIDATION_STRATEGY, ValidateCsv.VALIDATE_LINES_INDIVIDUALLY);

    runner.setProperty(ValidateCsv.SCHEMA, "Null, ParseDate(\"dd/MM/yyyy\"), Optional(ParseDouble())");

    runner.enqueue("Name,Birthdate,Weight\nJohn,22/11/1954,63.2\nBob,01/03/2004,45.0");
    runner.run();

    runner.assertTransferCount(ValidateCsv.REL_VALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_VALID).get(0).assertContentEquals("Name,Birthdate,Weight\nJohn,22/11/1954,63.2\nBob,01/03/2004,45.0");
    runner.assertTransferCount(ValidateCsv.REL_INVALID, 0);

    runner.clearTransferState();

    runner.enqueue("Name,Birthdate,Weight\nJohn,22/11/1954,63a2\nBob,01/032004,45.0");
    runner.run();

    runner.assertTransferCount(ValidateCsv.REL_VALID, 0);
    runner.assertTransferCount(ValidateCsv.REL_INVALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_INVALID).get(0).assertContentEquals("Name,Birthdate,Weight\nJohn,22/11/1954,63a2\nBob,01/032004,45.0");

    runner.clearTransferState();

    runner.enqueue("Name,Birthdate,Weight\nJohn,22/111954,63.2\nBob,01/03/2004,45.0");
    runner.run();

    runner.assertTransferCount(ValidateCsv.REL_VALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_VALID).get(0).assertContentEquals("Name,Birthdate,Weight\nBob,01/03/2004,45.0");
    runner.assertTransferCount(ValidateCsv.REL_INVALID, 1);
    runner.getFlowFilesForRelationship(ValidateCsv.REL_INVALID).get(0).assertContentEquals("Name,Birthdate,Weight\nJohn,22/111954,63.2");
}
 

@Test
public void testStorageClassesMultipart() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new PutS3Object());

    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    runner.setProperty(PutS3Object.MULTIPART_THRESHOLD, "50 MB");
    runner.setProperty(PutS3Object.MULTIPART_PART_SIZE, "50 MB");

    Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());

    for (StorageClass storageClass : StorageClass.values()) {
        runner.setProperty(PutS3Object.STORAGE_CLASS, storageClass.name());

        final Map<String, String> attrs = new HashMap<>();
        attrs.put("filename", "testStorageClasses/large_" + storageClass.name() + ".dat");
        runner.enqueue(new byte[50 * 1024 * 1024 + 1], attrs);

        runner.run();

        runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1);
        FlowFile file = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS).get(0);
        Assert.assertEquals(storageClass.toString(), file.getAttribute(PutS3Object.S3_STORAGECLASS_ATTR_KEY));

        runner.clearTransferState();
    }
}
 

@Test
public void testAllMatch() {
    final TestRunner runner = TestRunners.newTestRunner(new ScanAttribute());
    runner.setProperty(ScanAttribute.DICTIONARY_FILE, "src/test/resources/ScanAttribute/dictionary1");
    runner.setProperty(ScanAttribute.MATCHING_CRITERIA, ScanAttribute.MATCH_CRITERIA_ALL);
    runner.setProperty(ScanAttribute.ATTRIBUTE_PATTERN, "a.*");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "world");

    runner.enqueue(new byte[0], attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_MATCHED, 1);
    runner.clearTransferState();

    attributes.remove("abc");
    runner.enqueue(new byte[0], attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_MATCHED, 1);
    runner.clearTransferState();

    attributes.put("abc", "world");
    attributes.put("a world", "apart");
    runner.enqueue(new byte[0], attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_UNMATCHED, 1);
    runner.clearTransferState();

    attributes.put("abc", "world");
    attributes.put("a world", "hello");
    runner.enqueue(new byte[0], attributes);
    runner.run();
    runner.assertAllFlowFilesTransferred(ScanAttribute.REL_MATCHED, 1);
}
 

@Test
public void testWaitOvertakeSkip() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(EnforceOrder.class);

    runner.setProperty(EnforceOrder.GROUP_IDENTIFIER, "${group}");
    runner.setProperty(EnforceOrder.ORDER_ATTRIBUTE, "index");
    runner.setProperty(EnforceOrder.INITIAL_ORDER, "1");
    runner.setProperty(EnforceOrder.MAX_ORDER, "10");
    runner.assertValid();
    Ordered.enqueue(runner, "b", 1);
    Ordered.enqueue(runner, "a", 2);
    Ordered.enqueue(runner, "a", 1);
    Ordered.enqueue(runner, "a", 5); // waits for a.3 and a.4
    Ordered.enqueue(runner, "b", 3); // waits for b.2
    Ordered.enqueue(runner, "c", 9); // waits for c.1 to 8
    Ordered.enqueue(runner, "d", 10); // waits for d.1 to 9

    runner.run();

    List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    assertEquals(3, succeeded.size());
    final FirstInFirstOutPrioritizer fifo = new FirstInFirstOutPrioritizer();
    succeeded.sort(fifo);
    succeeded.get(0).assertContentEquals("a.1");
    succeeded.get(1).assertContentEquals("a.2");
    succeeded.get(2).assertContentEquals("b.1");

    List<MockFlowFile> waiting = runner.getFlowFilesForRelationship(EnforceOrder.REL_WAIT);
    assertEquals(4, waiting.size());
    waiting.get(0).assertContentEquals("a.5");
    waiting.get(1).assertContentEquals("b.3");
    waiting.get(2).assertContentEquals("c.9");
    waiting.get(3).assertContentEquals("d.10");
    waiting.get(0).assertAttributeExists("EnforceOrder.startedAt");
    waiting.get(1).assertAttributeExists("EnforceOrder.startedAt");
    waiting.get(2).assertAttributeExists("EnforceOrder.startedAt");
    waiting.get(3).assertAttributeExists("EnforceOrder.startedAt");

    final MockStateManager stateManager = runner.getStateManager();
    stateManager.assertStateEquals("a.target", "3", Scope.LOCAL);
    stateManager.assertStateEquals("b.target", "2", Scope.LOCAL);
    stateManager.assertStateEquals("c.target", "1", Scope.LOCAL);
    stateManager.assertStateEquals("d.target", "1", Scope.LOCAL);
    stateManager.assertStateSet("a.updatedAt", Scope.LOCAL);
    stateManager.assertStateSet("b.updatedAt", Scope.LOCAL);
    stateManager.assertStateSet("c.updatedAt", Scope.LOCAL);
    stateManager.assertStateSet("d.updatedAt", Scope.LOCAL);

    // Run it again with waiting files.
    runner.clearTransferState();
    waiting.forEach(runner::enqueue);
    runner.run();

    runner.assertAllFlowFilesTransferred(EnforceOrder.REL_WAIT, 4);
    waiting = runner.getFlowFilesForRelationship(EnforceOrder.REL_WAIT);

    // Run it again with shorter wait timeout to make overtaking happen.
    runner.clearTransferState();
    runner.setProperty(EnforceOrder.WAIT_TIMEOUT, "10 ms");
    Thread.sleep(20);
    waiting.forEach(runner::enqueue);
    Ordered.enqueue(runner, "b", 2); // arrived in time
    Ordered.enqueue(runner, "a", 6); // a.4 and a.5 have not arrived yet
    runner.run();

    succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    succeeded.sort(fifo);
    assertEquals(3, succeeded.size());
    succeeded.get(0).assertContentEquals("a.6"); // This is ok because a.5 was there.
    succeeded.get(1).assertContentEquals("b.2");
    succeeded.get(2).assertContentEquals("b.3");

    List<MockFlowFile> overtook = runner.getFlowFilesForRelationship(EnforceOrder.REL_OVERTOOK);
    assertEquals(3, overtook.size());
    overtook.get(0).assertContentEquals("a.5"); // overtook a.3.
    overtook.get(0).assertAttributeEquals(EnforceOrder.ATTR_EXPECTED_ORDER, "3");
    overtook.get(1).assertContentEquals("c.9"); // overtook c.1 - 8.
    overtook.get(1).assertAttributeEquals(EnforceOrder.ATTR_EXPECTED_ORDER, "1");
    overtook.get(2).assertContentEquals("d.10"); // overtook d.1 - 9.
    overtook.get(2).assertAttributeEquals(EnforceOrder.ATTR_EXPECTED_ORDER, "1");

    stateManager.assertStateEquals("a.target", "7", Scope.LOCAL);
    stateManager.assertStateEquals("b.target", "4", Scope.LOCAL);
    stateManager.assertStateEquals("c.target", "10", Scope.LOCAL); // it was c.9, so +1
    stateManager.assertStateEquals("d.target", "10", Scope.LOCAL); // it was d.10 (max) so don't +1

    // Simulate a.3 and a.4 arrive but too late..
    runner.clearTransferState();
    Ordered.enqueue(runner, "a", 3);
    Ordered.enqueue(runner, "a", 4);
    runner.run();

    runner.assertAllFlowFilesTransferred(EnforceOrder.REL_SKIPPED, 2);
    final List<MockFlowFile> skipped = runner.getFlowFilesForRelationship(EnforceOrder.REL_SKIPPED);
    skipped.get(0).assertContentEquals("a.3");
    skipped.get(0).assertAttributeExists(EnforceOrder.ATTR_DETAIL);
    skipped.get(1).assertContentEquals("a.4");
    skipped.get(1).assertAttributeExists(EnforceOrder.ATTR_DETAIL);

}
 

@Test
public void testMultipleFlowFilesSuccessfulInTransaction() throws InitializationException, ProcessException, SQLException, IOException {
    final TestRunner runner = TestRunners.newTestRunner(PutSQL.class);
    runner.addControllerService("dbcp", service);
    runner.enableControllerService(service);
    runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp");
    runner.setProperty(PutSQL.BATCH_SIZE, "1");

    recreateTable("PERSONS", createPersons);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("sql.args.1.type", String.valueOf(Types.INTEGER));
    attributes.put("sql.args.1.value", "1");

    attributes.put("sql.args.2.type", String.valueOf(Types.VARCHAR));
    attributes.put("sql.args.2.value", "Mark");

    attributes.put("sql.args.3.type", String.valueOf(Types.INTEGER));
    attributes.put("sql.args.3.value", "84");

    attributes.put("fragment.identifier", "1");
    attributes.put("fragment.count", "2");
    attributes.put("fragment.index", "0");
    runner.enqueue("INSERT INTO PERSONS (ID, NAME, CODE) VALUES (?, ?, ?)".getBytes(), attributes);
    runner.run();

    // No FlowFiles should be transferred because there were not enough flowfiles with the same fragment identifier
    runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 0);

    attributes.clear();
    attributes.put("fragment.identifier", "1");
    attributes.put("fragment.count", "2");
    attributes.put("fragment.index", "1");

    runner.clearTransferState();
    runner.enqueue("UPDATE PERSONS SET NAME='Leonard' WHERE ID=1".getBytes(), attributes);
    runner.run();

    // Both FlowFiles with fragment identifier 1 should be successful
    runner.assertTransferCount(PutSQL.REL_SUCCESS, 2);
    runner.assertTransferCount(PutSQL.REL_FAILURE, 0);
    runner.assertTransferCount(PutSQL.REL_RETRY, 0);
    for (final MockFlowFile mff : runner.getFlowFilesForRelationship(PutSQL.REL_SUCCESS)) {
        mff.assertAttributeEquals("fragment.identifier", "1");
    }

    try (final Connection conn = service.getConnection()) {
        try (final Statement stmt = conn.createStatement()) {
            final ResultSet rs = stmt.executeQuery("SELECT * FROM PERSONS");
            assertTrue(rs.next());
            assertEquals(1, rs.getInt(1));
            assertEquals("Leonard", rs.getString(2));
            assertEquals(84, rs.getInt(3));
            assertFalse(rs.next());
        }
    }
}
 

/**
 * Test decoding using a processor-wide schema and switches between flowfile schema and processor schema
 * @throws Exception
 */
@Test
public void onPropertyModified() throws Exception {
    TestRunner runner = TestRunners.newTestRunner(new ProtobufDecoder());

    HashMap<String, String> addressBookProperties = new HashMap<>();
    addressBookProperties.put("protobuf.messageType", "AddressBook");


    /*
        First try to decode using a schema set in a processor property
     */

    runner.setProperty("protobuf.schemaPath", ProtobufDecoderTest.class.getResource("/schemas/AddressBook.desc").getPath());
    runner.assertValid();

    runner.enqueue(ProtobufDecoderTest.class.getResourceAsStream("/data/AddressBook_basic.data"), addressBookProperties);

    runner.run(1);

    // Check if the flowfile has been successfully processed
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(ProtobufDecoder.SUCCESS);

    // Finally check the content
    MockFlowFile result = runner.getFlowFilesForRelationship(ProtobufDecoder.SUCCESS).get(0);
    ObjectMapper mapper = new ObjectMapper();
    JsonNode expected = mapper.readTree(this.getClass().getResourceAsStream("/data/AddressBook_basic.json"));
    JsonNode given = mapper.readTree(runner.getContentAsByteArray(result));
    Assert.assertEquals("The parsing result of AddressBook_basic.data is not as expected", expected, given);


    /*
        Then try to remove the schema from the processor property and see if it still parse
     */

    runner.clearTransferState();
    runner.removeProperty(runner.getProcessor().getPropertyDescriptor("protobuf.schemaPath"));
    Assert.assertFalse("The schema property should now be null", runner.getProcessContext().getProperty("protobuf.schemaPath").isSet());
    runner.assertValid();

    runner.enqueue(ProtobufDecoderTest.class.getResourceAsStream("/data/AddressBook_basic.data"), addressBookProperties);

    runner.run(1);

    // Check if the flowfile has been successfully processed
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(ProtobufDecoder.INVALID_SCHEMA);


    /*
        Finally add the property again to see if it works again
     */

    runner.clearTransferState();
    runner.setProperty("protobuf.schemaPath", ProtobufDecoderTest.class.getResource("/schemas/AddressBook.desc").getPath());
    runner.assertValid();

    runner.enqueue(ProtobufDecoderTest.class.getResourceAsStream("/data/AddressBook_basic.data"), addressBookProperties);

    runner.run(1);

    // Check if the flowfile has been successfully processed
    runner.assertQueueEmpty();
    runner.assertAllFlowFilesTransferred(ProtobufDecoder.SUCCESS);

    // Finally check the content
    result = runner.getFlowFilesForRelationship(ProtobufDecoder.SUCCESS).get(0);
    expected = mapper.readTree(this.getClass().getResourceAsStream("/data/AddressBook_basic.json"));
    given = mapper.readTree(runner.getContentAsByteArray(result));
    Assert.assertEquals("The parsing result of AddressBook_basic.data is not as expected", expected, given);

}
 

@Test
public void testFlowFileIsOnlyHeader() {
    final TestRunner runner = TestRunners.newTestRunner(new SplitText());
    runner.setProperty(SplitText.HEADER_LINE_COUNT, "0");
    runner.setProperty(SplitText.LINE_SPLIT_COUNT, "2");
    runner.setProperty(SplitText.FRAGMENT_MAX_SIZE, "50 B");
    runner.setProperty(SplitText.HEADER_MARKER, "Head");

    runner.enqueue("Header Line #1\nHeaderLine#2\n");
    runner.run();

    runner.assertTransferCount(SplitText.REL_FAILURE, 0);
    runner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    runner.getFlowFilesForRelationship(SplitText.REL_ORIGINAL).get(0).assertAttributeEquals("fragment.count", "1");
    runner.assertTransferCount(SplitText.REL_SPLITS, 1);

    // repeat with header cou8nt versus header marker
    runner.clearTransferState();
    runner.setProperty(SplitText.HEADER_LINE_COUNT, "2");
    runner.setProperty(SplitText.LINE_SPLIT_COUNT, "2");
    runner.setProperty(SplitText.FRAGMENT_MAX_SIZE, "50 B");

    runner.enqueue("Header Line #1\nHeaderLine #2\n");
    runner.run();

    runner.assertTransferCount(SplitText.REL_FAILURE, 0);
    runner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    runner.getFlowFilesForRelationship(SplitText.REL_ORIGINAL).get(0).assertAttributeEquals("fragment.count", "1");
    runner.assertTransferCount(SplitText.REL_SPLITS, 1);

    // repeat single header line with no newline characters
    runner.clearTransferState();
    runner.setProperty(SplitText.HEADER_LINE_COUNT, "1");
    runner.setProperty(SplitText.LINE_SPLIT_COUNT, "2");
    runner.setProperty(SplitText.FRAGMENT_MAX_SIZE, "50 B");

    runner.enqueue("Header Line #1");
    runner.run();

    runner.assertTransferCount(SplitText.REL_FAILURE, 0);
    runner.assertTransferCount(SplitText.REL_ORIGINAL, 1);
    runner.getFlowFilesForRelationship(SplitText.REL_ORIGINAL).get(0).assertAttributeEquals("fragment.count", "1");
    runner.assertTransferCount(SplitText.REL_SPLITS, 1);
}
 

@Test
public void testTTL() throws JMSException, InterruptedException {
    final PutJMS putJMS = spy(new PutJMS());
    final TestRunner runnerPut = TestRunners.newTestRunner(putJMS);
    runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
    runnerPut.setProperty(JmsProperties.URL, TEST_URL);
    runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE);
    runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
    runnerPut.setProperty(JmsProperties.MESSAGE_TTL, "3 s");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "file1.txt");
    runnerPut.enqueue("ttl10secNotExpired".getBytes(), attributes);

    runnerPut.run();

    assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
    assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());

    final GetJMSQueue getJmsQueue = new GetJMSQueue();
    final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue);
    runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER);
    runnerGet.setProperty(JmsProperties.URL, TEST_URL);
    runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix());
    runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE);
    runnerGet.setProperty(JmsProperties.TIMEOUT, "1 s");

    runnerGet.run();

    final List<MockFlowFile> flowFiles1 = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
    assertEquals(1, flowFiles1.size());
    final MockFlowFile successFlowFile1 = flowFiles1.get(0);
    successFlowFile1.assertContentEquals("ttl10secNotExpired");

    runnerPut.clearTransferState();
    runnerGet.clearTransferState();

    runnerPut.setProperty(JmsProperties.MESSAGE_TTL, "1 s");
    runnerPut.enqueue("ttl1secExpired".getBytes(), attributes);

    runnerPut.run();

    assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size());
    assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size());

    Thread.sleep(2000L);
    runnerGet.run();

    final List<MockFlowFile> flowFiles2 = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS);
    assertEquals(0, flowFiles2.size());
}