Java Code Examples for org.apache.nifi.util.TestRunner#setValidateExpressionUsage()

@Test
public void testRegexWithExpressionLanguageIsEscapedLineByLine() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "${replaceKey}");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "${replaceValue}");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("replaceKey", "R.*y");
    attributes.put("replaceValue", "Spider");
    runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/testFile.txt")));
}
 

@Test
public void testAttributeToContent() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.SEARCH_VALUE, ".*");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "${abc}");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "Good");
    runner.enqueue(Paths.get("src/test/resources/hello.txt"), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals("Good");
}
 

@Test
public void testReplacementWithExpressionLanguageIsEscapedLineByLine() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "(jo)");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "[${abc}]");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "$1");
    runner.enqueue(translateNewLines(Paths.get("src/test/resources/TestReplaceTextLineByLine/testFile.txt")), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals(translateNewLines(new File("src/test/resources/TestReplaceTextLineByLine/cu[$1]_Po[$1].txt")));
}
 

@Test
public void testDynamicEnvironment() throws Exception {
    File exJar = new File("src/test/resources/ExecuteCommand/TestDynamicEnvironment.jar");
    File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
    String jarPath = exJar.getAbsolutePath();
    exJar.setExecutable(true);
    final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
    controller.setProperty("NIFI_TEST_1", "testvalue1");
    controller.setProperty("NIFI_TEST_2", "testvalue2");
    controller.setValidateExpressionUsage(false);
    controller.enqueue(dummy.toPath());
    controller.setProperty(ExecuteStreamCommand.WORKING_DIR, "target");
    controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
    controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
    controller.run(1);
    controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
    controller.assertTransferCount(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP, 1);
    List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.OUTPUT_STREAM_RELATIONSHIP);
    byte[] byteArray = flowFiles.get(0).toByteArray();
    String result = new String(byteArray);
    Set<String> dynamicEnvironmentVariables = new HashSet<>(Arrays.asList(result.split("\r?\n")));
    assertFalse("Should contain at least two environment variables starting with NIFI", dynamicEnvironmentVariables.size() < 2);
    assertTrue("NIFI_TEST_1 environment variable is missing", dynamicEnvironmentVariables.contains("NIFI_TEST_1=testvalue1"));
    assertTrue("NIFI_TEST_2 environment variable is missing", dynamicEnvironmentVariables.contains("NIFI_TEST_2=testvalue2"));
}
 

@Test
public void testBackReferenceWithTooLargeOfIndexIsEscaped() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "(ell)");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1$2");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("replaceKey", "H.*o");
    attributes.put("replaceValue", "Good-bye");
    runner.enqueue(Paths.get("src/test/resources/hello.txt"), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals("Hell$2o, World!");
}
 

@Test
public void testJSONArrayFailure() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new DafJSONArrayToMultiline());
    runner.setValidateExpressionUsage(false);
    runner.assertValid();

    ClassLoader classLoader = getClass().getClassLoader();
    String testJson = "";
    try {
        testJson = IOUtils.toString(classLoader.getResourceAsStream("json/notvalid.json"));
    } catch (IOException e) {
        log.error("", e);
    }

    byte[] flowContent = testJson.getBytes();

    runner.enqueue(flowContent);
    runner.run();

    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship("failure");
    MockFlowFile result = successFlowFiles.get(0);
    result.assertContentEquals(classLoader.getResourceAsStream("json/notvalid.json"));
}
 

@Test
public void testBackReferenceWithInvalidReferenceIsEscaped() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "(ell)");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$d");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("replaceKey", "H.*o");
    attributes.put("replaceValue", "Good-bye");
    runner.enqueue(Paths.get("src/test/resources/hello.txt"), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals("H$do, World!");
}
 

@Test
public void testCapturingGroupInExpressionLanguage2() {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "(.*)/(.*?).jpg");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1/${ '$2':substring(0,1) }.png");

    final String csvIn =
          "1,2,3,https://123.jpg,[email protected]\n"
        + "3,2,1,https://321.jpg,[email protected]";
    final String expectedCsvOut =
        "1,2,3,https://1.png,[email protected]\n"
      + "3,2,1,https://3.png,[email protected]";

    runner.enqueue(csvIn.getBytes());

    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals(expectedCsvOut);
}
 

@Test
public void testCSVCleansingComplex() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new DafCSVCleansing());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(DafCSVCleansing.SEPARATOR_CHAR, "|");
    runner.setProperty(DafCSVCleansing.QUOTE_CHAR, "\\\"");
    runner.setProperty(DafCSVCleansing.ESCAPE_CHAR, "\\\\");
    runner.assertValid();

    ClassLoader classLoader = getClass().getClassLoader();
    String testCsv = "";
    try {
        testCsv = IOUtils.toString(classLoader.getResourceAsStream("csv/complex_test.csv"));
    } catch (IOException e) {
        log.error("", e);
    }

    byte[] flowContent = testCsv.getBytes();

    runner.enqueue(flowContent);
    runner.run();

    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship("success");
    MockFlowFile result = successFlowFiles.get(0);
    result.assertContentEquals(classLoader.getResourceAsStream("csv/complex_test_result.csv"));
}
 

@Test
public void testIgnoredStdinPutToAttribute() throws IOException {
    File exJar = new File("src/test/resources/ExecuteCommand/TestIngestAndUpdate.jar");
    File dummy = new File("src/test/resources/ExecuteCommand/1000bytes.txt");
    String jarPath = exJar.getAbsolutePath();
    exJar.setExecutable(true);
    final TestRunner controller = TestRunners.newTestRunner(ExecuteStreamCommand.class);
    controller.setValidateExpressionUsage(false);
    controller.enqueue(dummy.toPath());
    controller.setProperty(ExecuteStreamCommand.WORKING_DIR, "target");
    controller.setProperty(ExecuteStreamCommand.EXECUTION_COMMAND, "java");
    controller.setProperty(ExecuteStreamCommand.EXECUTION_ARGUMENTS, "-jar;" + jarPath);
    controller.setProperty(ExecuteStreamCommand.IGNORE_STDIN, "true");
    controller.setProperty(ExecuteStreamCommand.PUT_OUTPUT_IN_ATTRIBUTE, "executeStreamCommand.output");
    controller.run(1);
    controller.assertTransferCount(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP, 1);
    List<MockFlowFile> flowFiles = controller.getFlowFilesForRelationship(ExecuteStreamCommand.ORIGINAL_RELATIONSHIP);
    String result = flowFiles.get(0).getAttribute("executeStreamCommand.output");
    assertTrue("TestIngestAndUpdate.jar should not have received anything to modify",
            Pattern.compile("target:ModifiedResult\r?\n?").matcher(result).find());
}
 

@Test
public void testCSVCleansingTab() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new DafCSVCleansing());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(DafCSVCleansing.SEPARATOR_CHAR, "\\\t");
    runner.setProperty(DafCSVCleansing.QUOTE_CHAR, "\\\"");
    runner.setProperty(DafCSVCleansing.ESCAPE_CHAR, "\\\\");
    runner.assertValid();

    ClassLoader classLoader = getClass().getClassLoader();
    String testCsv = "";
    try {
        testCsv = IOUtils.toString(classLoader.getResourceAsStream("csv/tabs.csv"));
    } catch (IOException e) {
        log.error("", e);
    }

    byte[] flowContent = testCsv.getBytes();

    runner.enqueue(flowContent);
    runner.run();

    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship("success");
    MockFlowFile result = successFlowFiles.get(0);
    result.assertContentEquals(classLoader.getResourceAsStream("csv/tabsresult.csv"));
}
 

@Test
@Ignore("Authenticated Proxy : Comment this out if you want to run against local proxied ES.")
public void testFetchElasticsearchBasicBehindAuthenticatedProxy() {
    final TestRunner runner = TestRunners.newTestRunner(new FetchElasticsearchHttp());
    runner.setValidateExpressionUsage(true);

    runner.setProperty(FetchElasticsearchHttp.INDEX, "doc");
    runner.setProperty(FetchElasticsearchHttp.TYPE, "status");
    runner.setProperty(FetchElasticsearchHttp.DOC_ID, "${doc_id}");

    runner.setProperty(FetchElasticsearchHttp.PROXY_HOST, "localhost");
    runner.setProperty(FetchElasticsearchHttp.PROXY_PORT, "3328");
    runner.setProperty(FetchElasticsearchHttp.PROXY_USERNAME, "squid");
    runner.setProperty(FetchElasticsearchHttp.PROXY_PASSWORD, "changeme");
    runner.setProperty(FetchElasticsearchHttp.ES_URL, "http://172.18.0.2:9200");

    runner.assertValid();

    runner.enqueue(docExample, new HashMap<String, String>() {{
        put("doc_id", "28039652140");
    }});

    runner.enqueue(docExample);
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearchHttp.REL_SUCCESS, 1);
}
 

@Test
public void testRegexNoCaptureDefaultReplacement() throws IOException {
    // Test the old Default Regex and new Default Regex with the default replacement.  This should fail
    // because the regex does not create a capture group.
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "(?s:^.*$)");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "$1");
    runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.REGEX_REPLACE);
    runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.ENTIRE_TEXT);

    exception.expect(AssertionError.class);
    exception.expectMessage("java.lang.IndexOutOfBoundsException: No group 1");
    runner.enqueue("testing\n123".getBytes());
    runner.run();
}
 

/**
 * Demonstrates transforming the JSON object in an incoming FlowFile to output
 */
@Test
public void testJSONObjectCleansing() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new DafJSONCleansing());
    runner.setValidateExpressionUsage(false);
    runner.assertValid();

    ClassLoader classLoader = getClass().getClassLoader();
    String testJson = "";
    try {
        testJson = IOUtils.toString(classLoader.getResourceAsStream("json/test.json"));
    } catch (IOException e) {
        log.error("", e);
    }

    byte[] flowContent = testJson.getBytes();

    runner.enqueue(flowContent);
    runner.run();

    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship("success");
    MockFlowFile result = successFlowFiles.get(0);
    result.assertContentEquals(classLoader.getResourceAsStream("json/test.json"));
}
 

@Test
@Ignore("Comment this out if you want to run against local or test ES")
public void testFetchElasticsearchBatch() throws IOException {
    System.out.println("Starting test " + new Object() {
    }.getClass().getEnclosingMethod().getName());
    final TestRunner runner = TestRunners.newTestRunner(new FetchElasticsearchHttp());
    runner.setValidateExpressionUsage(true);

    //Local Cluster - Mac pulled from brew
    runner.setProperty(AbstractElasticsearchHttpProcessor.ES_URL, "http://127.0.0.1:9200");
    runner.setProperty(FetchElasticsearchHttp.INDEX, "doc");
    runner.setProperty(FetchElasticsearchHttp.TYPE, "status");
    runner.setProperty(FetchElasticsearchHttp.DOC_ID, "${doc_id}");
    runner.assertValid();

    for (int i = 0; i < 100; i++) {
        long newId = 28039652140L + i;
        final String newStrId = Long.toString(newId);
        runner.enqueue(docExample, new HashMap<String, String>() {{
            put("doc_id", newStrId);
        }});
    }
    runner.run(100);
    runner.assertAllFlowFilesTransferred(FetchElasticsearchHttp.REL_SUCCESS, 100);
}
 

/**
 * Demonstrates applying date/time parsing and formatting.
 * @throws Exception
 */
@Test
public void testDateTimeTransformJavascript() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new ExecuteScript());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(SCRIPT_ENGINE, "ECMAScript");
    runner.setProperty(ScriptingComponentUtils.SCRIPT_FILE, "src/test/resources/executescript/content/datetime.js");
    runner.setProperty(ScriptingComponentUtils.MODULES, "src/test/resources/executescript");
    runner.assertValid();
    String inputDateTime = "2018/12/23 10:01:23";

    runner.enqueue(inputDateTime);
    runner.run();

    List<MockFlowFile> failedFlowFiles = runner.getFlowFilesForRelationship(ExecuteScript.REL_FAILURE);
    Assert.assertEquals(0, failedFlowFiles.size());

    runner.assertAllFlowFilesTransferred("success", 1);
    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship("success");
    MockFlowFile result = successFlowFiles.get(0);
    byte[] flowFileBytes = result.toByteArray();
    String outputDateTime = new String(flowFileBytes);
    Assert.assertEquals("2018/12/23 05:01:23", outputDateTime);
}
 

@Test
public void testWithMultipleMatches() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.SEARCH_VALUE, "l");
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "R");

    runner.enqueue(Paths.get("src/test/resources/hello.txt"));
    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals("HeRRo, WorRd!");
}
 

@Test
public void testAppendWithCarriageReturnNewLine() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ReplaceText());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(ReplaceText.REPLACEMENT_VALUE, "!");
    runner.setProperty(ReplaceText.REPLACEMENT_STRATEGY, ReplaceText.APPEND);
    runner.setProperty(ReplaceText.EVALUATION_MODE, ReplaceText.LINE_BY_LINE);

    runner.enqueue("hello\r\nthere\r\nsir".getBytes());
    runner.run();

    runner.assertAllFlowFilesTransferred(ReplaceText.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ReplaceText.REL_SUCCESS).get(0);
    out.assertContentEquals("hello!\r\nthere!\r\nsir!");
}
 

@Test
@Ignore("Comment this out if you want to run against local or test ES")
public void testPutElasticSearchBatch() throws IOException {
    System.out.println("Starting test " + new Object() {
    }.getClass().getEnclosingMethod().getName());
    final TestRunner runner = TestRunners.newTestRunner(new PutElasticsearch());
    runner.setValidateExpressionUsage(false);

    //Local Cluster - Mac pulled from brew
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.CLUSTER_NAME, "elasticsearch_brew");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(PutElasticsearch.INDEX, "doc");
    runner.setProperty(PutElasticsearch.BATCH_SIZE, "100");

    runner.setProperty(PutElasticsearch.TYPE, "status");
    runner.setProperty(PutElasticsearch.ID_ATTRIBUTE, "doc_id");
    runner.assertValid();


    String message = convertStreamToString(docExample);
    for (int i = 0; i < 100; i++) {

        long newId = 28039652140L + i;
        final String newStrId = Long.toString(newId);
        runner.enqueue(message.getBytes(), new HashMap<String, String>() {{
            put("doc_id", newStrId);
        }});

    }

    runner.run();

    runner.assertAllFlowFilesTransferred(PutElasticsearch.REL_SUCCESS, 100);
}
 

/**
 * Tests a script that throws a ProcessException within.
 * The expected result is that the exception will be propagated.
 *
 * @throws Exception Any error encountered while testing
 */
@Test(expected = AssertionError.class)
public void testInvokeScriptCausesException() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(new InvokeScriptedProcessor());
    runner.setValidateExpressionUsage(false);
    runner.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE, "ECMAScript");
    runner.setProperty(ScriptingComponentUtils.SCRIPT_BODY, getFileContentsAsString(
            TEST_RESOURCE_LOCATION + "javascript/testInvokeScriptCausesException.js")
    );
    runner.assertValid();
    runner.enqueue("test content".getBytes(StandardCharsets.UTF_8));
    runner.run();

}