org.apache.nifi.util.FlowFileUnpackagerV3 Java Examples

@Test
public void testSendAsFlowFile() throws Exception {
    setup(null);
    runner.setProperty(PostHTTP.URL, server.getUrl());
    runner.setProperty(PostHTTP.SEND_AS_FLOWFILE, "true");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
    Assert.assertNull(receivedAttrs.get("Content-Length"));
}
 

@Test
public void testSendAsFlowFile() throws Exception {
    setup(null);
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.URL, server.getUrl());
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.SEND_AS_FLOWFILE, "true");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
    Assert.assertNull(receivedAttrs.get("Content-Length"));
}
 

private void checkBatch(TestServer server, CaptureServlet servlet, Set<String> actualContent, int expectedCount) throws Exception {
    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();
    Set<String> actualFFContent = new HashSet<>();
    Set<String> actualPostContent = new HashSet<>();

    runner.assertAllFlowFilesTransferred(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS, expectedCount);

    // confirm that all FlowFiles transferred to 'success' have the same URL
    // also accumulate content to verify later
    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS);
    for (int i = 0; i < expectedCount; i++) {
        MockFlowFile mff = successFlowFiles.get(i);
        mff.assertAttributeEquals("url", server.getUrl());
        String content = new String(mff.toByteArray());
        actualFFContent.add(content);
    }

    // confirm that all FlowFiles POSTed to server have the same URL
    // also accumulate content to verify later
    try (ByteArrayInputStream bais = new ByteArrayInputStream(servlet.getLastPost());
        ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
        for (int i = 0; i < expectedCount; i++) {
            Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
            String receivedContent = new String(baos.toByteArray());
            actualPostContent.add(receivedContent);
            assertEquals(server.getUrl(), receivedAttrs.get("url"));
            assertTrue(unpacker.hasMoreData() || i == (expectedCount - 1));
            baos.reset();
        }
    }

    // confirm that the transferred and POSTed content match
    assertEquals(actualFFContent, actualPostContent);

    // accumulate actial content
    actualContent.addAll(actualPostContent);
    runner.clearTransferState();
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();
    PackageFormat packagingFormat = PackageFormat.getFormat(context.getProperty(PACKAGING_FORMAT).getValue().toLowerCase());
    if (packagingFormat == PackageFormat.AUTO_DETECT_FORMAT) {
        packagingFormat = null;
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No mime.type attribute set for {}; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        for (PackageFormat format: PackageFormat.values()) {
            if (mimeType.toLowerCase().equals(format.getMimeType())) {
                packagingFormat = format;
            }
        }
        if (packagingFormat == null) {
            logger.info("Cannot unpack {} because its mime.type attribute is set to '{}', which is not a format that can be unpacked; routing to 'success'", new Object[]{flowFile, mimeType});
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }

    // set the Unpacker to use for this FlowFile.  FlowFileUnpackager objects maintain state and are not reusable.
    final Unpacker unpacker;
    final boolean addFragmentAttrs;
    switch (packagingFormat) {
    case TAR_FORMAT:
    case X_TAR_FORMAT:
        unpacker = tarUnpacker;
        addFragmentAttrs = true;
        break;
    case ZIP_FORMAT:
        unpacker = zipUnpacker;
        addFragmentAttrs = true;
        break;
    case FLOWFILE_STREAM_FORMAT_V2:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV2());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_STREAM_FORMAT_V3:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV3());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_TAR_FORMAT:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV1());
        addFragmentAttrs = false;
        break;
    case AUTO_DETECT_FORMAT:
    default:
        // The format of the unpacker should be known before initialization
        throw new ProcessException(packagingFormat + " is not a valid packaging format");
    }

    final List<FlowFile> unpacked = new ArrayList<>();
    try {
        unpacker.unpack(session, flowFile, unpacked);
        if (unpacked.isEmpty()) {
            logger.error("Unable to unpack {} because it does not appear to have any entries; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        if (addFragmentAttrs) {
            finishFragmentAttributes(session, flowFile, unpacked);
        }
        session.transfer(unpacked, REL_SUCCESS);
        final String fragmentId = unpacked.size() > 0 ? unpacked.get(0).getAttribute(FRAGMENT_ID) : null;
        flowFile = FragmentAttributes.copyAttributesToOriginal(session, flowFile, fragmentId, unpacked.size());
        session.transfer(flowFile, REL_ORIGINAL);
        session.getProvenanceReporter().fork(flowFile, unpacked);
        logger.info("Unpacked {} into {} and transferred to success", new Object[]{flowFile, unpacked});
    } catch (final ProcessException | InvalidPathException e) {
        logger.error("Unable to unpack {} due to {}; routing to failure", new Object[]{flowFile, e});
        session.transfer(flowFile, REL_FAILURE);
        session.remove(unpacked);
    }
}
 

@Test
public void testSendAsFlowFileSecure() throws Exception {
    final Map<String, String> sslProps = new HashMap<>();
    sslProps.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/localhost-ks.jks");
    sslProps.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "localtest");
    sslProps.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
    sslProps.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/localhost-ts.jks");
    sslProps.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "localtest");
    sslProps.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
    sslProps.put(TestServer.NEED_CLIENT_AUTH, "true");
    setup(sslProps);

    final SSLContextService sslContextService = new StandardSSLContextService();
    runner.addControllerService("ssl-context", sslContextService);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE, "src/test/resources/localhost-ts.jks");
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_PASSWORD, "localtest");
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_TYPE, "JKS");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE, "src/test/resources/localhost-ks.jks");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_PASSWORD, "localtest");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_TYPE, "JKS");
    runner.enableControllerService(sslContextService);

    runner.setProperty(PostHTTP.URL, server.getSecureUrl());
    runner.setProperty(PostHTTP.SEND_AS_FLOWFILE, "true");
    runner.setProperty(PostHTTP.SSL_CONTEXT_SERVICE, "ssl-context");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final ComponentLog logger = getLogger();
    PackageFormat packagingFormat = PackageFormat.getFormat(context.getProperty(PACKAGING_FORMAT).getValue().toLowerCase());
    if (packagingFormat == PackageFormat.AUTO_DETECT_FORMAT) {
        packagingFormat = null;
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No mime.type attribute set for {}; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        for (PackageFormat format: PackageFormat.values()) {
            if (mimeType.toLowerCase().equals(format.getMimeType())) {
                packagingFormat = format;
            }
        }
        if (packagingFormat == null) {
            logger.info("Cannot unpack {} because its mime.type attribute is set to '{}', which is not a format that can be unpacked; routing to 'success'", new Object[]{flowFile, mimeType});
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }

    // set the Unpacker to use for this FlowFile.  FlowFileUnpackager objects maintain state and are not reusable.
    final Unpacker unpacker;
    final boolean addFragmentAttrs;
    switch (packagingFormat) {
    case TAR_FORMAT:
    case X_TAR_FORMAT:
        unpacker = tarUnpacker;
        addFragmentAttrs = true;
        break;
    case ZIP_FORMAT:
        unpacker = zipUnpacker;
        addFragmentAttrs = true;
        break;
    case FLOWFILE_STREAM_FORMAT_V2:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV2());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_STREAM_FORMAT_V3:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV3());
        addFragmentAttrs = false;
        break;
    case FLOWFILE_TAR_FORMAT:
        unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV1());
        addFragmentAttrs = false;
        break;
    case AUTO_DETECT_FORMAT:
    default:
        // The format of the unpacker should be known before initialization
        throw new ProcessException(packagingFormat + " is not a valid packaging format");
    }

    final List<FlowFile> unpacked = new ArrayList<>();
    try {
        unpacker.unpack(session, flowFile, unpacked);
        if (unpacked.isEmpty()) {
            logger.error("Unable to unpack {} because it does not appear to have any entries; routing to failure", new Object[]{flowFile});
            session.transfer(flowFile, REL_FAILURE);
            return;
        }

        if (addFragmentAttrs) {
            finishFragmentAttributes(session, flowFile, unpacked);
        }
        session.transfer(unpacked, REL_SUCCESS);
        final String fragmentId = unpacked.size() > 0 ? unpacked.get(0).getAttribute(FRAGMENT_ID) : null;
        flowFile = FragmentAttributes.copyAttributesToOriginal(session, flowFile, fragmentId, unpacked.size());
        session.transfer(flowFile, REL_ORIGINAL);
        session.getProvenanceReporter().fork(flowFile, unpacked);
        logger.info("Unpacked {} into {} and transferred to success", new Object[]{flowFile, unpacked});
    } catch (final Exception e) {
        logger.error("Unable to unpack {} due to {}; routing to failure", new Object[]{flowFile, e});
        session.transfer(flowFile, REL_FAILURE);
        session.remove(unpacked);
    }
}
 

@Test
public void testSendAsFlowFileSecure() throws Exception {
    final Map<String, String> sslProps = new HashMap<>();
    sslProps.put(StandardSSLContextService.KEYSTORE.getName(), KEYSTORE_PATH);
    sslProps.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    sslProps.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), JKS_TYPE);
    sslProps.put(StandardSSLContextService.TRUSTSTORE.getName(), TRUSTSTORE_PATH);
    sslProps.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    sslProps.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), JKS_TYPE);
    sslProps.put(TestServer.NEED_CLIENT_AUTH, "true");
    setup(sslProps);

    final SSLContextService sslContextService = new StandardSSLContextService();
    runner.addControllerService("ssl-context", sslContextService);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE, TRUSTSTORE_PATH);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_PASSWORD, KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_TYPE, JKS_TYPE);
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE, KEYSTORE_PATH);
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_PASSWORD, KEYSTORE_AND_TRUSTSTORE_PASSWORD);
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_TYPE, JKS_TYPE);
    runner.enableControllerService(sslContextService);

    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.URL, server.getSecureUrl());
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.SEND_AS_FLOWFILE, "true");
    runner.setProperty(org.apache.nifi.processors.standard.PostHTTP.SSL_CONTEXT_SERVICE, "ssl-context");

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");

    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);

    runner.run(1);
    runner.assertAllFlowFilesTransferred(org.apache.nifi.processors.standard.PostHTTP.REL_SUCCESS);

    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);

    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();

    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));

    assertTrue(unpacker.hasMoreData());

    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();

    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
}