Java Code Examples for org.apache.nifi.stream.io.StreamUtils#copy()

@Override
public long exportTo(final ContentClaim claim, final Path destination, final boolean append, final long offset, final long length) throws IOException {
    if (claim == null) {
        if (append) {
            return 0L;
        }
        Files.createFile(destination);
        return 0L;
    }

    final StandardOpenOption openOption = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;
    try (final InputStream in = read(claim);
            final OutputStream destinationStream = Files.newOutputStream(destination, openOption)) {

        if (offset > 0) {
            StreamUtils.skip(in, offset);
        }

        StreamUtils.copy(in, destinationStream, length);
        return length;
    }
}
 

@Override
public ContentClaim clone(final ContentClaim original, final boolean lossTolerant) throws IOException {
    final ContentClaim createdClaim = create(lossTolerant);

    try (final InputStream dataIn = read(original)) {
        final ContentRepository createdClaimRepo = lossTolerant ? this : getBackupRepository();
        if (createdClaimRepo == null) {
            throw new IllegalStateException("Cannot create non-loss-tolerant ContentClaim because there is no persistent Content Repository configured");
        }

        try (final OutputStream dataOut = createdClaimRepo.write(createdClaim)) {
            StreamUtils.copy(dataIn, dataOut);
        }
    }

    return createdClaim;
}
 

@Override
public void encode(final DataPacket dataPacket, final OutputStream encodedOut) throws IOException {
    final DataOutputStream out = new DataOutputStream(encodedOut);

    final Map<String, String> attributes = dataPacket.getAttributes();
    out.writeInt(attributes.size());
    for (final Map.Entry<String, String> entry : attributes.entrySet()) {
        writeString(entry.getKey(), out);
        writeString(entry.getValue(), out);
    }

    out.writeLong(dataPacket.getSize());

    final InputStream in = dataPacket.getData();
    StreamUtils.copy(in, encodedOut);
    encodedOut.flush();
}
 

private static void readStdoutReadable(final boolean ignoreStdin, final OutputStream stdinWritable,
                                       final ComponentLog logger, final InputStream incomingFlowFileIS) throws IOException {
    Thread writerThread = new Thread(new Runnable() {

        @Override
        public void run() {
            if (!ignoreStdin) {
                try {
                    StreamUtils.copy(incomingFlowFileIS, stdinWritable);
                } catch (IOException e) {
                    // This is unlikely to occur, and isn't handled at the moment
                    // Bug captured in NIFI-1194
                    logger.error("Failed to write flow file to stdin due to {}", new Object[]{e}, e);
                }
            }
            // MUST close the output stream to the stdin so that whatever is reading knows
            // there is no more data.
            IOUtils.closeQuietly(stdinWritable);
        }
    });
    writerThread.setDaemon(true);
    writerThread.start();
}
 

@Override
public long merge(final Collection<ContentClaim> claims, final ContentClaim destination, final byte[] header, final byte[] footer, final byte[] demarcator) throws IOException {
    if (claims.contains(destination)) {
        throw new IllegalArgumentException("destination cannot be within claims");
    }

    try (final ByteCountingOutputStream out = new ByteCountingOutputStream(write(destination))) {
        if (header != null) {
            out.write(header);
        }

        int i = 0;
        for (final ContentClaim claim : claims) {
            try (final InputStream in = read(claim)) {
                StreamUtils.copy(in, out);
            }

            if (++i < claims.size() && demarcator != null) {
                out.write(demarcator);
            }
        }

        if (footer != null) {
            out.write(footer);
        }

        return out.getBytesWritten();
    }
}
 

@Override
public long exportTo(ContentClaim claim, OutputStream destination, long offset, long length) throws IOException {
    final InputStream in = read(claim);
    try {
        StreamUtils.skip(in, offset);
        StreamUtils.copy(in, destination, length);
    } finally {
        IOUtils.closeQuietly(in);
    }
    return length;
}
 

@Override
public long exportTo(final ContentClaim claim, final Path destination, final boolean append, final long offset, final long length) throws IOException {
    if (claim == null && offset > 0) {
        throw new IllegalArgumentException("Cannot specify an offset of " + offset + " for a null claim");
    }
    if (claim == null) {
        if (append) {
            return 0L;
        }
        Files.createFile(destination);
        return 0L;
    }

    final long claimSize = size(claim);
    if (offset > claimSize) {
        throw new IllegalArgumentException("Offset of " + offset + " exceeds claim size of " + claimSize);

    }

    try (final InputStream in = read(claim);
            final FileOutputStream fos = new FileOutputStream(destination.toFile(), append)) {
        if (offset > 0) {
            StreamUtils.skip(in, offset);
        }
        StreamUtils.copy(in, fos, length);
        if (alwaysSync) {
            fos.getFD().sync();
        }
        return length;
    }
}
 

@Override
public long exportTo(ContentClaim claim, OutputStream destination) throws IOException {
    final InputStream in = read(claim);
    try {
        return StreamUtils.copy(in, destination);
    } finally {
        IOUtils.closeQuietly(in);
    }
}
 

@Override
public long merge(final Collection<ContentClaim> claims, final ContentClaim destination, final byte[] header, final byte[] footer, final byte[] demarcator) throws IOException {
    if (claims.contains(destination)) {
        throw new IllegalArgumentException("destination cannot be within claims");
    }

    try (final ByteCountingOutputStream out = new ByteCountingOutputStream(write(destination))) {
        if (header != null) {
            out.write(header);
        }

        int i = 0;
        for (final ContentClaim claim : claims) {
            try (final InputStream in = read(claim)) {
                StreamUtils.copy(in, out);
            }

            if (++i < claims.size() && demarcator != null) {
                out.write(demarcator);
            }
        }

        if (footer != null) {
            out.write(footer);
        }

        return out.getBytesWritten();
    }
}
 

@Override
public long exportTo(final ContentClaim claim, final OutputStream destination) throws IOException {
    if (claim == null) {
        return 0L;
    }

    try (final InputStream in = read(claim)) {
        return StreamUtils.copy(in, destination);
    }
}
 

@Test
public void testWriteWithNoContent() throws IOException {
    final ContentClaim claim1 = repository.create(false);
    try (final OutputStream out = repository.write(claim1)) {
        out.write("Hello".getBytes());
    }

    final ContentClaim claim2 = repository.create(false);
    assertEquals(claim1.getResourceClaim(), claim2.getResourceClaim());
    try (final OutputStream out = repository.write(claim2)) {

    }

    final ContentClaim claim3 = repository.create(false);
    assertEquals(claim1.getResourceClaim(), claim3.getResourceClaim());
    try (final OutputStream out = repository.write(claim3)) {
        out.write(" World".getBytes());
    }

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (final InputStream in = repository.read(claim1)) {
        StreamUtils.copy(in, baos);
    }

    assertEquals("Hello", baos.toString());

    baos.reset();
    try (final InputStream in = repository.read(claim2)) {
        StreamUtils.copy(in, baos);
    }
    assertEquals("", baos.toString());
    assertEquals(0, baos.size());

    baos.reset();
    try (final InputStream in = repository.read(claim3)) {
        StreamUtils.copy(in, baos);
    }
    assertEquals(" World", baos.toString());
}
 

@Override
public void process(InputStream in, OutputStream out) throws IOException {
    try (Base32OutputStream bos = new Base32OutputStream(out)) {
        StreamUtils.copy(in, bos);
    }
}
 

@Override
public long importFrom(final InputStream content, final ContentClaim claim) throws IOException {
    try (final OutputStream out = write(claim, false)) {
        return StreamUtils.copy(content, out);
    }
}
 

@Override
public void exportTo(final FlowFile source, final OutputStream destination) {
    validateRecordState(source);
    final StandardRepositoryRecord record = records.get(source);

    if(record.getCurrentClaim() == null) {
        return;
    }

    try {
        ensureNotAppending(record.getCurrentClaim());
        claimCache.flush(record.getCurrentClaim());
    } catch (final IOException e) {
        throw new FlowFileAccessException("Failed to access ContentClaim for " + source.toString(), e);
    }

    try (final InputStream rawIn = getInputStream(source, record.getCurrentClaim(), record.getCurrentClaimOffset(), true);
            final InputStream limitedIn = new LimitedInputStream(rawIn, source.getSize());
            final InputStream disableOnCloseIn = new DisableOnCloseInputStream(limitedIn);
            final ByteCountingInputStream countingStream = new ByteCountingInputStream(disableOnCloseIn, this.bytesRead)) {

        // We want to differentiate between IOExceptions thrown by the repository and IOExceptions thrown from
        // Processor code. As a result, as have the FlowFileAccessInputStream that catches IOException from the repository
        // and translates into either FlowFileAccessException or ContentNotFoundException. We keep track of any
        // ContentNotFoundException because if it is thrown, the Processor code may catch it and do something else with it
        // but in reality, if it is thrown, we want to know about it and handle it, even if the Processor code catches it.
        final FlowFileAccessInputStream ffais = new FlowFileAccessInputStream(countingStream, source, record.getCurrentClaim());
        boolean cnfeThrown = false;

        try {
            recursionSet.add(source);
            StreamUtils.copy(ffais, destination, source.getSize());
        } catch (final ContentNotFoundException cnfe) {
            cnfeThrown = true;
            throw cnfe;
        } finally {
            recursionSet.remove(source);
            IOUtils.closeQuietly(ffais);
            // if cnfeThrown is true, we don't need to re-throw the Exception; it will propagate.
            if (!cnfeThrown && ffais.getContentNotFoundException() != null) {
                throw ffais.getContentNotFoundException();
            }
        }

    } catch (final ContentNotFoundException nfe) {
        handleContentNotFound(nfe, record);
    } catch (final IOException ex) {
        throw new ProcessException("IOException thrown from " + connectableDescription + ": " + ex.toString(), ex);
    }
}
 

@Test(timeout=10000)
public void testRequestAddedToService() throws InitializationException, MalformedURLException, IOException, InterruptedException {
    final TestRunner runner = TestRunners.newTestRunner(HandleHttpRequest.class);
    runner.setProperty(HandleHttpRequest.PORT, "0");

    final MockHttpContextMap contextMap = new MockHttpContextMap();
    runner.addControllerService("http-context-map", contextMap);
    runner.enableControllerService(contextMap);
    runner.setProperty(HandleHttpRequest.HTTP_CONTEXT_MAP, "http-context-map");

    // trigger processor to stop but not shutdown.
    runner.run(1, false);
    try {
        final Thread httpThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    final int port = ((HandleHttpRequest) runner.getProcessor()).getPort();
                    final HttpURLConnection connection = (HttpURLConnection) new URL("http://localhost:"
                            + port + "/my/path?query=true&value1=value1&value2=&value3&value4=apple=orange").openConnection();
                    connection.setDoOutput(false);
                    connection.setRequestMethod("GET");
                    connection.setRequestProperty("header1", "value1");
                    connection.setRequestProperty("header2", "");
                    connection.setRequestProperty("header3", "apple=orange");
                    connection.setConnectTimeout(3000);
                    connection.setReadTimeout(3000);

                    StreamUtils.copy(connection.getInputStream(), new NullOutputStream());
                } catch (final Throwable t) {
                    t.printStackTrace();
                    Assert.fail(t.toString());
                }
            }
        });
        httpThread.start();

        while ( runner.getFlowFilesForRelationship(HandleHttpRequest.REL_SUCCESS).isEmpty() ) {
            // process the request.
            runner.run(1, false, false);
        }

        runner.assertAllFlowFilesTransferred(HandleHttpRequest.REL_SUCCESS, 1);
        assertEquals(1, contextMap.size());

        final MockFlowFile mff = runner.getFlowFilesForRelationship(HandleHttpRequest.REL_SUCCESS).get(0);
        mff.assertAttributeEquals("http.query.param.query", "true");
        mff.assertAttributeEquals("http.query.param.value1", "value1");
        mff.assertAttributeEquals("http.query.param.value2", "");
        mff.assertAttributeEquals("http.query.param.value3", "");
        mff.assertAttributeEquals("http.query.param.value4", "apple=orange");
        mff.assertAttributeEquals("http.headers.header1", "value1");
        mff.assertAttributeEquals("http.headers.header3", "apple=orange");
    } finally {
        // shut down the server
        runner.run(1, true);
    }
}
 

@Override
public void process(InputStream in, OutputStream out) throws IOException {
    try (Base32OutputStream bos = new Base32OutputStream(out)) {
        StreamUtils.copy(in, bos);
    }
}
 

private void testMerge(final String header, final String footer, final String demarcator) throws IOException {
    final int count = 4;
    final String content = "The quick brown fox jumps over the lazy dog";
    final List<ContentClaim> claims = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        final ContentClaim claim = repository.create(true);
        claims.add(claim);
        try (final OutputStream out = repository.write(claim)) {
            out.write(content.getBytes());
        }
    }

    final ContentClaim destination = repository.create(true);
    final byte[] headerBytes = header == null ? null : header.getBytes();
    final byte[] footerBytes = footer == null ? null : footer.getBytes();
    final byte[] demarcatorBytes = demarcator == null ? null : demarcator.getBytes();
    repository.merge(claims, destination, headerBytes, footerBytes, demarcatorBytes);

    final StringBuilder sb = new StringBuilder();
    if (header != null) {
        sb.append(header);
    }
    for (int i = 0; i < count; i++) {
        sb.append(content);
        if (demarcator != null && i != count - 1) {
            sb.append(demarcator);
        }
    }
    if (footer != null) {
        sb.append(footer);
    }
    final String expectedText = sb.toString();
    final byte[] expected = expectedText.getBytes();

    final ByteArrayOutputStream baos = new ByteArrayOutputStream((int) destination.getLength());
    try (final InputStream in = repository.read(destination)) {
        StreamUtils.copy(in, baos);
    }
    final byte[] actual = baos.toByteArray();
    assertTrue(Arrays.equals(expected, actual));
}
 

@Override
public void process(InputStream in, OutputStream out) throws IOException {
    try (Base64InputStream bis = new Base64InputStream(new ValidatingBase64InputStream(in))) {
        StreamUtils.copy(bis, out);
    }
}
 

public static String readContents(DataPacket packet) throws IOException {
    ByteArrayOutputStream os = new ByteArrayOutputStream((int) packet.getSize());
    StreamUtils.copy(packet.getData(), os);
    return new String(os.toByteArray(), "UTF-8");
}
 

/**
 * Gets the content for the specified flowfile in the specified connection.
 *
 * @param clientId      Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
 * @param connectionId  The connection id
 * @param flowFileUuid  The flowfile uuid
 * @param clusterNodeId The cluster node id
 * @return The content stream
 * @throws InterruptedException if interrupted
 */
@GET
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.WILDCARD)
@Path("{id}/flowfiles/{flowfile-uuid}/content")
@ApiOperation(
        value = "Gets the content for a FlowFile in a Connection.",
        authorizations = {
                @Authorization(value = "Read Source Data - /data/{component-type}/{uuid}", type = "")
        }
)
@ApiResponses(
        value = {
                @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
                @ApiResponse(code = 401, message = "Client could not be authenticated."),
                @ApiResponse(code = 403, message = "Client is not authorized to make this request."),
                @ApiResponse(code = 404, message = "The specified resource could not be found."),
                @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.")
        }
)
public Response downloadFlowFileContent(
        @ApiParam(
                value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.",
                required = false
        )
        @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) final ClientIdParameter clientId,
        @ApiParam(
                value = "The connection id.",
                required = true
        )
        @PathParam("id") final String connectionId,
        @ApiParam(
                value = "The flowfile uuid.",
                required = true
        )
        @PathParam("flowfile-uuid") final String flowFileUuid,
        @ApiParam(
                value = "The id of the node where the content exists if clustered.",
                required = false
        )
        @QueryParam("clusterNodeId") final String clusterNodeId) throws InterruptedException {

    // replicate if cluster manager
    if (isReplicateRequest()) {
        // determine where this request should be sent
        if (clusterNodeId == null) {
            throw new IllegalArgumentException("The id of the node in the cluster is required.");
        } else {
            // get the target node and ensure it exists
            final NodeIdentifier targetNode = getClusterCoordinator().getNodeIdentifier(clusterNodeId);
            if (targetNode == null) {
                throw new UnknownNodeException("The specified cluster node does not exist.");
            }

            return replicate(HttpMethod.GET, targetNode);
        }
    }

    // NOTE - deferred authorization so we can consider flowfile attributes in the access decision

    // get the uri of the request
    final String uri = generateResourceUri("flowfile-queues", connectionId, "flowfiles", flowFileUuid, "content");

    // get an input stream to the content
    final DownloadableContent content = serviceFacade.getContent(connectionId, flowFileUuid, uri);

    // generate a streaming response
    final StreamingOutput response = new StreamingOutput() {
        @Override
        public void write(final OutputStream output) throws IOException, WebApplicationException {
            try (InputStream is = content.getContent()) {
                // stream the content to the response
                StreamUtils.copy(is, output);

                // flush the response
                output.flush();
            }
        }
    };

    // use the appropriate content type
    String contentType = content.getType();
    if (contentType == null) {
        contentType = MediaType.APPLICATION_OCTET_STREAM;
    }

    return generateOkResponse(response).type(contentType).header("Content-Disposition", String.format("attachment; filename=\"%s\"", content.getFilename())).build();
}