Java Code Examples for org.apache.nifi.util.FormatUtils#formatDataSize()

static void checkThreshold(final String pathName, final Path path, final int threshold, final ComponentLog logger) {
    final File file = path.toFile();
    final long totalBytes = file.getTotalSpace();
    final long freeBytes = file.getFreeSpace();
    final long usedBytes = totalBytes - freeBytes;

    final double usedPercent = (double) usedBytes / (double) totalBytes * 100D;

    if (usedPercent >= threshold) {
        final String usedSpace = FormatUtils.formatDataSize(usedBytes);
        final String totalSpace = FormatUtils.formatDataSize(totalBytes);
        final String freeSpace = FormatUtils.formatDataSize(freeBytes);

        final double freePercent = (double) freeBytes / (double) totalBytes * 100D;

        final String message = String.format("%1$s exceeds configured threshold of %2$s%%, having %3$s / %4$s (%5$.2f%%) used and %6$s (%7$.2f%%) free",
                pathName, threshold, usedSpace, totalSpace, usedPercent, freeSpace, freePercent);
        logger.warn(message);
    }
}
 

private String toDiff(final long oldValue, final long newValue, final boolean formatDataSize, final boolean formatTime) {
    if (formatDataSize && formatTime) {
        throw new IllegalArgumentException("Cannot format units as both data size and time");
    }

    final long diff = Math.abs(newValue - oldValue);

    final String formattedDiff = formatDataSize ? FormatUtils.formatDataSize(diff)
            : (formatTime ? FormatUtils.formatHoursMinutesSeconds(diff, TimeUnit.NANOSECONDS) : String.valueOf(diff));

    if (oldValue > newValue) {
        return " (-" + formattedDiff + ")";
    } else {
        return " (+" + formattedDiff + ")";
    }
}
 

private String toDiff(final long oldValue, final long newValue, final boolean formatDataSize, final boolean formatTime) {
    if (formatDataSize && formatTime) {
        throw new IllegalArgumentException("Cannot format units as both data size and time");
    }

    final long diff = Math.abs(newValue - oldValue);

    final String formattedDiff = formatDataSize ? FormatUtils.formatDataSize(diff)
            : (formatTime ? FormatUtils.formatHoursMinutesSeconds(diff, TimeUnit.NANOSECONDS) : String.valueOf(diff));

    if (oldValue > newValue) {
        return " (-" + formattedDiff + ")";
    } else {
        return " (+" + formattedDiff + ")";
    }
}
 

static void checkThreshold(final String pathName, final Path path, final int threshold, final ComponentLog logger) {
    final File file = path.toFile();
    final long totalBytes = file.getTotalSpace();
    final long freeBytes = file.getFreeSpace();
    final long usedBytes = totalBytes - freeBytes;

    final double usedPercent = (double) usedBytes / (double) totalBytes * 100D;

    if (usedPercent >= threshold) {
        final String usedSpace = FormatUtils.formatDataSize(usedBytes);
        final String totalSpace = FormatUtils.formatDataSize(totalBytes);
        final String freeSpace = FormatUtils.formatDataSize(freeBytes);

        final double freePercent = (double) freeBytes / (double) totalBytes * 100D;

        final String message = String.format("%1$s exceeds configured threshold of %2$s%%, having %3$s / %4$s (%5$.2f%%) used and %6$s (%7$.2f%%) free",
                pathName, threshold, usedSpace, totalSpace, usedPercent, freeSpace, freePercent);
        logger.warn(message);
    }
}
 

public static String formatDataSize(final Long longStatus) {
    if (longStatus == null) {
        return EMPTY_BYTES;
    }
    if (longStatus == 0L) {
        return ZERO_BYTES;
    }

    return FormatUtils.formatDataSize(longStatus);
}
 

@Override
public String toString() {
    return "HeapSize[memoryPool=" + memoryPoolName + ", " + FormatUtils.formatDataSize(usedBefore) + " => " + FormatUtils.formatDataSize(usedAfter) + "]";
}
 

@Override
protected void doDelete(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final X509Certificate[] certs = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
    String foundSubject = DEFAULT_FOUND_SUBJECT;
    if (certs != null && certs.length > 0) {
        for (final X509Certificate cert : certs) {
            foundSubject = cert.getSubjectDN().getName();
            if (authorizedPattern.matcher(foundSubject).matches()) {
                break;
            } else {
                logger.warn(processor + " rejecting transfer attempt from " + foundSubject + " because the DN is not authorized");
                response.sendError(HttpServletResponse.SC_FORBIDDEN, "not allowed based on dn");
                return;
            }
        }
    }

    final String uri = request.getRequestURI();
    final int slashIndex = uri.lastIndexOf("/");
    int questionIndex = uri.indexOf("?");
    if (questionIndex < 0) {
        questionIndex = uri.length();
    }

    final String uuid = uri.substring(slashIndex + 1, questionIndex);
    final FlowFileEntryTimeWrapper timeWrapper = flowFileMap.remove(uuid);
    if (timeWrapper == null) {
        logger.warn("received DELETE for HOLD with ID " + uuid + " from Remote Host: [" + request.getRemoteHost()
                + "] Port [" + request.getRemotePort() + "] SubjectDN [" + foundSubject + "], but no HOLD exists with that ID; sending response with Status Code 404");
        response.sendError(HttpServletResponse.SC_NOT_FOUND);
        return;
    }

    try {
        final Set<FlowFile> flowFiles = timeWrapper.getFlowFiles();

        final long transferTime = System.currentTimeMillis() - timeWrapper.getEntryTime();
        long totalFlowFileSize = 0;
        for (final FlowFile flowFile : flowFiles) {
            totalFlowFileSize += flowFile.getSize();
        }

        double seconds = (double) transferTime / 1000D;
        if (seconds <= 0D) {
            seconds = .00000001D;
        }
        final double bytesPerSecond = ((double) totalFlowFileSize / seconds);
        final String transferRate = FormatUtils.formatDataSize(bytesPerSecond) + "/sec";

        logger.info("received {} files/{} bytes from Remote Host: [{}] Port [{}] SubjectDN [{}] in {} milliseconds at a rate of {}; "
                + "transferring to 'success': {}",
                new Object[]{flowFiles.size(), totalFlowFileSize, request.getRemoteHost(), request.getRemotePort(), foundSubject, transferTime, transferRate, flowFiles});

        final ProcessSession session = timeWrapper.getSession();
        session.transfer(flowFiles, ListenHTTP.RELATIONSHIP_SUCCESS);
        session.commit();

        response.setStatus(HttpServletResponse.SC_OK);
        response.flushBuffer();
    } catch (final Throwable t) {
        timeWrapper.getSession().rollback();
        logger.error("received DELETE for HOLD with ID {} from Remote Host: [{}] Port [{}] SubjectDN [{}], but failed to process the request due to {}",
                new Object[]{uuid, request.getRemoteHost(), request.getRemotePort(), foundSubject, t.toString()});
        if (logger.isDebugEnabled()) {
            logger.error("", t);
        }

        response.sendError(HttpServletResponse.SC_NOT_FOUND);
    }
}
 

private int transferFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session, final FlowFile firstFlowFile) throws IOException, ProtocolException {
    FlowFile flowFile = firstFlowFile;

    try {
        final String userDn = transaction.getCommunicant().getDistinguishedName();
        final long startSendingNanos = System.nanoTime();
        final StopWatch stopWatch = new StopWatch(true);
        long bytesSent = 0L;

        final SiteToSiteClientConfig siteToSiteClientConfig = getSiteToSiteClient().getConfig();
        final long maxBatchBytes = siteToSiteClientConfig.getPreferredBatchSize();
        final int maxBatchCount = siteToSiteClientConfig.getPreferredBatchCount();
        final long preferredBatchDuration = siteToSiteClientConfig.getPreferredBatchDuration(TimeUnit.NANOSECONDS);
        final long maxBatchDuration = preferredBatchDuration > 0 ? preferredBatchDuration : BATCH_SEND_NANOS;


        final Set<FlowFile> flowFilesSent = new HashSet<>();
        boolean continueTransaction = true;
        while (continueTransaction) {
            final long startNanos = System.nanoTime();
            // call codec.encode within a session callback so that we have the InputStream to read the FlowFile
            final FlowFile toWrap = flowFile;
            session.read(flowFile, new InputStreamCallback() {
                @Override
                public void process(final InputStream in) throws IOException {
                    final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
                    transaction.send(dataPacket);
                }
            });

            final long transferNanos = System.nanoTime() - startNanos;
            final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);

            flowFilesSent.add(flowFile);
            bytesSent += flowFile.getSize();
            logger.debug("{} Sent {} to {}", this, flowFile, transaction.getCommunicant().getUrl());

            final String transitUri = transaction.getCommunicant().createTransitUri(flowFile.getAttribute(CoreAttributes.UUID.key()));
            flowFile = session.putAttribute(flowFile, SiteToSiteAttributes.S2S_PORT_ID.key(), getTargetIdentifier());
            session.getProvenanceReporter().send(flowFile, transitUri, "Remote DN=" + userDn, transferMillis, false);
            session.remove(flowFile);

            final long sendingNanos = System.nanoTime() - startSendingNanos;

            if (maxBatchCount > 0 && flowFilesSent.size() >= maxBatchCount) {
                flowFile = null;
            } else if (maxBatchBytes > 0 && bytesSent >= maxBatchBytes) {
                flowFile = null;
            } else if (sendingNanos >= maxBatchDuration) {
                flowFile = null;
            } else {
                flowFile = session.get();
            }

            continueTransaction = (flowFile != null);
        }

        transaction.confirm();

        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        stopWatch.stop();
        final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesSent);

        transaction.complete();
        session.commit();

        final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
        logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
            this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});

        return flowFilesSent.size();
    } catch (final Exception e) {
        session.rollback();
        throw e;
    }

}
 

protected int commitReceiveTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
    CommunicationsSession commsSession = peer.getCommunicationsSession();
    ProcessSession session = transaction.getSession();
    final Response confirmTransactionResponse = readTransactionResponse(false, commsSession);
    logger.debug("{} Received {} from {}", this, confirmTransactionResponse, peer);

    switch (confirmTransactionResponse.getCode()) {
        case CONFIRM_TRANSACTION:
            break;
        case BAD_CHECKSUM:
            session.rollback();
            throw new IOException(this + " Received a BadChecksum response from peer " + peer);
        default:
            throw new ProtocolException(this + " Received unexpected Response Code from peer " + peer + " : " + confirmTransactionResponse + "; expected 'Confirm Transaction' Response Code");
    }

    // Commit the session so that we have persisted the data
    session.commit();

    if (transaction.getContext().getAvailableRelationships().isEmpty()) {
        // Confirm that we received the data and the peer can now discard it but that the peer should not
        // send any more data for a bit
        logger.debug("{} Sending TRANSACTION_FINISHED_BUT_DESTINATION_FULL to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL, commsSession);
    } else {
        // Confirm that we received the data and the peer can now discard it
        logger.debug("{} Sending TRANSACTION_FINISHED to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED, commsSession);
    }

    Set<FlowFile> flowFilesReceived = transaction.getFlowFilesSent();
    long bytesReceived = transaction.getBytesSent();
    StopWatch stopWatch = transaction.getStopWatch();
    stopWatch.stop();
    final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
    final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
    final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    final String dataSize = FormatUtils.formatDataSize(bytesReceived);
    logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
        this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});

    return flowFilesReceived.size();
}
 

private void printProcessorStatus(final ProcessGroupStatus groupStatus, final StringBuilder builder, final boolean showDeltas) {
    final List<ProcessorStatus> processorStatuses = new ArrayList<>();
    populateProcessorStatuses(groupStatus, processorStatuses);
    Collections.sort(processorStatuses, new Comparator<ProcessorStatus>() {
        @Override
        public int compare(final ProcessorStatus o1, final ProcessorStatus o2) {
            if (o1 == null && o2 == null) {
                return 0;
            }
            if (o1 == null) {
                return 1;
            }
            if (o2 == null) {
                return -1;
            }

            return -Long.compare(o1.getProcessingNanos(), o2.getProcessingNanos());
        }
    });

    for (final ProcessorStatus processorStatus : processorStatuses) {
        // get the stats
        final String input = processorStatus.getInputCount() + " / " + FormatUtils.formatDataSize(processorStatus.getInputBytes());
        final String output = processorStatus.getOutputCount() + " / " + FormatUtils.formatDataSize(processorStatus.getOutputBytes());
        final String read = FormatUtils.formatDataSize(processorStatus.getBytesRead());
        final String written = FormatUtils.formatDataSize(processorStatus.getBytesWritten());
        final String invocations = String.valueOf(processorStatus.getInvocations());

        final long nanos = processorStatus.getProcessingNanos();
        final String procTime = FormatUtils.formatHoursMinutesSeconds(nanos, TimeUnit.NANOSECONDS);

        String runStatus = "";
        if (processorStatus.getRunStatus() != null) {
            runStatus = processorStatus.getRunStatus().toString();
        }

        final String inputDiff;
        final String outputDiff;
        final String readDiff;
        final String writtenDiff;
        final String invocationsDiff;
        final String procTimeDiff;

        final ProcessorStatus lastStatus = lastProcessorStatus.get(processorStatus.getId());
        if (showDeltas && lastStatus != null) {
            inputDiff = toDiff(lastStatus.getInputCount(), lastStatus.getInputBytes(), processorStatus.getInputCount(), processorStatus.getInputBytes());
            outputDiff = toDiff(lastStatus.getOutputCount(), lastStatus.getOutputBytes(), processorStatus.getOutputCount(), processorStatus.getOutputBytes());
            readDiff = toDiff(lastStatus.getBytesRead(), processorStatus.getBytesRead(), true, false);
            writtenDiff = toDiff(lastStatus.getBytesWritten(), processorStatus.getBytesWritten(), true, false);
            invocationsDiff = toDiff(lastStatus.getInvocations(), processorStatus.getInvocations());
            procTimeDiff = toDiff(lastStatus.getProcessingNanos(), processorStatus.getProcessingNanos(), false, true);
        } else {
            inputDiff = toDiff(0L, 0L, processorStatus.getInputCount(), processorStatus.getInputBytes());
            outputDiff = toDiff(0L, 0L, processorStatus.getOutputCount(), processorStatus.getOutputBytes());
            readDiff = toDiff(0L, processorStatus.getBytesRead(), true, false);
            writtenDiff = toDiff(0L, processorStatus.getBytesWritten(), true, false);
            invocationsDiff = toDiff(0L, processorStatus.getInvocations());
            procTimeDiff = toDiff(0L, processorStatus.getProcessingNanos(), false, true);
        }

        if (showDeltas) {
            builder.append(String.format(processorLineFormat,
                    processorStatus.getName(),
                    processorStatus.getId(),
                    processorStatus.getType(),
                    runStatus,
                    input + inputDiff,
                    output + outputDiff,
                    read + readDiff,
                    written + writtenDiff,
                    invocations + invocationsDiff,
                    procTime + procTimeDiff));
        } else {
            builder.append(String.format(processorLineFormat,
                    processorStatus.getName(),
                    processorStatus.getId(),
                    processorStatus.getType(),
                    runStatus,
                    input,
                    output,
                    read,
                    written,
                    invocations,
                    procTime));
        }

        lastProcessorStatus.put(processorStatus.getId(), processorStatus);
    }
}
 

private void printConnectionStatus(final ProcessGroupStatus groupStatus, final StringBuilder builder, final boolean showDeltas) {
    final List<ConnectionStatus> connectionStatuses = new ArrayList<>();
    populateConnectionStatuses(groupStatus, connectionStatuses);
    Collections.sort(connectionStatuses, new Comparator<ConnectionStatus>() {
        @Override
        public int compare(final ConnectionStatus o1, final ConnectionStatus o2) {
            if (o1 == null && o2 == null) {
                return 0;
            }
            if (o1 == null) {
                return 1;
            }
            if (o2 == null) {
                return -1;
            }

            return -Long.compare(o1.getQueuedBytes(), o2.getQueuedBytes());
        }
    });

    for (final ConnectionStatus connectionStatus : connectionStatuses) {
        final String input = connectionStatus.getInputCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getInputBytes());
        final String output = connectionStatus.getOutputCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getOutputBytes());
        final String queued = connectionStatus.getQueuedCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getQueuedBytes());

        final String inputDiff;
        final String outputDiff;
        final String queuedDiff;

        final ConnectionStatus lastStatus = lastConnectionStatus.get(connectionStatus.getId());
        if (showDeltas && lastStatus != null) {
            inputDiff = toDiff(lastStatus.getInputCount(), lastStatus.getInputBytes(), connectionStatus.getInputCount(), connectionStatus.getInputBytes());
            outputDiff = toDiff(lastStatus.getOutputCount(), lastStatus.getOutputBytes(), connectionStatus.getOutputCount(), connectionStatus.getOutputBytes());
            queuedDiff = toDiff(lastStatus.getQueuedCount(), lastStatus.getQueuedBytes(), connectionStatus.getQueuedCount(), connectionStatus.getQueuedBytes());
        } else {
            inputDiff = toDiff(0L, 0L, connectionStatus.getInputCount(), connectionStatus.getInputBytes());
            outputDiff = toDiff(0L, 0L, connectionStatus.getOutputCount(), connectionStatus.getOutputBytes());
            queuedDiff = toDiff(0L, 0L, connectionStatus.getQueuedCount(), connectionStatus.getQueuedBytes());
        }

        if (showDeltas) {
            builder.append(String.format(connectionLineFormat,
                    connectionStatus.getId(),
                    connectionStatus.getSourceName(),
                    connectionStatus.getName(),
                    connectionStatus.getDestinationName(),
                    input + inputDiff,
                    output + outputDiff,
                    queued + queuedDiff));
        } else {
            builder.append(String.format(connectionLineFormat,
                    connectionStatus.getId(),
                    connectionStatus.getSourceName(),
                    connectionStatus.getName(),
                    connectionStatus.getDestinationName(),
                    input,
                    output,
                    queued));
        }

        lastConnectionStatus.put(connectionStatus.getId(), connectionStatus);
    }
}
 

@Override
protected void doDelete(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final X509Certificate[] certs = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
    String foundSubject = DEFAULT_FOUND_SUBJECT;
    if (certs != null && certs.length > 0) {
        for (final X509Certificate cert : certs) {
            foundSubject = cert.getSubjectDN().getName();
            if (authorizedPattern.matcher(foundSubject).matches()) {
                break;
            } else {
                logger.warn(processor + " rejecting transfer attempt from " + foundSubject + " because the DN is not authorized");
                response.sendError(HttpServletResponse.SC_FORBIDDEN, "not allowed based on dn");
                return;
            }
        }
    }

    final String uri = request.getRequestURI();
    final int slashIndex = uri.lastIndexOf("/");
    int questionIndex = uri.indexOf("?");
    if (questionIndex < 0) {
        questionIndex = uri.length();
    }

    final String uuid = uri.substring(slashIndex + 1, questionIndex);
    final FlowFileEntryTimeWrapper timeWrapper = flowFileMap.remove(uuid);
    if (timeWrapper == null) {
        logger.warn("received DELETE for HOLD with ID " + uuid + " from Remote Host: [" + request.getRemoteHost()
                + "] Port [" + request.getRemotePort() + "] SubjectDN [" + foundSubject + "], but no HOLD exists with that ID; sending response with Status Code 404");
        response.sendError(HttpServletResponse.SC_NOT_FOUND);
        return;
    }

    try {
        final Set<FlowFile> flowFiles = timeWrapper.getFlowFiles();

        final long transferTime = System.currentTimeMillis() - timeWrapper.getEntryTime();
        long totalFlowFileSize = 0;
        for (final FlowFile flowFile : flowFiles) {
            totalFlowFileSize += flowFile.getSize();
        }

        double seconds = (double) transferTime / 1000D;
        if (seconds <= 0D) {
            seconds = .00000001D;
        }
        final double bytesPerSecond = ((double) totalFlowFileSize / seconds);
        final String transferRate = FormatUtils.formatDataSize(bytesPerSecond) + "/sec";

        logger.info("received {} files/{} bytes from Remote Host: [{}] Port [{}] SubjectDN [{}] in {} milliseconds at a rate of {}; "
                + "transferring to 'success': {}",
                new Object[]{flowFiles.size(), totalFlowFileSize, request.getRemoteHost(), request.getRemotePort(), foundSubject, transferTime, transferRate, flowFiles});

        final ProcessSession session = timeWrapper.getSession();
        session.transfer(flowFiles, ListenHTTP.RELATIONSHIP_SUCCESS);
        session.commit();

        response.setStatus(HttpServletResponse.SC_OK);
        response.flushBuffer();
    } catch (final Throwable t) {
        timeWrapper.getSession().rollback();
        logger.error("received DELETE for HOLD with ID {} from Remote Host: [{}] Port [{}] SubjectDN [{}], but failed to process the request due to {}",
                new Object[]{uuid, request.getRemoteHost(), request.getRemotePort(), foundSubject, t.toString()});
        if (logger.isDebugEnabled()) {
            logger.error("", t);
        }

        response.sendError(HttpServletResponse.SC_NOT_FOUND);
    }
}
 

private int receiveFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session) throws IOException, ProtocolException {
    final String userDn = transaction.getCommunicant().getDistinguishedName();

    final StopWatch stopWatch = new StopWatch(true);
    final Set<FlowFile> flowFilesReceived = new HashSet<>();
    long bytesReceived = 0L;

    while (true) {
        final long start = System.nanoTime();
        final DataPacket dataPacket = transaction.receive();
        if (dataPacket == null) {
            break;
        }

        FlowFile flowFile = session.create();
        flowFile = session.putAllAttributes(flowFile, dataPacket.getAttributes());

        final Communicant communicant = transaction.getCommunicant();
        final String host = StringUtils.isEmpty(communicant.getHost()) ? "unknown" : communicant.getHost();
        final String port = communicant.getPort() < 0 ? "unknown" : String.valueOf(communicant.getPort());

        final Map<String,String> attributes = new HashMap<>(2);
        attributes.put(SiteToSiteAttributes.S2S_HOST.key(), host);
        attributes.put(SiteToSiteAttributes.S2S_ADDRESS.key(), host + ":" + port);

        flowFile = session.putAllAttributes(flowFile, attributes);

        flowFile = session.importFrom(dataPacket.getData(), flowFile);
        final long receiveNanos = System.nanoTime() - start;
        flowFilesReceived.add(flowFile);

        String sourceFlowFileIdentifier = dataPacket.getAttributes().get(CoreAttributes.UUID.key());
        if (sourceFlowFileIdentifier == null) {
            sourceFlowFileIdentifier = "<Unknown Identifier>";
        }

        final String transitUri = transaction.getCommunicant().createTransitUri(sourceFlowFileIdentifier);
        session.getProvenanceReporter().receive(flowFile, transitUri, "urn:nifi:" + sourceFlowFileIdentifier,
                "Remote DN=" + userDn, TimeUnit.NANOSECONDS.toMillis(receiveNanos));

        session.transfer(flowFile, Relationship.ANONYMOUS);
        bytesReceived += dataPacket.getSize();
    }

    // Confirm that what we received was the correct data.
    transaction.confirm();

    // Commit the session so that we have persisted the data
    session.commit();

    transaction.complete();

    if (!flowFilesReceived.isEmpty()) {
        stopWatch.stop();
        final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
        final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesReceived);
        logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
            this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});
    }

    return flowFilesReceived.size();
}
 

private int transferFlowFiles(final Transaction transaction, final ProcessContext context, final ProcessSession session, final FlowFile firstFlowFile) throws IOException, ProtocolException {
    FlowFile flowFile = firstFlowFile;

    try {
        final String userDn = transaction.getCommunicant().getDistinguishedName();
        final long startSendingNanos = System.nanoTime();
        final StopWatch stopWatch = new StopWatch(true);
        long bytesSent = 0L;

        final Set<FlowFile> flowFilesSent = new HashSet<>();
        boolean continueTransaction = true;
        while (continueTransaction) {
            final long startNanos = System.nanoTime();
            // call codec.encode within a session callback so that we have the InputStream to read the FlowFile
            final FlowFile toWrap = flowFile;
            session.read(flowFile, new InputStreamCallback() {
                @Override
                public void process(final InputStream in) throws IOException {
                    final DataPacket dataPacket = new StandardDataPacket(toWrap.getAttributes(), in, toWrap.getSize());
                    transaction.send(dataPacket);
                }
            });

            final long transferNanos = System.nanoTime() - startNanos;
            final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);

            flowFilesSent.add(flowFile);
            bytesSent += flowFile.getSize();
            logger.debug("{} Sent {} to {}", this, flowFile, transaction.getCommunicant().getUrl());

            final String transitUri = transaction.getCommunicant().createTransitUri(flowFile.getAttribute(CoreAttributes.UUID.key()));
            session.getProvenanceReporter().send(flowFile, transitUri, "Remote DN=" + userDn, transferMillis, false);
            session.remove(flowFile);

            final long sendingNanos = System.nanoTime() - startSendingNanos;
            if (sendingNanos < BATCH_SEND_NANOS) {
                flowFile = session.get();
            } else {
                flowFile = null;
            }

            continueTransaction = (flowFile != null);
        }

        transaction.confirm();

        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        stopWatch.stop();
        final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
        final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
        final String dataSize = FormatUtils.formatDataSize(bytesSent);

        transaction.complete();
        session.commit();

        final String flowFileDescription = (flowFilesSent.size() < 20) ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
        logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
            this, flowFileDescription, dataSize, transaction.getCommunicant().getUrl(), uploadMillis, uploadDataRate});

        return flowFilesSent.size();
    } catch (final Exception e) {
        session.rollback();
        throw e;
    }

}
 

protected int commitReceiveTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
    CommunicationsSession commsSession = peer.getCommunicationsSession();
    ProcessSession session = transaction.getSession();
    final Response confirmTransactionResponse = readTransactionResponse(false, commsSession);
    logger.debug("{} Received {} from {}", this, confirmTransactionResponse, peer);

    switch (confirmTransactionResponse.getCode()) {
        case CONFIRM_TRANSACTION:
            break;
        case BAD_CHECKSUM:
            session.rollback();
            throw new IOException(this + " Received a BadChecksum response from peer " + peer);
        default:
            throw new ProtocolException(this + " Received unexpected Response Code from peer " + peer + " : " + confirmTransactionResponse + "; expected 'Confirm Transaction' Response Code");
    }

    // Commit the session so that we have persisted the data
    session.commit();

    if (transaction.getContext().getAvailableRelationships().isEmpty()) {
        // Confirm that we received the data and the peer can now discard it but that the peer should not
        // send any more data for a bit
        logger.debug("{} Sending TRANSACTION_FINISHED_BUT_DESTINATION_FULL to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL, commsSession);
    } else {
        // Confirm that we received the data and the peer can now discard it
        logger.debug("{} Sending TRANSACTION_FINISHED to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED, commsSession);
    }

    Set<FlowFile> flowFilesReceived = transaction.getFlowFilesSent();
    long bytesReceived = transaction.getBytesSent();
    StopWatch stopWatch = transaction.getStopWatch();
    stopWatch.stop();
    final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
    final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
    final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    final String dataSize = FormatUtils.formatDataSize(bytesReceived);
    logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[]{
        this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});

    return flowFilesReceived.size();
}
 

protected int commitTransferTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
    ProcessSession session = transaction.getSession();
    Set<FlowFile> flowFilesSent = transaction.getFlowFilesSent();

    // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response
    CommunicationsSession commsSession = peer.getCommunicationsSession();
    final Response transactionConfirmationResponse = readTransactionResponse(true, commsSession);
    if (transactionConfirmationResponse.getCode() == ResponseCode.CONFIRM_TRANSACTION) {
        // Confirm Checksum and echo back the confirmation.
        logger.debug("{} Received {}  from {}", this, transactionConfirmationResponse, peer);
        final String receivedCRC = transactionConfirmationResponse.getMessage();

        if (getVersionNegotiator().getVersion() > 3) {
            String calculatedCRC = transaction.getCalculatedCRC();
            if (!receivedCRC.equals(calculatedCRC)) {
                writeTransactionResponse(true, ResponseCode.BAD_CHECKSUM, commsSession);
                session.rollback();
                throw new IOException(this + " Sent data to peer " + peer + " but calculated CRC32 Checksum as "
                        + calculatedCRC + " while peer calculated CRC32 Checksum as " + receivedCRC
                        + "; canceling transaction and rolling back session");
            }
        }

        writeTransactionResponse(true, ResponseCode.CONFIRM_TRANSACTION, commsSession, "");

    } else {
        throw new ProtocolException("Expected to receive 'Confirm Transaction' response from peer " + peer + " but received " + transactionConfirmationResponse);
    }

    final String flowFileDescription = flowFilesSent.size() < 20 ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";

    final Response transactionResponse;
    try {
        transactionResponse = readTransactionResponse(true, commsSession);
    } catch (final IOException e) {
        logger.error("{} Failed to receive a response from {} when expecting a TransactionFinished Indicator."
                + " It is unknown whether or not the peer successfully received/processed the data."
                + " Therefore, {} will be rolled back, possibly resulting in data duplication of {}",
                this, peer, session, flowFileDescription);
        session.rollback();
        throw e;
    }

    logger.debug("{} received {} from {}", new Object[]{this, transactionResponse, peer});
    if (transactionResponse.getCode() == ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL) {
        peer.penalize(port.getIdentifier(), port.getYieldPeriod(TimeUnit.MILLISECONDS));
    } else if (transactionResponse.getCode() != ResponseCode.TRANSACTION_FINISHED) {
        throw new ProtocolException("After sending data, expected TRANSACTION_FINISHED response but got " + transactionResponse);
    }

    session.commit();

    StopWatch stopWatch = transaction.getStopWatch();
    long bytesSent = transaction.getBytesSent();
    stopWatch.stop();
    final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
    final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    final String dataSize = FormatUtils.formatDataSize(bytesSent);
    logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[]{
        this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate});

    return flowFilesSent.size();
}
 

public static String formatDataSize(final Long longStatus) {
    return longStatus == null ? "-" : FormatUtils.formatDataSize(longStatus);
}
 

private void printProcessorStatus(final ProcessGroupStatus groupStatus, final StringBuilder builder, final boolean showDeltas) {
    final List<ProcessorStatus> processorStatuses = new ArrayList<>();
    populateProcessorStatuses(groupStatus, processorStatuses);
    Collections.sort(processorStatuses, new Comparator<ProcessorStatus>() {
        @Override
        public int compare(final ProcessorStatus o1, final ProcessorStatus o2) {
            if (o1 == null && o2 == null) {
                return 0;
            }
            if (o1 == null) {
                return 1;
            }
            if (o2 == null) {
                return -1;
            }

            return -Long.compare(o1.getProcessingNanos(), o2.getProcessingNanos());
        }
    });

    for (final ProcessorStatus processorStatus : processorStatuses) {
        // get the stats
        final String input = processorStatus.getInputCount() + " / " + FormatUtils.formatDataSize(processorStatus.getInputBytes());
        final String output = processorStatus.getOutputCount() + " / " + FormatUtils.formatDataSize(processorStatus.getOutputBytes());
        final String read = FormatUtils.formatDataSize(processorStatus.getBytesRead());
        final String written = FormatUtils.formatDataSize(processorStatus.getBytesWritten());
        final String invocations = String.valueOf(processorStatus.getInvocations());

        final long nanos = processorStatus.getProcessingNanos();
        final String procTime = FormatUtils.formatHoursMinutesSeconds(nanos, TimeUnit.NANOSECONDS);

        String runStatus = "";
        if (processorStatus.getRunStatus() != null) {
            runStatus = processorStatus.getRunStatus().toString();
        }

        final String inputDiff;
        final String outputDiff;
        final String readDiff;
        final String writtenDiff;
        final String invocationsDiff;
        final String procTimeDiff;

        final ProcessorStatus lastStatus = lastProcessorStatus.get(processorStatus.getId());
        if (showDeltas && lastStatus != null) {
            inputDiff = toDiff(lastStatus.getInputCount(), lastStatus.getInputBytes(), processorStatus.getInputCount(), processorStatus.getInputBytes());
            outputDiff = toDiff(lastStatus.getOutputCount(), lastStatus.getOutputBytes(), processorStatus.getOutputCount(), processorStatus.getOutputBytes());
            readDiff = toDiff(lastStatus.getBytesRead(), processorStatus.getBytesRead(), true, false);
            writtenDiff = toDiff(lastStatus.getBytesWritten(), processorStatus.getBytesWritten(), true, false);
            invocationsDiff = toDiff(lastStatus.getInvocations(), processorStatus.getInvocations());
            procTimeDiff = toDiff(lastStatus.getProcessingNanos(), processorStatus.getProcessingNanos(), false, true);
        } else {
            inputDiff = toDiff(0L, 0L, processorStatus.getInputCount(), processorStatus.getInputBytes());
            outputDiff = toDiff(0L, 0L, processorStatus.getOutputCount(), processorStatus.getOutputBytes());
            readDiff = toDiff(0L, processorStatus.getBytesRead(), true, false);
            writtenDiff = toDiff(0L, processorStatus.getBytesWritten(), true, false);
            invocationsDiff = toDiff(0L, processorStatus.getInvocations());
            procTimeDiff = toDiff(0L, processorStatus.getProcessingNanos(), false, true);
        }

        if (showDeltas) {
            builder.append(String.format(processorLineFormat,
                    processorStatus.getName(),
                    processorStatus.getId(),
                    processorStatus.getType(),
                    runStatus,
                    input + inputDiff,
                    output + outputDiff,
                    read + readDiff,
                    written + writtenDiff,
                    invocations + invocationsDiff,
                    procTime + procTimeDiff));
        } else {
            builder.append(String.format(processorLineFormat,
                    processorStatus.getName(),
                    processorStatus.getId(),
                    processorStatus.getType(),
                    runStatus,
                    input,
                    output,
                    read,
                    written,
                    invocations,
                    procTime));
        }

        lastProcessorStatus.put(processorStatus.getId(), processorStatus);
    }
}
 

private void printConnectionStatus(final ProcessGroupStatus groupStatus, final StringBuilder builder, final boolean showDeltas) {
    final List<ConnectionStatus> connectionStatuses = new ArrayList<>();
    populateConnectionStatuses(groupStatus, connectionStatuses);
    Collections.sort(connectionStatuses, new Comparator<ConnectionStatus>() {
        @Override
        public int compare(final ConnectionStatus o1, final ConnectionStatus o2) {
            if (o1 == null && o2 == null) {
                return 0;
            }
            if (o1 == null) {
                return 1;
            }
            if (o2 == null) {
                return -1;
            }

            return -Long.compare(o1.getQueuedBytes(), o2.getQueuedBytes());
        }
    });

    for (final ConnectionStatus connectionStatus : connectionStatuses) {
        final String input = connectionStatus.getInputCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getInputBytes());
        final String output = connectionStatus.getOutputCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getOutputBytes());
        final String queued = connectionStatus.getQueuedCount() + " / " + FormatUtils.formatDataSize(connectionStatus.getQueuedBytes());

        final String inputDiff;
        final String outputDiff;
        final String queuedDiff;

        final ConnectionStatus lastStatus = lastConnectionStatus.get(connectionStatus.getId());
        if (showDeltas && lastStatus != null) {
            inputDiff = toDiff(lastStatus.getInputCount(), lastStatus.getInputBytes(), connectionStatus.getInputCount(), connectionStatus.getInputBytes());
            outputDiff = toDiff(lastStatus.getOutputCount(), lastStatus.getOutputBytes(), connectionStatus.getOutputCount(), connectionStatus.getOutputBytes());
            queuedDiff = toDiff(lastStatus.getQueuedCount(), lastStatus.getQueuedBytes(), connectionStatus.getQueuedCount(), connectionStatus.getQueuedBytes());
        } else {
            inputDiff = toDiff(0L, 0L, connectionStatus.getInputCount(), connectionStatus.getInputBytes());
            outputDiff = toDiff(0L, 0L, connectionStatus.getOutputCount(), connectionStatus.getOutputBytes());
            queuedDiff = toDiff(0L, 0L, connectionStatus.getQueuedCount(), connectionStatus.getQueuedBytes());
        }

        if (showDeltas) {
            builder.append(String.format(connectionLineFormat,
                    connectionStatus.getId(),
                    connectionStatus.getSourceName(),
                    connectionStatus.getName(),
                    connectionStatus.getDestinationName(),
                    input + inputDiff,
                    output + outputDiff,
                    queued + queuedDiff));
        } else {
            builder.append(String.format(connectionLineFormat,
                    connectionStatus.getId(),
                    connectionStatus.getSourceName(),
                    connectionStatus.getName(),
                    connectionStatus.getDestinationName(),
                    input,
                    output,
                    queued));
        }

        lastConnectionStatus.put(connectionStatus.getId(), connectionStatus);
    }
}
 

/**
 * Formats a value like {@code 1048576} to {@code 1 MB} for easier human consumption.
 *
 * @param byteSize the size in bytes
 * @return a String representing the size in the most appropriate unit, with the units
 */
private static String formatSize(int byteSize) {
    return FormatUtils.formatDataSize(byteSize);
}