Java Code Examples for org.apache.nifi.provenance.ProvenanceEventRecord#getFileSize()

The following examples show how to use org.apache.nifi.provenance.ProvenanceEventRecord#getFileSize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StandardProcessSession.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up 
private void updateEventRepository(final Checkpoint checkpoint) {
    int flowFilesReceived = 0;
    int flowFilesSent = 0;
    long bytesReceived = 0L;
    long bytesSent = 0L;

    for (final ProvenanceEventRecord event : checkpoint.reportedEvents) {
        if (isSpuriousForkEvent(event, checkpoint.removedFlowFiles)) {
            continue;
        }

        switch (event.getEventType()) {
            case SEND:
                flowFilesSent++;
                bytesSent += event.getFileSize();
                break;
            case RECEIVE:
            case FETCH:
                flowFilesReceived++;
                bytesReceived += event.getFileSize();
                break;
            default:
                break;
        }
    }

    try {
        // update event repository
        final Connectable connectable = context.getConnectable();
        final StandardFlowFileEvent flowFileEvent = new StandardFlowFileEvent(connectable.getIdentifier());
        flowFileEvent.setBytesRead(checkpoint.bytesRead);
        flowFileEvent.setBytesWritten(checkpoint.bytesWritten);
        flowFileEvent.setContentSizeIn(checkpoint.contentSizeIn);
        flowFileEvent.setContentSizeOut(checkpoint.contentSizeOut);
        flowFileEvent.setContentSizeRemoved(checkpoint.removedBytes);
        flowFileEvent.setFlowFilesIn(checkpoint.flowFilesIn);
        flowFileEvent.setFlowFilesOut(checkpoint.flowFilesOut);
        flowFileEvent.setFlowFilesRemoved(checkpoint.removedCount);
        flowFileEvent.setFlowFilesReceived(flowFilesReceived);
        flowFileEvent.setBytesReceived(bytesReceived);
        flowFileEvent.setFlowFilesSent(flowFilesSent);
        flowFileEvent.setBytesSent(bytesSent);

        long lineageMillis = 0L;
        for (final Map.Entry<FlowFileRecord, StandardRepositoryRecord> entry : checkpoint.records.entrySet()) {
            final FlowFile flowFile = entry.getKey();
            final long lineageDuration = System.currentTimeMillis() - flowFile.getLineageStartDate();
            lineageMillis += lineageDuration;
        }
        flowFileEvent.setAggregateLineageMillis(lineageMillis);

        context.getFlowFileEventRepository().updateRepository(flowFileEvent);

        for (final FlowFileEvent connectionEvent : checkpoint.connectionCounts.values()) {
            context.getFlowFileEventRepository().updateRepository(connectionEvent);
        }
    } catch (final IOException ioe) {
        LOG.error("FlowFile Event Repository failed to update", ioe);
    }
}
Example 2
Source File: StandardProcessSession.java    From nifi with Apache License 2.0 4 votes vote down vote up 
private void updateEventRepository(final Checkpoint checkpoint) {
    int flowFilesReceived = 0;
    int flowFilesSent = 0;
    long bytesReceived = 0L;
    long bytesSent = 0L;

    for (final ProvenanceEventRecord event : checkpoint.reportedEvents) {
        if (isSpuriousForkEvent(event, checkpoint.removedFlowFiles)) {
            continue;
        }

        switch (event.getEventType()) {
            case SEND:
                flowFilesSent++;
                bytesSent += event.getFileSize();
                break;
            case RECEIVE:
            case FETCH:
                flowFilesReceived++;
                bytesReceived += event.getFileSize();
                break;
            default:
                break;
        }
    }

    try {
        // update event repository
        final Connectable connectable = context.getConnectable();
        final StandardFlowFileEvent flowFileEvent = new StandardFlowFileEvent();
        flowFileEvent.setBytesRead(checkpoint.bytesRead);
        flowFileEvent.setBytesWritten(checkpoint.bytesWritten);
        flowFileEvent.setContentSizeIn(checkpoint.contentSizeIn);
        flowFileEvent.setContentSizeOut(checkpoint.contentSizeOut);
        flowFileEvent.setContentSizeRemoved(checkpoint.removedBytes);
        flowFileEvent.setFlowFilesIn(checkpoint.flowFilesIn);
        flowFileEvent.setFlowFilesOut(checkpoint.flowFilesOut);
        flowFileEvent.setFlowFilesRemoved(checkpoint.removedCount);
        flowFileEvent.setFlowFilesReceived(flowFilesReceived);
        flowFileEvent.setBytesReceived(bytesReceived);
        flowFileEvent.setFlowFilesSent(flowFilesSent);
        flowFileEvent.setBytesSent(bytesSent);

        final long now = System.currentTimeMillis();
        long lineageMillis = 0L;
        for (final StandardRepositoryRecord record : checkpoint.records.values()) {
            final FlowFile flowFile = record.getCurrent();
            final long lineageDuration = now - flowFile.getLineageStartDate();
            lineageMillis += lineageDuration;
        }
        flowFileEvent.setAggregateLineageMillis(lineageMillis);

        final Map<String, Long> counters = combineCounters(checkpoint.countersOnCommit, checkpoint.immediateCounters);
        flowFileEvent.setCounters(counters);

        context.getFlowFileEventRepository().updateRepository(flowFileEvent, connectable.getIdentifier());

        for (final Map.Entry<String, StandardFlowFileEvent> entry : checkpoint.connectionCounts.entrySet()) {
            context.getFlowFileEventRepository().updateRepository(entry.getValue(), entry.getKey());
        }
    } catch (final IOException ioe) {
        LOG.error("FlowFile Event Repository failed to update", ioe);
    }
}