Java Code Examples for org.apache.nifi.provenance.ProvenanceEventRecord#getEventType()

/**
 * Checks if the given event is a spurious FORK, meaning that the FORK has a
 * single child and that child was removed in this session. This happens
 * when a Processor calls #create(FlowFile) and then removes the created
 * FlowFile.
 *
 * @param event event
 * @return true if spurious fork
 */
private boolean isSpuriousForkEvent(final ProvenanceEventRecord event, final Set<String> removedFlowFiles) {
    if (event.getEventType() == ProvenanceEventType.FORK) {
        final List<String> childUuids = event.getChildUuids();
        if (childUuids != null && childUuids.size() == 1 && removedFlowFiles.contains(childUuids.get(0))) {
            return true;
        }
    }

    return false;
}
 

/**
 * Checks if the given event is a spurious ROUTE, meaning that the ROUTE
 * indicates that a FlowFile was routed to a relationship with only 1
 * connection and that Connection is the Connection from which the FlowFile
 * was pulled. I.e., the FlowFile was really routed nowhere.
 *
 * @param event event
 * @param records records
 * @return true if spurious route
 */
private boolean isSpuriousRouteEvent(final ProvenanceEventRecord event, final Map<FlowFileRecord, StandardRepositoryRecord> records) {
    if (event.getEventType() == ProvenanceEventType.ROUTE) {
        final String relationshipName = event.getRelationship();
        final Relationship relationship = new Relationship.Builder().name(relationshipName).build();
        final Collection<Connection> connectionsForRelationship = this.context.getConnections(relationship);

        // If the number of connections for this relationship is not 1, then we can't ignore this ROUTE event,
        // as it may be cloning the FlowFile and adding to multiple connections.
        if (connectionsForRelationship.size() == 1) {
            for (final Map.Entry<FlowFileRecord, StandardRepositoryRecord> entry : records.entrySet()) {
                final FlowFileRecord flowFileRecord = entry.getKey();
                if (event.getFlowFileUuid().equals(flowFileRecord.getAttribute(CoreAttributes.UUID.key()))) {
                    final StandardRepositoryRecord repoRecord = entry.getValue();
                    if (repoRecord.getOriginalQueue() == null) {
                        return false;
                    }

                    final String originalQueueId = repoRecord.getOriginalQueue().getIdentifier();
                    final Connection destinationConnection = connectionsForRelationship.iterator().next();
                    final String destinationQueueId = destinationConnection.getFlowFileQueue().getIdentifier();
                    return originalQueueId.equals(destinationQueueId);
                }
            }
        }
    }

    return false;
}
 

private void removeForkEvents(final FlowFile flowFile) {
    for (final ProvenanceEventBuilder builder : forkEventBuilders.values()) {
        final ProvenanceEventRecord event = builder.build();

        if (event.getEventType() == ProvenanceEventType.FORK) {
            builder.removeChildFlowFile(flowFile);
        }
    }
}
 

private void removeForkEvents(final FlowFile flowFile) {
    for (final ProvenanceEventBuilder builder : forkEventBuilders.values()) {
        final ProvenanceEventRecord event = builder.build();

        if (event.getEventType() == ProvenanceEventType.FORK) {
            builder.removeChildFlowFile(flowFile);
        }
    }
}
 

private EdgeNode createEdge(ProvenanceRecords prs, int srcIdx, int tgtIdx) {
    // Generate C created a FlowFile
    final ProvenanceEventRecord srcR = prs.get(srcIdx);
    // Then Remote Input Port sent it
    final ProvenanceEventRecord tgtR = prs.get(tgtIdx);
    final EventNode src = new EventNode(srcR);
    final EventNode tgt = new EventNode(tgtR);
    final EdgeNode edge = new EdgeNode(srcR.getComponentType() + " to " + tgtR.getEventType(), src, tgt);
    return edge;
}
 

/**
 * Checks if the given event is a spurious ROUTE, meaning that the ROUTE
 * indicates that a FlowFile was routed to a relationship with only 1
 * connection and that Connection is the Connection from which the FlowFile
 * was pulled. I.e., the FlowFile was really routed nowhere.
 *
 * @param event event
 * @param records records
 * @return true if spurious route
 */
private boolean isSpuriousRouteEvent(final ProvenanceEventRecord event, final Map<Long, StandardRepositoryRecord> records) {
    if (event.getEventType() == ProvenanceEventType.ROUTE) {
        final String relationshipName = event.getRelationship();
        final Relationship relationship = new Relationship.Builder().name(relationshipName).build();
        final Collection<Connection> connectionsForRelationship = this.context.getConnections(relationship);

        // If the number of connections for this relationship is not 1, then we can't ignore this ROUTE event,
        // as it may be cloning the FlowFile and adding to multiple connections.
        if (connectionsForRelationship.size() == 1) {
            for (final StandardRepositoryRecord repoRecord : records.values()) {
                final FlowFileRecord flowFileRecord = repoRecord.getCurrent();
                if (event.getFlowFileUuid().equals(flowFileRecord.getAttribute(CoreAttributes.UUID.key()))) {
                    if (repoRecord.getOriginalQueue() == null) {
                        return false;
                    }

                    final String originalQueueId = repoRecord.getOriginalQueue().getIdentifier();
                    final Connection destinationConnection = connectionsForRelationship.iterator().next();
                    final String destinationQueueId = destinationConnection.getFlowFileQueue().getIdentifier();
                    return originalQueueId.equals(destinationQueueId);
                }
            }
        }
    }

    return false;
}
 

/**
 * Checks if the given event is a spurious FORK, meaning that the FORK has a
 * single child and that child was removed in this session. This happens
 * when a Processor calls #create(FlowFile) and then removes the created
 * FlowFile.
 *
 * @param event event
 * @return true if spurious fork
 */
private boolean isSpuriousForkEvent(final ProvenanceEventRecord event, final Set<String> removedFlowFiles) {
    if (event.getEventType() == ProvenanceEventType.FORK) {
        final List<String> childUuids = event.getChildUuids();
        if (childUuids != null && childUuids.size() == 1 && removedFlowFiles.contains(childUuids.get(0))) {
            return true;
        }
    }

    return false;
}
 

@Test
public void test200() throws Exception {
    addHandler(new GetOrHeadHandler());

    runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");

    createFlowFiles(runner);

    // Verify only one FlowFile gets created/sent
    runner.run();
    runner.run();

    runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
    runner.assertPenalizeCount(0);

    // expected in request status.code and status.message
    // original flow file (+attributes)
    final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
    bundle.assertContentEquals("Hello".getBytes("UTF-8"));
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle.assertAttributeEquals("Foo", "Bar");

    // expected in response
    // status code, status message, all headers from server response --> ff attributes
    // server response message body into payload of ff
    final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
    bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle1.assertAttributeEquals("Foo", "Bar");
    bundle1.assertAttributeEquals("Content-Type", "text/plain;charset=iso-8859-1");

    final List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    assertEquals(2, provEvents.size());
    boolean forkEvent = false;
    boolean fetchEvent = false;
    for (final ProvenanceEventRecord event : provEvents) {
        if (event.getEventType() == ProvenanceEventType.FORK) {
            forkEvent = true;
        } else if (event.getEventType() == ProvenanceEventType.FETCH) {
            fetchEvent = true;
        }
    }

    assertTrue(forkEvent);
    assertTrue(fetchEvent);
}
 

@Test
public void test200() throws Exception {
    addHandler(new GetOrHeadHandler());
    setupEndpointAndRegion();

    runner.setProperty(InvokeAWSGatewayApi.PROP_RESOURCE_NAME, "/status/200");
    createFlowFiles(runner);

    // verify that call works with or without flowfile being sent for GET
    // there should only be 1 REQ, but 2 RESPONSE
    runner.run();
    runner.run();

    runner.assertTransferCount(InvokeAWSGatewayApi.REL_SUCCESS_REQ_NAME, 1);
    runner.assertTransferCount(InvokeAWSGatewayApi.REL_RESPONSE_NAME, 2);
    runner.assertTransferCount(InvokeAWSGatewayApi.REL_RETRY_NAME, 0);
    runner.assertTransferCount(InvokeAWSGatewayApi.REL_NO_RETRY_NAME, 0);
    runner.assertTransferCount(InvokeAWSGatewayApi.REL_FAILURE_NAME, 0);
    runner.assertPenalizeCount(0);

    // expected in request status.code and status.message
    // original flow file (+attributes)
    final MockFlowFile bundle = runner
        .getFlowFilesForRelationship(InvokeAWSGatewayApi.REL_SUCCESS_REQ_NAME).get(0);
    bundle.assertContentEquals("Hello".getBytes("UTF-8"));
    bundle.assertAttributeEquals(InvokeAWSGatewayApi.STATUS_CODE, "200");
    bundle.assertAttributeEquals(InvokeAWSGatewayApi.STATUS_MESSAGE, "OK");
    bundle.assertAttributeEquals("Foo", "Bar");

    // expected in response of each message
    // status code, status message, all headers from server response --> ff attributes
    // server response message body into payload of ff
    final MockFlowFile bundle1 = runner
        .getFlowFilesForRelationship(InvokeAWSGatewayApi.REL_RESPONSE_NAME).get(0);
    bundle1.assertContentEquals("{\"status\":\"200\"}".getBytes("UTF-8"));
    bundle1.assertAttributeEquals(InvokeAWSGatewayApi.STATUS_CODE, "200");
    bundle1.assertAttributeEquals(InvokeAWSGatewayApi.STATUS_MESSAGE, "OK");
    bundle1.assertAttributeEquals("Foo", "Bar");
    bundle1.assertAttributeEquals("Content-Type", "application/json");

    final MockFlowFile bundle2 = runner
        .getFlowFilesForRelationship(InvokeAWSGatewayApi.REL_RESPONSE_NAME).get(1);
    bundle1.assertContentEquals("{\"status\":\"200\"}".getBytes("UTF-8"));
    bundle1.assertAttributeEquals(InvokeAWSGatewayApi.STATUS_CODE, "200");
    bundle1.assertAttributeEquals(InvokeAWSGatewayApi.STATUS_MESSAGE, "OK");
    bundle1.assertAttributeEquals("Foo", "Bar");
    bundle1.assertAttributeEquals("Content-Type", "application/json");

    final List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    assertEquals(3, provEvents.size());
    boolean forkEvent = false;
    boolean fetchEvent = false;
    boolean recieveEvent = false;
    for (final ProvenanceEventRecord event : provEvents) {
        if (event.getEventType() == ProvenanceEventType.FORK) {
            forkEvent = true;
        } else if (event.getEventType() == ProvenanceEventType.FETCH) {
            fetchEvent = true;
        } else if (event.getEventType() == ProvenanceEventType.RECEIVE) {
            recieveEvent = true;
        }
    }

    assertTrue(forkEvent);
    assertTrue(fetchEvent);
    assertTrue(recieveEvent);
}
 

@Test
public void testUpdateAttributesThenJoin() throws IOException {
    final FlowFileRecord flowFileRecord1 = new StandardFlowFileRecord.Builder()
            .id(1L)
            .addAttribute("uuid", "11111111-1111-1111-1111-111111111111")
            .entryDate(System.currentTimeMillis())
            .build();

    final FlowFileRecord flowFileRecord2 = new StandardFlowFileRecord.Builder()
            .id(2L)
            .addAttribute("uuid", "22222222-2222-2222-2222-222222222222")
            .entryDate(System.currentTimeMillis())
            .build();

    flowFileQueue.put(flowFileRecord1);
    flowFileQueue.put(flowFileRecord2);

    FlowFile ff1 = session.get();
    FlowFile ff2 = session.get();

    ff1 = session.putAttribute(ff1, "index", "1");
    ff2 = session.putAttribute(ff2, "index", "2");

    final List<FlowFile> parents = new ArrayList<>(2);
    parents.add(ff1);
    parents.add(ff2);

    final FlowFile child = session.create(parents);

    final Relationship rel = new Relationship.Builder().name("A").build();

    session.transfer(ff1, rel);
    session.transfer(ff2, rel);
    session.transfer(child, rel);

    session.commit();

    final List<ProvenanceEventRecord> events = provenanceRepo.getEvents(0L, 1000);

    // We should have a JOIN and 2 ATTRIBUTE_MODIFIED's
    assertEquals(3, events.size());

    int joinCount = 0;
    int ff1UpdateCount = 0;
    int ff2UpdateCount = 0;

    for (final ProvenanceEventRecord event : events) {
        switch (event.getEventType()) {
            case JOIN:
                assertEquals(child.getAttribute("uuid"), event.getFlowFileUuid());
                joinCount++;
                break;
            case ATTRIBUTES_MODIFIED:
                if (event.getFlowFileUuid().equals(ff1.getAttribute("uuid"))) {
                    ff1UpdateCount++;
                } else if (event.getFlowFileUuid().equals(ff2.getAttribute("uuid"))) {
                    ff2UpdateCount++;
                } else {
                    Assert.fail("Got ATTRIBUTE_MODIFIED for wrong FlowFile: " + event.getFlowFileUuid());
                }
                break;
            default:
                Assert.fail("Unexpected event type: " + event);
        }
    }

    assertEquals(1, joinCount);
    assertEquals(1, ff1UpdateCount);
    assertEquals(1, ff2UpdateCount);

    assertEquals(1, joinCount);
}
 

private String getReplayFailureReason(final ProvenanceEventRecord event) {
    // Check that the event is a valid type.
    final ProvenanceEventType type = event.getEventType();
    if (type == ProvenanceEventType.JOIN) {
        return "Cannot replay events that are created from multiple parents";
    }

    // Make sure event has the Content Claim info
    final Long contentSize = event.getPreviousFileSize();
    final String contentClaimId = event.getPreviousContentClaimIdentifier();
    final String contentClaimSection = event.getPreviousContentClaimSection();
    final String contentClaimContainer = event.getPreviousContentClaimContainer();

    if (contentSize == null || contentClaimId == null || contentClaimSection == null || contentClaimContainer == null) {
        return "Cannot replay data from Provenance Event because the event does not contain the required Content Claim";
    }

    try {
        final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(contentClaimContainer, contentClaimSection, contentClaimId, false, false);
        final ContentClaim contentClaim = new StandardContentClaim(resourceClaim, event.getPreviousContentClaimOffset());

        if (!contentRepository.isAccessible(contentClaim)) {
            return "Content is no longer available in Content Repository";
        }
    } catch (final IOException ioe) {
        return "Failed to determine whether or not content was available in Content Repository due to " + ioe.toString();
    }

    // Make sure that the source queue exists
    if (event.getSourceQueueIdentifier() == null) {
        return "Cannot replay data from Provenance Event because the event does not specify the Source FlowFile Queue";
    }

    final Set<Connection> connections = flowManager.findAllConnections();
    FlowFileQueue queue = null;
    for (final Connection connection : connections) {
        if (event.getSourceQueueIdentifier().equals(connection.getIdentifier())) {
            queue = connection.getFlowFileQueue();
            break;
        }
    }

    if (queue == null) {
        return "Cannot replay data from Provenance Event because the Source FlowFile Queue with ID " + event.getSourceQueueIdentifier() + " no longer exists";
    }

    return null;
}
 

private void updateEventRepository(final Checkpoint checkpoint) {
    int flowFilesReceived = 0;
    int flowFilesSent = 0;
    long bytesReceived = 0L;
    long bytesSent = 0L;

    for (final ProvenanceEventRecord event : checkpoint.reportedEvents) {
        if (isSpuriousForkEvent(event, checkpoint.removedFlowFiles)) {
            continue;
        }

        switch (event.getEventType()) {
            case SEND:
                flowFilesSent++;
                bytesSent += event.getFileSize();
                break;
            case RECEIVE:
            case FETCH:
                flowFilesReceived++;
                bytesReceived += event.getFileSize();
                break;
            default:
                break;
        }
    }

    try {
        // update event repository
        final Connectable connectable = context.getConnectable();
        final StandardFlowFileEvent flowFileEvent = new StandardFlowFileEvent();
        flowFileEvent.setBytesRead(checkpoint.bytesRead);
        flowFileEvent.setBytesWritten(checkpoint.bytesWritten);
        flowFileEvent.setContentSizeIn(checkpoint.contentSizeIn);
        flowFileEvent.setContentSizeOut(checkpoint.contentSizeOut);
        flowFileEvent.setContentSizeRemoved(checkpoint.removedBytes);
        flowFileEvent.setFlowFilesIn(checkpoint.flowFilesIn);
        flowFileEvent.setFlowFilesOut(checkpoint.flowFilesOut);
        flowFileEvent.setFlowFilesRemoved(checkpoint.removedCount);
        flowFileEvent.setFlowFilesReceived(flowFilesReceived);
        flowFileEvent.setBytesReceived(bytesReceived);
        flowFileEvent.setFlowFilesSent(flowFilesSent);
        flowFileEvent.setBytesSent(bytesSent);

        final long now = System.currentTimeMillis();
        long lineageMillis = 0L;
        for (final StandardRepositoryRecord record : checkpoint.records.values()) {
            final FlowFile flowFile = record.getCurrent();
            final long lineageDuration = now - flowFile.getLineageStartDate();
            lineageMillis += lineageDuration;
        }
        flowFileEvent.setAggregateLineageMillis(lineageMillis);

        final Map<String, Long> counters = combineCounters(checkpoint.countersOnCommit, checkpoint.immediateCounters);
        flowFileEvent.setCounters(counters);

        context.getFlowFileEventRepository().updateRepository(flowFileEvent, connectable.getIdentifier());

        for (final Map.Entry<String, StandardFlowFileEvent> entry : checkpoint.connectionCounts.entrySet()) {
            context.getFlowFileEventRepository().updateRepository(entry.getValue(), entry.getKey());
        }
    } catch (final IOException ioe) {
        LOG.error("FlowFile Event Repository failed to update", ioe);
    }
}
 

@Test
public void test200Auth() throws Exception {
    addHandler(new BasicAuthHandler());

    final String username = "basic_user";
    final String password = "basic_password";

    runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
    runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_USERNAME, username);
    runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_PASSWORD, password);
    final byte[] creds = String.format("%s:%s", username, password).getBytes(StandardCharsets.UTF_8);
    final String expAuth = String.format("Basic %s", new String(encodeBase64(creds)));

    createFlowFiles(runner);

    runner.run();

    runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
    runner.assertPenalizeCount(0);

    // expected in request status.code and status.message
    // original flow file (+attributes)
    final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
    bundle.assertContentEquals("Hello".getBytes("UTF-8"));
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle.assertAttributeEquals("Foo", "Bar");

    // expected in response
    // status code, status message, all headers from server response --> ff attributes
    // server response message body into payload of ff
    final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
    final String bundle1Content = new String(bundle1.toByteArray(), StandardCharsets.UTF_8);
    assertTrue(bundle1Content.startsWith(expAuth)); // use startsWith instead of equals so we can ignore line endings
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle1.assertAttributeEquals("Foo", "Bar");
    bundle1.assertAttributeEquals("Content-Type", "text/plain;charset=iso-8859-1");

    final List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    assertEquals(2, provEvents.size());
    boolean forkEvent = false;
    boolean fetchEvent = false;
    for (final ProvenanceEventRecord event : provEvents) {
        if (event.getEventType() == ProvenanceEventType.FORK) {
            forkEvent = true;
        } else if (event.getEventType() == ProvenanceEventType.FETCH) {
            fetchEvent = true;
        }
    }

    assertTrue(forkEvent);
    assertTrue(fetchEvent);
}
 

@Test
public void test200() throws Exception {
    addHandler(new GetOrHeadHandler());

    runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");

    createFlowFiles(runner);

    // Verify only one FlowFile gets created/sent
    runner.run();
    runner.run();

    runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
    runner.assertPenalizeCount(0);

    // expected in request status.code and status.message
    // original flow file (+attributes)
    final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
    bundle.assertContentEquals("Hello".getBytes("UTF-8"));
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle.assertAttributeEquals("Foo", "Bar");

    // expected in response
    // status code, status message, all headers from server response --> ff attributes
    // server response message body into payload of ff
    final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
    bundle1.assertContentEquals("/status/200".getBytes("UTF-8"));
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle1.assertAttributeEquals("Foo", "Bar");
    bundle1.assertAttributeEquals("Content-Type", "text/plain;charset=iso-8859-1");

    final List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    assertEquals(2, provEvents.size());
    boolean forkEvent = false;
    boolean fetchEvent = false;
    for (final ProvenanceEventRecord event : provEvents) {
        if (event.getEventType() == ProvenanceEventType.FORK) {
            forkEvent = true;
        } else if (event.getEventType() == ProvenanceEventType.FETCH) {
            fetchEvent = true;
        }
    }

    assertTrue(forkEvent);
    assertTrue(fetchEvent);
}
 

@Test
public void testUpdateAttributesThenJoin() throws IOException {
    final FlowFileRecord flowFileRecord1 = new StandardFlowFileRecord.Builder()
            .id(1L)
            .addAttribute("uuid", "11111111-1111-1111-1111-111111111111")
            .entryDate(System.currentTimeMillis())
            .build();

    final FlowFileRecord flowFileRecord2 = new StandardFlowFileRecord.Builder()
            .id(2L)
            .addAttribute("uuid", "22222222-2222-2222-2222-222222222222")
            .entryDate(System.currentTimeMillis())
            .build();

    flowFileQueue.put(flowFileRecord1);
    flowFileQueue.put(flowFileRecord2);

    FlowFile ff1 = session.get();
    FlowFile ff2 = session.get();

    ff1 = session.putAttribute(ff1, "index", "1");
    ff2 = session.putAttribute(ff2, "index", "2");

    final List<FlowFile> parents = new ArrayList<>(2);
    parents.add(ff1);
    parents.add(ff2);

    final FlowFile child = session.create(parents);

    final Relationship rel = new Relationship.Builder().name("A").build();

    session.transfer(ff1, rel);
    session.transfer(ff2, rel);
    session.transfer(child, rel);

    session.commit();

    final List<ProvenanceEventRecord> events = provenanceRepo.getEvents(0L, 1000);

    // We should have a JOIN and 2 ATTRIBUTE_MODIFIED's
    assertEquals(3, events.size());

    int joinCount = 0;
    int ff1UpdateCount = 0;
    int ff2UpdateCount = 0;

    for (final ProvenanceEventRecord event : events) {
        switch (event.getEventType()) {
            case JOIN:
                assertEquals(child.getAttribute("uuid"), event.getFlowFileUuid());
                joinCount++;
                break;
            case ATTRIBUTES_MODIFIED:
                if (event.getFlowFileUuid().equals(ff1.getAttribute("uuid"))) {
                    ff1UpdateCount++;
                } else if (event.getFlowFileUuid().equals(ff2.getAttribute("uuid"))) {
                    ff2UpdateCount++;
                } else {
                    Assert.fail("Got ATTRIBUTE_MODIFIED for wrong FlowFile: " + event.getFlowFileUuid());
                }
                break;
            default:
                Assert.fail("Unexpected event type: " + event);
        }
    }

    assertEquals(1, joinCount);
    assertEquals(1, ff1UpdateCount);
    assertEquals(1, ff2UpdateCount);

    assertEquals(1, joinCount);
}
 

private String getReplayFailureReason(final ProvenanceEventRecord event) {
    // Check that the event is a valid type.
    final ProvenanceEventType type = event.getEventType();
    if (type == ProvenanceEventType.JOIN) {
        return "Cannot replay events that are created from multiple parents";
    }

    // Make sure event has the Content Claim info
    final Long contentSize = event.getPreviousFileSize();
    final String contentClaimId = event.getPreviousContentClaimIdentifier();
    final String contentClaimSection = event.getPreviousContentClaimSection();
    final String contentClaimContainer = event.getPreviousContentClaimContainer();

    if (contentSize == null || contentClaimId == null || contentClaimSection == null || contentClaimContainer == null) {
        return "Cannot replay data from Provenance Event because the event does not contain the required Content Claim";
    }

    try {
        final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(contentClaimContainer, contentClaimSection, contentClaimId, false, false);
        final ContentClaim contentClaim = new StandardContentClaim(resourceClaim, event.getPreviousContentClaimOffset());

        if (!contentRepository.isAccessible(contentClaim)) {
            return "Content is no longer available in Content Repository";
        }
    } catch (final IOException ioe) {
        return "Failed to determine whether or not content was available in Content Repository due to " + ioe.toString();
    }

    // Make sure that the source queue exists
    if (event.getSourceQueueIdentifier() == null) {
        return "Cannot replay data from Provenance Event because the event does not specify the Source FlowFile Queue";
    }

    final List<Connection> connections = getGroup(getRootGroupId()).findAllConnections();
    FlowFileQueue queue = null;
    for (final Connection connection : connections) {
        if (event.getSourceQueueIdentifier().equals(connection.getIdentifier())) {
            queue = connection.getFlowFileQueue();
            break;
        }
    }

    if (queue == null) {
        return "Cannot replay data from Provenance Event because the Source FlowFile Queue with ID " + event.getSourceQueueIdentifier() + " no longer exists";
    }

    return null;
}
 

private void updateEventRepository(final Checkpoint checkpoint) {
    int flowFilesReceived = 0;
    int flowFilesSent = 0;
    long bytesReceived = 0L;
    long bytesSent = 0L;

    for (final ProvenanceEventRecord event : checkpoint.reportedEvents) {
        if (isSpuriousForkEvent(event, checkpoint.removedFlowFiles)) {
            continue;
        }

        switch (event.getEventType()) {
            case SEND:
                flowFilesSent++;
                bytesSent += event.getFileSize();
                break;
            case RECEIVE:
            case FETCH:
                flowFilesReceived++;
                bytesReceived += event.getFileSize();
                break;
            default:
                break;
        }
    }

    try {
        // update event repository
        final Connectable connectable = context.getConnectable();
        final StandardFlowFileEvent flowFileEvent = new StandardFlowFileEvent(connectable.getIdentifier());
        flowFileEvent.setBytesRead(checkpoint.bytesRead);
        flowFileEvent.setBytesWritten(checkpoint.bytesWritten);
        flowFileEvent.setContentSizeIn(checkpoint.contentSizeIn);
        flowFileEvent.setContentSizeOut(checkpoint.contentSizeOut);
        flowFileEvent.setContentSizeRemoved(checkpoint.removedBytes);
        flowFileEvent.setFlowFilesIn(checkpoint.flowFilesIn);
        flowFileEvent.setFlowFilesOut(checkpoint.flowFilesOut);
        flowFileEvent.setFlowFilesRemoved(checkpoint.removedCount);
        flowFileEvent.setFlowFilesReceived(flowFilesReceived);
        flowFileEvent.setBytesReceived(bytesReceived);
        flowFileEvent.setFlowFilesSent(flowFilesSent);
        flowFileEvent.setBytesSent(bytesSent);

        long lineageMillis = 0L;
        for (final Map.Entry<FlowFileRecord, StandardRepositoryRecord> entry : checkpoint.records.entrySet()) {
            final FlowFile flowFile = entry.getKey();
            final long lineageDuration = System.currentTimeMillis() - flowFile.getLineageStartDate();
            lineageMillis += lineageDuration;
        }
        flowFileEvent.setAggregateLineageMillis(lineageMillis);

        context.getFlowFileEventRepository().updateRepository(flowFileEvent);

        for (final FlowFileEvent connectionEvent : checkpoint.connectionCounts.values()) {
            context.getFlowFileEventRepository().updateRepository(connectionEvent);
        }
    } catch (final IOException ioe) {
        LOG.error("FlowFile Event Repository failed to update", ioe);
    }
}
 

@Test
public void test200Auth() throws Exception {
    addHandler(new BasicAuthHandler());

    final String username = "basic_user";
    final String password = "basic_password";

    runner.setProperty(InvokeHTTP.PROP_URL, url + "/status/200");
    runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_USERNAME, username);
    runner.setProperty(InvokeHTTP.PROP_BASIC_AUTH_PASSWORD, password);
    final byte[] creds = String.format("%s:%s", username, password).getBytes(StandardCharsets.UTF_8);
    final String expAuth = String.format("Basic %s", new String(encodeBase64(creds)));

    createFlowFiles(runner);

    runner.run();

    runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
    runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
    runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
    runner.assertPenalizeCount(0);

    // expected in request status.code and status.message
    // original flow file (+attributes)
    final MockFlowFile bundle = runner.getFlowFilesForRelationship(InvokeHTTP.REL_SUCCESS_REQ).get(0);
    bundle.assertContentEquals("Hello".getBytes("UTF-8"));
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle.assertAttributeEquals("Foo", "Bar");

    // expected in response
    // status code, status message, all headers from server response --> ff attributes
    // server response message body into payload of ff
    final MockFlowFile bundle1 = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
    final String bundle1Content = new String(bundle1.toByteArray(), StandardCharsets.UTF_8);
    assertTrue(bundle1Content.startsWith(expAuth)); // use startsWith instead of equals so we can ignore line endings
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
    bundle1.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
    bundle1.assertAttributeEquals("Foo", "Bar");
    bundle1.assertAttributeEquals("Content-Type", "text/plain;charset=iso-8859-1");

    final List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents();
    assertEquals(2, provEvents.size());
    boolean forkEvent = false;
    boolean fetchEvent = false;
    for (final ProvenanceEventRecord event : provEvents) {
        if (event.getEventType() == ProvenanceEventType.FORK) {
            forkEvent = true;
        } else if (event.getEventType() == ProvenanceEventType.FETCH) {
            fetchEvent = true;
        }
    }

    assertTrue(forkEvent);
    assertTrue(fetchEvent);
}