Java Code Examples for org.apache.nifi.provenance.ProvenanceEventRecord#getEventId()

@Override
public List<ProvenanceEventRecord> getProvenanceEvents(long firstEventId, int maxRecords) throws IOException {
    if (firstEventId < 0 || maxRecords < 1) {
        throw new IllegalArgumentException();
    }

    final List<ProvenanceEventRecord> records = new ArrayList<>();

    for (final ProvenanceEventRecord record : provenanceRecords) {
        if (record.getEventId() >= firstEventId) {
            records.add(record);
            if (records.size() >= maxRecords) {
                return records;
            }
        }
    }

    return records;
}
 

@Override
public long getMaxEventId() throws IOException {
    if (tocReader != null) {
        final long lastBlockOffset = tocReader.getLastBlockOffset();
        skipToBlock(tocReader.getBlockIndex(lastBlockOffset));
    }

    ProvenanceEventRecord record;
    ProvenanceEventRecord lastRecord = null;
    try {
        while ((record = nextRecord()) != null) {
            lastRecord = record;
        }
    } catch (final EOFException eof) {
        // This can happen if we stop NIFi while the record is being written.
        // This is OK, we just ignore this record. The session will not have been
        // committed, so we can just process the FlowFile again.
    }

    return lastRecord == null ? -1L : lastRecord.getEventId();
}
 

@Override
public synchronized StorageSummary writeRecord(final ProvenanceEventRecord record) throws IOException {
    if (isDirty()) {
        throw new IOException("Cannot update Provenance Repository because this Record Writer has already failed to write to the Repository");
    }

    try {
        final long recordIdentifier = record.getEventId() == -1L ? idGenerator.getAndIncrement() : record.getEventId();
        final long startBytes = byteCountingOut.getBytesWritten();

        ensureStreamState(recordIdentifier, startBytes);
        writeRecord(record, recordIdentifier, out);

        recordCount++;
        final long bytesWritten = byteCountingOut.getBytesWritten();
        final long serializedLength = bytesWritten - startBytes;
        final TocWriter tocWriter = getTocWriter();
        final Integer blockIndex = tocWriter == null ? null : tocWriter.getCurrentBlockIndex();
        final String storageLocation = getStorageLocation();
        return new StorageSummary(recordIdentifier, storageLocation, blockIndex, serializedLength, bytesWritten);
    } catch (final IOException ioe) {
        markDirty();
        throw ioe;
    }
}
 

@Override
public Optional<ProvenanceEventRecord> getEvent(final long id) throws IOException {
    final Optional<File> option = getPathForEventId(id);
    if (!option.isPresent()) {
        return Optional.empty();
    }

    try (final RecordReader reader = recordReaderFactory.newRecordReader(option.get(), Collections.emptyList(), config.getMaxAttributeChars())) {
        final Optional<ProvenanceEventRecord> eventOption = reader.skipToEvent(id);
        if (!eventOption.isPresent()) {
            return eventOption;
        }

        // If an event is returned, the event may be the one we want, or it may be an event with a
        // higher event ID, if the desired event is not in the record reader. So we need to get the
        // event and check the Event ID to know whether to return the empty optional or the Optional
        // that was returned.
        final ProvenanceEventRecord event = eventOption.get();
        if (event.getEventId() == id) {
            return eventOption;
        } else {
            return Optional.empty();
        }
    }
}
 

@Override
public List<ProvenanceEventRecord> getProvenanceEvents(long firstEventId, int maxRecords) throws IOException {
    if (firstEventId < 0 || maxRecords < 1) {
        throw new IllegalArgumentException();
    }

    final List<ProvenanceEventRecord> records = new ArrayList<>();

    for (final ProvenanceEventRecord record : provenanceRecords) {
        if (record.getEventId() >= firstEventId) {
            records.add(record);
            if (records.size() >= maxRecords) {
                return records;
            }
        }
    }

    return records;
}
 

@Override
public long getMaxEventId() throws IOException {
    if (tocReader != null) {
        final long lastBlockOffset = tocReader.getLastBlockOffset();
        skipToBlock(tocReader.getBlockIndex(lastBlockOffset));
    }

    ProvenanceEventRecord record;
    ProvenanceEventRecord lastRecord = null;
    try {
        while ((record = nextRecord()) != null) {
            lastRecord = record;
        }
    } catch (final EOFException eof) {
        // This can happen if we stop NIFi while the record is being written.
        // This is OK, we just ignore this record. The session will not have been
        // committed, so we can just process the FlowFile again.
    }

    return lastRecord == null ? -1L : lastRecord.getEventId();
}
 

@Override
public synchronized StorageSummary writeRecord(final ProvenanceEventRecord record) throws IOException {
    if (isDirty()) {
        throw new IOException("Cannot update Provenance Repository because this Record Writer has already failed to write to the Repository");
    }

    try {
        final long recordIdentifier = record.getEventId() == -1L ? idGenerator.getAndIncrement() : record.getEventId();
        final long startBytes = byteCountingOut.getBytesWritten();

        ensureStreamState(recordIdentifier, startBytes);
        writeRecord(record, recordIdentifier, out);

        recordCount++;
        final long bytesWritten = byteCountingOut.getBytesWritten();
        final long serializedLength = bytesWritten - startBytes;
        final TocWriter tocWriter = getTocWriter();
        final Integer blockIndex = tocWriter == null ? null : tocWriter.getCurrentBlockIndex();
        final String storageLocation = getStorageLocation();
        return new StorageSummary(recordIdentifier, storageLocation, blockIndex, serializedLength, bytesWritten);
    } catch (final IOException ioe) {
        markDirty();
        throw ioe;
    }
}
 

@Override
public Optional<ProvenanceEventRecord> getEvent(final long id) throws IOException {
    final Optional<File> option = getPathForEventId(id);
    if (!option.isPresent()) {
        return Optional.empty();
    }

    try (final RecordReader reader = recordReaderFactory.newRecordReader(option.get(), Collections.emptyList(), config.getMaxAttributeChars())) {
        final Optional<ProvenanceEventRecord> eventOption = reader.skipToEvent(id);
        if (!eventOption.isPresent()) {
            return eventOption;
        }

        // If an event is returned, the event may be the one we want, or it may be an event with a
        // higher event ID, if the desired event is not in the record reader. So we need to get the
        // event and check the Event ID to know whether to return the empty optional or the Optional
        // that was returned.
        final ProvenanceEventRecord event = eventOption.get();
        if (event.getEventId() == id) {
            return eventOption;
        } else {
            return Optional.empty();
        }
    }
}
 

private long updateLastEventId(final List<ProvenanceEventRecord> events, final StateManager stateManager) {
    if (events == null || events.isEmpty()) {
        return firstEventId;
    }

    // Store the id of the last event so we know where we left off
    final ProvenanceEventRecord lastEvent = events.get(events.size() - 1);
    final String lastEventId = String.valueOf(lastEvent.getEventId());
    try {
        Map<String, String> newMapOfState = new HashMap<>();
        newMapOfState.put(LAST_EVENT_ID_KEY, lastEventId);
        stateManager.setState(newMapOfState, Scope.LOCAL);
    } catch (final IOException ioe) {
        logger.error("Failed to update state to {} due to {}; this could result in events being re-sent after a restart. The message of {} was: {}",
                new Object[]{lastEventId, ioe, ioe, ioe.getMessage()}, ioe);
    }

    return lastEvent.getEventId() + 1;
}
 

@Override
public boolean moveNext() {
    if (provenanceEvents == null) {
        return false;
    }
    currentRow = null;
    if (currentIndex == provenanceEvents.size()) {
        // Need to fetch a new set of rows, starting with the last ID + 1
        try {
            provenanceEvents = provenanceEventRepository.getEvents(currentId + 1, FETCH_SIZE);
            currentIndex = 0;
        } catch (IOException ioe) {
            logger.error("Error retrieving provenance events, queries will return no further rows");
            return false;
        }
    }

    if (provenanceEvents.isEmpty()) {
        // If we are out of data, close the InputStream. We do this because
        // Calcite does not necessarily call our close() method.
        close();
        try {
            onFinish();
        } catch (final Exception e) {
            logger.error("Failed to perform tasks when enumerator was finished", e);
        }

        return false;
    }

    ProvenanceEventRecord provenanceEvent = provenanceEvents.get(currentIndex);
    currentRow = filterColumns(provenanceEvent);
    currentId = provenanceEvent.getEventId();
    currentIndex++;
    return true;
}
 

static JsonObject serialize(final JsonBuilderFactory factory, final JsonObjectBuilder builder, final ProvenanceEventRecord event, final DateFormat df,
    final String componentName, final String hostname, final URL nifiUrl, final String applicationName, final String platform, final String nodeIdentifier) {
    addField(builder, "eventId", UUID.randomUUID().toString());
    addField(builder, "eventOrdinal", event.getEventId());
    addField(builder, "eventType", event.getEventType().name());
    addField(builder, "timestampMillis", event.getEventTime());
    addField(builder, "timestamp", df.format(event.getEventTime()));
    addField(builder, "durationMillis", event.getEventDuration());
    addField(builder, "lineageStart", event.getLineageStartDate());
    addField(builder, "details", event.getDetails());
    addField(builder, "componentId", event.getComponentId());
    addField(builder, "componentType", event.getComponentType());
    addField(builder, "componentName", componentName);
    addField(builder, "entityId", event.getFlowFileUuid());
    addField(builder, "entityType", "org.apache.nifi.flowfile.FlowFile");
    addField(builder, "entitySize", event.getFileSize());
    addField(builder, "previousEntitySize", event.getPreviousFileSize());
    addField(builder, factory, "updatedAttributes", event.getUpdatedAttributes());
    addField(builder, factory, "previousAttributes", event.getPreviousAttributes());

    addField(builder, "actorHostname", hostname);
    if (nifiUrl != null) {
        // TO get URL Prefix, we just remove the /nifi from the end of the URL. We know that the URL ends with
        // "/nifi" because the Property Validator enforces it
        final String urlString = nifiUrl.toString();
        final String urlPrefix = urlString.substring(0, urlString.length() - DESTINATION_URL_PATH.length());

        final String contentUriBase = urlPrefix + "/nifi-api/provenance-events/" + event.getEventId() + "/content/";
        final String nodeIdSuffix = nodeIdentifier == null ? "" : "?clusterNodeId=" + nodeIdentifier;
        addField(builder, "contentURI", contentUriBase + "output" + nodeIdSuffix);
        addField(builder, "previousContentURI", contentUriBase + "input" + nodeIdSuffix);
    }

    addField(builder, factory, "parentIds", event.getParentUuids());
    addField(builder, factory, "childIds", event.getChildUuids());
    addField(builder, "transitUri", event.getTransitUri());
    addField(builder, "remoteIdentifier", event.getSourceSystemFlowFileIdentifier());
    addField(builder, "alternateIdentifier", event.getAlternateIdentifierUri());
    addField(builder, "platform", platform);
    addField(builder, "application", applicationName);

    return builder.build();
}
 

private JsonObject serialize(final JsonBuilderFactory factory, final JsonObjectBuilder builder,
                final ProvenanceEventRecord event, final DateFormat df, final String componentName,
                final String processGroupId, final String processGroupName, final String hostname,
                final URL nifiUrl, final String applicationName, final String platform,
                final String nodeIdentifier, Boolean allowNullValues) {
        addField(builder, "eventId", UUID.randomUUID().toString(), allowNullValues);
        addField(builder, "eventOrdinal", event.getEventId(), allowNullValues);
        addField(builder, "eventType", event.getEventType().name(), allowNullValues);
        addField(builder, "timestampMillis", event.getEventTime(), allowNullValues);
        addField(builder, "timestamp", df.format(event.getEventTime()), allowNullValues);
        addField(builder, "durationMillis", event.getEventDuration(), allowNullValues);
        addField(builder, "lineageStart", event.getLineageStartDate(), allowNullValues);
        addField(builder, "details", event.getDetails(), allowNullValues);
        addField(builder, "componentId", event.getComponentId(), allowNullValues);
        addField(builder, "componentType", event.getComponentType(), allowNullValues);
        addField(builder, "componentName", componentName, allowNullValues);
        addField(builder, "processGroupId", processGroupId, allowNullValues);
        addField(builder, "processGroupName", processGroupName, allowNullValues);
        addField(builder, "entityId", event.getFlowFileUuid(), allowNullValues);
        addField(builder, "entityType", "org.apache.nifi.flowfile.FlowFile", allowNullValues);
        addField(builder, "entitySize", event.getFileSize(), allowNullValues);
        addField(builder, "previousEntitySize", event.getPreviousFileSize(), allowNullValues);
        addField(builder, factory, "updatedAttributes", event.getUpdatedAttributes(), allowNullValues);
        addField(builder, factory, "previousAttributes", event.getPreviousAttributes(), allowNullValues);

        addField(builder, "actorHostname", hostname, allowNullValues);
        if (nifiUrl != null) {
                // TO get URL Prefix, we just remove the /nifi from the end of the URL. We know
                // that the URL ends with
                // "/nifi" because the Property Validator enforces it
                final String urlString = nifiUrl.toString();
                final String urlPrefix = urlString.substring(0,
                                urlString.length() - DESTINATION_URL_PATH.length());

                final String contentUriBase = urlPrefix + "/nifi-api/provenance-events/" + event.getEventId()
                                + "/content/";
                final String nodeIdSuffix = nodeIdentifier == null ? "" : "?clusterNodeId=" + nodeIdentifier;
                addField(builder, "contentURI", contentUriBase + "output" + nodeIdSuffix, allowNullValues);
                addField(builder, "previousContentURI", contentUriBase + "input" + nodeIdSuffix,
                                allowNullValues);
        }

        addField(builder, factory, "parentIds", event.getParentUuids(), allowNullValues);
        addField(builder, factory, "childIds", event.getChildUuids(), allowNullValues);
        addField(builder, "transitUri", event.getTransitUri(), allowNullValues);
        addField(builder, "remoteIdentifier", event.getSourceSystemFlowFileIdentifier(), allowNullValues);
        addField(builder, "alternateIdentifier", event.getAlternateIdentifierUri(), allowNullValues);
        addField(builder, "platform", platform, allowNullValues);
        addField(builder, "application", applicationName, allowNullValues);
        return builder.build();
}
 

private JsonObject serialize(final JsonBuilderFactory factory, final JsonObjectBuilder builder, final ProvenanceEventRecord event, final DateFormat df,
        final String componentName, final String processGroupId, final String processGroupName, final String hostname, final URL nifiUrl, final String applicationName,
        final String platform, final String nodeIdentifier, Boolean allowNullValues) {
    addField(builder, "eventId", UUID.randomUUID().toString(), allowNullValues);
    addField(builder, "eventOrdinal", event.getEventId(), allowNullValues);
    addField(builder, "eventType", event.getEventType().name(), allowNullValues);
    addField(builder, "timestampMillis", event.getEventTime(), allowNullValues);
    addField(builder, "timestamp", df.format(event.getEventTime()), allowNullValues);
    addField(builder, "durationMillis", event.getEventDuration(), allowNullValues);
    addField(builder, "lineageStart", event.getLineageStartDate(), allowNullValues);
    addField(builder, "details", event.getDetails(), allowNullValues);
    addField(builder, "componentId", event.getComponentId(), allowNullValues);
    addField(builder, "componentType", event.getComponentType(), allowNullValues);
    addField(builder, "componentName", componentName, allowNullValues);
    addField(builder, "processGroupId", processGroupId, allowNullValues);
    addField(builder, "processGroupName", processGroupName, allowNullValues);
    addField(builder, "entityId", event.getFlowFileUuid(), allowNullValues);
    addField(builder, "entityType", "org.apache.nifi.flowfile.FlowFile", allowNullValues);
    addField(builder, "entitySize", event.getFileSize(), allowNullValues);
    addField(builder, "previousEntitySize", event.getPreviousFileSize(), allowNullValues);
    addField(builder, factory, "updatedAttributes", event.getUpdatedAttributes(), allowNullValues);
    addField(builder, factory, "previousAttributes", event.getPreviousAttributes(), allowNullValues);

    addField(builder, "actorHostname", hostname, allowNullValues);
    if (nifiUrl != null) {
        // TO get URL Prefix, we just remove the /nifi from the end of the URL. We know that the URL ends with
        // "/nifi" because the Property Validator enforces it
        final String urlString = nifiUrl.toString();
        final String urlPrefix = urlString.substring(0, urlString.length() - DESTINATION_URL_PATH.length());

        final String contentUriBase = urlPrefix + "/nifi-api/provenance-events/" + event.getEventId() + "/content/";
        final String nodeIdSuffix = nodeIdentifier == null ? "" : "?clusterNodeId=" + nodeIdentifier;
        addField(builder, "contentURI", contentUriBase + "output" + nodeIdSuffix, allowNullValues);
        addField(builder, "previousContentURI", contentUriBase + "input" + nodeIdSuffix, allowNullValues);
    }

    addField(builder, factory, "parentIds", event.getParentUuids(), allowNullValues);
    addField(builder, factory, "childIds", event.getChildUuids(), allowNullValues);
    addField(builder, "transitUri", event.getTransitUri(), allowNullValues);
    addField(builder, "remoteIdentifier", event.getSourceSystemFlowFileIdentifier(), allowNullValues);
    addField(builder, "alternateIdentifier", event.getAlternateIdentifierUri(), allowNullValues);
    addField(builder, "platform", platform, allowNullValues);
    addField(builder, "application", applicationName, allowNullValues);

    return builder.build();
}
 

private Object filterColumns(final ProvenanceEventRecord provenanceEvent) {
    if (provenanceEvent == null) {
        return null;
    }

    final ArrayList<Object> rowList = new ArrayList<>();
    rowList.add(provenanceEvent.getEventId());
    rowList.add(provenanceEvent.getEventType().name());
    rowList.add(provenanceEvent.getEventTime());
    rowList.add(provenanceEvent.getEventDuration());
    rowList.add(provenanceEvent.getLineageStartDate());
    rowList.add(provenanceEvent.getDetails());
    rowList.add(provenanceEvent.getComponentId());
    rowList.add(componentMapHolder.getComponentName(provenanceEvent.getComponentId()));
    rowList.add(provenanceEvent.getComponentType());
    final String processGroupId = componentMapHolder.getProcessGroupId(provenanceEvent.getComponentId(), provenanceEvent.getComponentType());
    rowList.add(processGroupId);
    rowList.add(componentMapHolder.getComponentName(processGroupId));
    rowList.add(provenanceEvent.getFlowFileUuid());
    rowList.add("org.apache.nifi.flowfile.FlowFile"); // entityType
    rowList.add(provenanceEvent.getFileSize());
    rowList.add(provenanceEvent.getPreviousFileSize());
    rowList.add(provenanceEvent.getUpdatedAttributes());
    rowList.add(provenanceEvent.getPreviousAttributes());
    if (nodeIdentifier != null) {
        final String contentPathBase = "/nifi-api/provenance-events/" + provenanceEvent.getEventId() + "/content/";
        final String nodeIdSuffix = "?clusterNodeId=" + nodeIdentifier;
        rowList.add(contentPathBase + "output" + nodeIdSuffix);
        rowList.add(contentPathBase + "input" + nodeIdSuffix);
    } else {
        rowList.add(null);
        rowList.add(null);
    }
    rowList.add(provenanceEvent.getParentUuids());
    rowList.add(provenanceEvent.getChildUuids());
    rowList.add(provenanceEvent.getTransitUri());
    rowList.add(provenanceEvent.getSourceSystemFlowFileIdentifier());
    rowList.add(provenanceEvent.getAlternateIdentifierUri());

    final Object[] row = rowList.toArray();

    // If we want no fields just return null
    if (fields == null) {
        return row;
    }

    // If we want only a single field, then Calcite is going to expect us to return
    // the actual value, NOT a 1-element array of values.
    if (fields.length == 1) {
        final int desiredCellIndex = fields[0];
        return row[desiredCellIndex];
    }

    // Create a new Object array that contains only the desired fields.
    final Object[] filtered = new Object[fields.length];
    for (int i = 0; i < fields.length; i++) {
        final int indexToKeep = fields[i];
        filtered[i] = row[indexToKeep];
    }

    return filtered;
}