Java Code Examples for org.apache.nifi.provenance.ProvenanceEventRecord#getEventTime()

void performMaintenance() {
    try {
        final List<ProvenanceEventRecord> firstEvents = eventStore.getEvents(0, 1);
        if (firstEvents.isEmpty()) {
            return;
        }

        final ProvenanceEventRecord firstEvent = firstEvents.get(0);
        final long earliestEventTime = firstEvent.getEventTime();
        logger.debug("First Event Time is {} ({}) with Event ID {}; will delete any Lucene Index that is older than this",
            earliestEventTime, new Date(earliestEventTime), firstEvent.getEventId());
        final List<File> indicesBeforeEarliestEvent = directoryManager.getDirectoriesBefore(earliestEventTime);

        for (final File index : indicesBeforeEarliestEvent) {
            logger.debug("Index directory {} is now expired. Attempting to remove index", index);
            tryDeleteIndex(index);
        }
    } catch (final Exception e) {
        logger.error("Failed to perform background maintenance procedures", e);
        eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, "Failed to perform maintenance of Provenance Repository. See logs for more information.");
    }
}
 

void performMaintenance() {
    try {
        final List<ProvenanceEventRecord> firstEvents = eventStore.getEvents(0, 1);

        final long earliestEventTime;
        if (firstEvents.isEmpty()) {
            earliestEventTime = System.currentTimeMillis();
            logger.debug("Found no events in the Provenance Repository. In order to perform maintenace of the indices, "
                + "will assume that the first event time is now ({})", System.currentTimeMillis());
        } else {
            final ProvenanceEventRecord firstEvent = firstEvents.get(0);
            earliestEventTime = firstEvent.getEventTime();
            logger.debug("First Event Time is {} ({}) with Event ID {}; will delete any Lucene Index that is older than this",
                earliestEventTime, new Date(earliestEventTime), firstEvent.getEventId());
        }

        final List<File> indicesBeforeEarliestEvent = directoryManager.getDirectoriesBefore(earliestEventTime);

        for (final File index : indicesBeforeEarliestEvent) {
            logger.debug("Index directory {} is now expired. Attempting to remove index", index);
            tryDeleteIndex(index);
        }
    } catch (final Exception e) {
        logger.error("Failed to perform background maintenance procedures", e);
        eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, "Failed to perform maintenance of Provenance Repository. See logs for more information.");
    }
}
 

EventIterator getEventsByTimestamp(final long minTimestmap, final long maxTimestamp) throws IOException {
    // Get a list of all Files and order them based on their ID such that the largest ID is first.
    // This allows us to step through the event files in order and read the first event in the file.
    // If the first event comes after out maxTimestamp, then we know that all other events do as well,
    // so we can ignore that file. Otherwise, we must add it to our list of Files that may contain events
    // within the given time range. If we then reach a file whose first event comes before our minTimestamp,
    // this means that all other files that we later encounter will have a max timestamp that comes before
    // our earliest event time, so we can stop adding files at that point.
    final List<File> eventFiles = getEventFilesFromDisk().sorted(DirectoryUtils.LARGEST_ID_FIRST).collect(Collectors.toList());
    if (eventFiles.isEmpty()) {
        return EventIterator.EMPTY;
    }

    final List<File> relevantEventFiles = new ArrayList<>();
    for (final File eventFile : eventFiles) {
        final ProvenanceEventRecord firstEvent = getFirstEvent(eventFile);
        if (firstEvent == null) {
            return EventIterator.EMPTY;
        }

        final long eventTime = firstEvent.getEventTime();

        if (eventTime > maxTimestamp) {
            continue;
        }

        relevantEventFiles.add(eventFile);

        if (eventTime < minTimestmap) {
            break;
        }
    }

    final EventIterator rawEventIterator = new SequentialRecordReaderEventIterator(relevantEventFiles, recordReaderFactory, 0, Integer.MAX_VALUE);
    return rawEventIterator.filter(event -> event.getEventTime() >= minTimestmap && event.getEventTime() <= maxTimestamp);
}
 

@Override
public void reindexEvents(final Map<ProvenanceEventRecord, StorageSummary> events) {
    final EventIndexTask indexTask = new EventIndexTask(documentQueue, config, indexManager, directoryManager, EventIndexTask.DEFAULT_MAX_EVENTS_PER_COMMIT, eventReporter);

    File lastIndexDir = null;
    long lastEventTime = -2L;

    final List<IndexableDocument> indexableDocs = new ArrayList<>(events.size());
    for (final Map.Entry<ProvenanceEventRecord, StorageSummary> entry : events.entrySet()) {
        final ProvenanceEventRecord event = entry.getKey();
        final StorageSummary summary = entry.getValue();

        for (final CachedQuery cachedQuery : cachedQueries) {
            cachedQuery.update(event, summary);
        }

        final Document document = eventConverter.convert(event, summary);
        if (document == null) {
            logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event);
        } else {
            final File indexDir;
            if (event.getEventTime() == lastEventTime) {
                indexDir = lastIndexDir;
            } else {
                final List<File> files = getDirectoryManager().getDirectories(event.getEventTime(), null);
                indexDir = files.isEmpty() ? null : files.get(0);
                lastIndexDir = indexDir;
            }

            final IndexableDocument doc = new IndexableDocument(document, summary, indexDir);
            indexableDocs.add(doc);
        }
    }

    try {
        indexTask.reIndex(indexableDocs, CommitPreference.PREVENT_COMMIT);
    } catch (final IOException ioe) {
        logger.error("Failed to reindex some Provenance Events", ioe);
        eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, "Failed to re-index some Provenance Events. "
            + "Some Provenance Events may not be available for querying. See logs for more information.");
    }
}
 

@Override
public void reindexEvents(final Map<ProvenanceEventRecord, StorageSummary> events) {
    if (newestIndexDefunct) {
        logger.info("Will avoid re-indexing {} events because the newest index is defunct, so it will be re-indexed in the background", events.size());
        return;
    }

    final EventIndexTask indexTask = new EventIndexTask(documentQueue, indexManager, directoryManager, EventIndexTask.DEFAULT_MAX_EVENTS_PER_COMMIT, eventReporter);

    File lastIndexDir = null;
    long lastEventTime = -2L;

    final List<IndexableDocument> indexableDocs = new ArrayList<>(events.size());
    for (final Map.Entry<ProvenanceEventRecord, StorageSummary> entry : events.entrySet()) {
        final ProvenanceEventRecord event = entry.getKey();
        final StorageSummary summary = entry.getValue();

        for (final CachedQuery cachedQuery : cachedQueries) {
            cachedQuery.update(event, summary);
        }

        final Document document = eventConverter.convert(event, summary);
        if (document == null) {
            logger.debug("Received Provenance Event {} to index but it contained no information that should be indexed, so skipping it", event.getEventId());
        } else {
            final File indexDir;
            if (event.getEventTime() == lastEventTime) {
                indexDir = lastIndexDir;
            } else {
                final List<File> files = getDirectoryManager().getDirectories(event.getEventTime(), null, false);
                if (files.isEmpty()) {
                    final String partitionName = summary.getPartitionName().get();
                    indexDir = getDirectoryManager().getWritableIndexingDirectory(event.getEventTime(), partitionName);
                } else {
                    indexDir = files.get(0);
                }

                lastIndexDir = indexDir;
            }

            final IndexableDocument doc = new IndexableDocument(document, summary, indexDir);
            indexableDocs.add(doc);
        }
    }

    try {
        indexTask.reIndex(indexableDocs, CommitPreference.PREVENT_COMMIT);
    } catch (final IOException ioe) {
        logger.error("Failed to reindex some Provenance Events", ioe);
        eventReporter.reportEvent(Severity.ERROR, EVENT_CATEGORY, "Failed to re-index some Provenance Events. "
            + "Some Provenance Events may not be available for querying. See logs for more information.");
    }
}