Java Code Examples for org.apache.nifi.provenance.serialization.StorageSummary#getEventId()

The following examples show how to use org.apache.nifi.provenance.serialization.StorageSummary#getEventId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestPartitionedWriteAheadEventStore.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSingleWriteThenRead() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    assertEquals(-1, store.getMaxEventId());
    final ProvenanceEventRecord event1 = createEvent();
    final StorageResult result = store.addEvents(Collections.singleton(event1));

    final StorageSummary summary = result.getStorageLocations().values().iterator().next();
    final long eventId = summary.getEventId();
    final ProvenanceEventRecord eventWithId = addId(event1, eventId);

    assertEquals(0, store.getMaxEventId());

    final ProvenanceEventRecord read = store.getEvent(eventId).get();
    assertEquals(eventWithId, read);
}
Example 2
Source File: TestPartitionedWriteAheadEventStore.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSingleWriteThenRead() throws IOException {
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(createConfig(), writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();

    assertEquals(-1, store.getMaxEventId());
    final ProvenanceEventRecord event1 = createEvent();
    final StorageResult result = store.addEvents(Collections.singleton(event1));

    final StorageSummary summary = result.getStorageLocations().values().iterator().next();
    final long eventId = summary.getEventId();
    final ProvenanceEventRecord eventWithId = addId(event1, eventId);

    assertEquals(0, store.getMaxEventId());

    final ProvenanceEventRecord read = store.getEvent(eventId).get();
    assertEquals(eventWithId, read);
}
Example 3
Source File: WriteAheadStorePartition.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
private Map<ProvenanceEventRecord, StorageSummary> addEvents(final Iterable<ProvenanceEventRecord> events, final RecordWriter writer) throws IOException {
    final Map<ProvenanceEventRecord, StorageSummary> locationMap = new HashMap<>();

    try {
        long maxId = -1L;
        int numEvents = 0;
        for (final ProvenanceEventRecord nextEvent : events) {
            final StorageSummary writerSummary = writer.writeRecord(nextEvent);
            final StorageSummary summaryWithIndex = new StorageSummary(writerSummary.getEventId(), writerSummary.getStorageLocation(), this.partitionName,
                writerSummary.getBlockIndex(), writerSummary.getSerializedLength(), writerSummary.getBytesWritten());
            locationMap.put(nextEvent, summaryWithIndex);
            maxId = summaryWithIndex.getEventId();
            numEvents++;
        }

        if (numEvents == 0) {
            return locationMap;
        }

        writer.flush();

        // Update max event id to be equal to be the greater of the current value or the
        // max value just written.
        final long maxIdWritten = maxId;
        this.maxEventId.getAndUpdate(cur -> maxIdWritten > cur ? maxIdWritten : cur);

        if (config.isAlwaysSync()) {
            writer.sync();
        }
    } catch (final Exception e) {
        // We need to set the repoDirty flag before we release the lock for this journal.
        // Otherwise, another thread may write to this journal -- this is a problem because
        // the journal contains part of our record but not all of it. Writing to the end of this
        // journal will result in corruption!
        writer.markDirty();
        throw e;
    }

    return locationMap;
}
Example 4
Source File: WriteAheadStorePartition.java    From nifi with Apache License 2.0 5 votes vote down vote up 
private Map<ProvenanceEventRecord, StorageSummary> addEvents(final Iterable<ProvenanceEventRecord> events, final RecordWriter writer) throws IOException {
    final Map<ProvenanceEventRecord, StorageSummary> locationMap = new HashMap<>();

    try {
        long maxId = -1L;
        int numEvents = 0;
        for (final ProvenanceEventRecord nextEvent : events) {
            final StorageSummary writerSummary = writer.writeRecord(nextEvent);
            final StorageSummary summaryWithIndex = new StorageSummary(writerSummary.getEventId(), writerSummary.getStorageLocation(), this.partitionName,
                writerSummary.getBlockIndex(), writerSummary.getSerializedLength(), writerSummary.getBytesWritten());
            locationMap.put(nextEvent, summaryWithIndex);
            maxId = summaryWithIndex.getEventId();
            numEvents++;
        }

        if (numEvents == 0) {
            return locationMap;
        }

        writer.flush();

        // Update max event id to be equal to be the greater of the current value or the
        // max value just written.
        final long maxIdWritten = maxId;
        this.maxEventId.getAndUpdate(cur -> Math.max(maxIdWritten, cur));

        if (config.isAlwaysSync()) {
            writer.sync();
        }
    } catch (final Exception e) {
        // We need to set the repoDirty flag before we release the lock for this journal.
        // Otherwise, another thread may write to this journal -- this is a problem because
        // the journal contains part of our record but not all of it. Writing to the end of this
        // journal will result in corruption!
        writer.markDirty();
        throw e;
    }

    return locationMap;
}