org.apache.nifi.provenance.serialization.RecordWriter Java Examples

@Override
public synchronized void close() throws IOException {
    this.closed.set(true);
    writeLock.lock();
    try {
        logger.debug("Obtained write lock for close");

        scheduledExecService.shutdownNow();
        rolloverExecutor.shutdownNow();
        queryExecService.shutdownNow();

        getIndexManager().close();

        if (writers != null) {
            for (final RecordWriter writer : writers) {
                writer.close();
            }
        }
    } finally {
        writeLock.unlock();
    }
}
 

@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

protected RecordWriter[] createWriters(final RepositoryConfiguration config, final long initialRecordId) throws IOException {
    final List<File> storageDirectories = new ArrayList<>(config.getStorageDirectories().values());

    final RecordWriter[] writers = new RecordWriter[config.getJournalCount()];
    for (int i = 0; i < config.getJournalCount(); i++) {
        final File storageDirectory = storageDirectories.get(i % storageDirectories.size());
        final File journalDirectory = new File(storageDirectory, "journals");
        final File journalFile = new File(journalDirectory, String.valueOf(initialRecordId) + ".journal." + i);

        writers[i] = RecordWriters.newSchemaRecordWriter(journalFile, idGenerator, false, false);
        writers[i].writeHeader(initialRecordId);
    }

    logger.info("Created new Provenance Event Writers for events starting with ID {}", initialRecordId);
    return writers;
}
 

@Override
public synchronized void close() throws IOException {
    this.closed.set(true);
    writeLock.lock();
    try {
        logger.debug("Obtained write lock for close");

        scheduledExecService.shutdownNow();
        rolloverExecutor.shutdownNow();
        queryExecService.shutdownNow();

        getIndexManager().close();

        if (writers != null) {
            for (final RecordWriter writer : writers) {
                writer.close();
            }
        }
    } finally {
        writeLock.unlock();
    }
}
 

@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);
    final String expectedTransitUri = "nifi://unit-test";
        final int expectedBlockIndex = 0;

    assertRecoveredRecord(journalFile, tocReader, expectedTransitUri, expectedBlockIndex);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

protected RecordWriter[] createWriters(final RepositoryConfiguration config, final long initialRecordId) throws IOException {
    final List<File> storageDirectories = new ArrayList<>(config.getStorageDirectories().values());

    final RecordWriter[] writers = new RecordWriter[config.getJournalCount()];
    for (int i = 0; i < config.getJournalCount(); i++) {
        final File storageDirectory = storageDirectories.get(i % storageDirectories.size());
        final File journalDirectory = new File(storageDirectory, "journals");
        final File journalFile = new File(journalDirectory, String.valueOf(initialRecordId) + ".journal." + i);

        writers[i] = RecordWriters.newSchemaRecordWriter(journalFile, idGenerator, false, false);
        writers[i].writeHeader(initialRecordId);
    }

    logger.info("Created new Provenance Event Writers for events starting with ID {}", initialRecordId);
    return writers;
}
 

@Override
public synchronized void close() throws IOException {
    this.closed.set(true);
    writeLock.lock();
    try {
        logger.debug("Obtained write lock for close");

        scheduledExecService.shutdownNow();
        rolloverExecutor.shutdownNow();
        queryExecService.shutdownNow();

        getIndexManager().close();

        if (writers != null) {
            for (final RecordWriter writer : writers) {
                writer.close();
            }
        }
    } finally {
        writeLock.unlock();
    }
}
 

protected RecordWriter[] createWriters(final RepositoryConfiguration config, final long initialRecordId) throws IOException {
    final List<File> storageDirectories = new ArrayList<>(config.getStorageDirectories().values());

    final RecordWriter[] writers = new RecordWriter[config.getJournalCount()];
    for (int i = 0; i < config.getJournalCount(); i++) {
        final File storageDirectory = storageDirectories.get(i % storageDirectories.size());
        final File journalDirectory = new File(storageDirectory, "journals");
        final File journalFile = new File(journalDirectory, String.valueOf(initialRecordId) + ".journal." + i);

        writers[i] = RecordWriters.newSchemaRecordWriter(journalFile, idGenerator, false, false);
        writers[i].writeHeader(initialRecordId);
    }

    logger.info("Created new Provenance Event Writers for events starting with ID {}", initialRecordId);
    return writers;
}
 

@BeforeClass
public static void findJournalSizes() throws IOException {
    // determine header and record size

    final Map<String, String> attributes = new HashMap<>();
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", "12345678-0000-0000-0000-012345678912");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();
    builder.setComponentId("2345");
    final ProvenanceEventRecord record2 = builder.build();

    final File tempRecordFile = tempFolder.newFile("record.tmp");
    System.out.println("findJournalSizes position 0 = " + tempRecordFile.length());

    final AtomicLong idGenerator = new AtomicLong(0L);
    final RecordWriter writer = RecordWriters.newSchemaRecordWriter(tempRecordFile, idGenerator, false, false);
    writer.writeHeader(12345L);
    writer.flush();
    headerSize = Long.valueOf(tempRecordFile.length()).intValue();
    writer.writeRecord(record);
    writer.flush();
    recordSize = Long.valueOf(tempRecordFile.length()).intValue() - headerSize;
    writer.writeRecord(record2);
    writer.flush();
    recordSize2 = Long.valueOf(tempRecordFile.length()).intValue() - headerSize - recordSize;
    writer.close();

    System.out.println("headerSize =" + headerSize);
    System.out.println("recordSize =" + recordSize);
    System.out.println("recordSize2=" + recordSize2);
}
 

private Map<ProvenanceEventRecord, StorageSummary> addEvents(final Iterable<ProvenanceEventRecord> events, final RecordWriter writer) throws IOException {
    final Map<ProvenanceEventRecord, StorageSummary> locationMap = new HashMap<>();

    try {
        long maxId = -1L;
        int numEvents = 0;
        for (final ProvenanceEventRecord nextEvent : events) {
            final StorageSummary writerSummary = writer.writeRecord(nextEvent);
            final StorageSummary summaryWithIndex = new StorageSummary(writerSummary.getEventId(), writerSummary.getStorageLocation(), this.partitionName,
                writerSummary.getBlockIndex(), writerSummary.getSerializedLength(), writerSummary.getBytesWritten());
            locationMap.put(nextEvent, summaryWithIndex);
            maxId = summaryWithIndex.getEventId();
            numEvents++;
        }

        if (numEvents == 0) {
            return locationMap;
        }

        writer.flush();

        // Update max event id to be equal to be the greater of the current value or the
        // max value just written.
        final long maxIdWritten = maxId;
        this.maxEventId.getAndUpdate(cur -> Math.max(maxIdWritten, cur));

        if (config.isAlwaysSync()) {
            writer.sync();
        }
    } catch (final Exception e) {
        // We need to set the repoDirty flag before we release the lock for this journal.
        // Otherwise, another thread may write to this journal -- this is a problem because
        // the journal contains part of our record but not all of it. Writing to the end of this
        // journal will result in corruption!
        writer.markDirty();
        throw e;
    }

    return locationMap;
}
 

RecordWriter[] getWriters() {
    Class klass = MiNiFiPersistentProvenanceRepository.class;
    Field writersField;
    RecordWriter[] writers = null;
    try {
        writersField = klass.getDeclaredField("writers");
        writersField.setAccessible(true);
        writers = (RecordWriter[]) writersField.get(this);
    } catch (NoSuchFieldException | IllegalAccessException e) {
        e.printStackTrace();
    }
    return writers;
}
 

RecordWriter[] getWriters() {
    Class klass = PersistentProvenanceRepository.class;
    Field writersField;
    RecordWriter[] writers = null;
    try {
        writersField = klass.getDeclaredField("writers");
        writersField.setAccessible(true);
        writers = (RecordWriter[]) writersField.get(this);
    } catch (NoSuchFieldException | IllegalAccessException e) {
        e.printStackTrace();
    }
    return writers;
}
 

public RecordWriterLease(final RecordWriter writer, final long maxBytes, final int maxEvents, final long maxMillis) {
    this.writer = writer;
    this.maxBytes = maxBytes;
    this.maxEvents = maxEvents;

    // The max timestamp that we want to write to this lease is X number of milliseconds into the future.
    // We don't want X to be more than the given max millis. However, we also don't want to allow it to get too large. If it
    // becomes >= Integer.MAX_VALUE, we could have some timestamp offsets that rollover into the negative range.
    // To avoid that, we could use a value that is no more than Integer.MAX_VALUE. But since the event may be persisted
    // a bit after the lease has been obtained, we subtract 1 hour from that time to give ourselves a little buffer room.
    this.maxSystemTime = System.currentTimeMillis() + Math.min(maxMillis, Integer.MAX_VALUE - TimeUnit.HOURS.toMillis(1));
}
 

@BeforeClass
public static void findJournalSizes() throws IOException {
    // determine header and record size

    final Map<String, String> attributes = new HashMap<>();
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", "12345678-0000-0000-0000-012345678912");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();
    builder.setComponentId("2345");
    final ProvenanceEventRecord record2 = builder.build();

    final File tempRecordFile = tempFolder.newFile("record.tmp");
    System.out.println("findJournalSizes position 0 = " + tempRecordFile.length());

    final AtomicLong idGenerator = new AtomicLong(0L);
    final RecordWriter writer = RecordWriters.newSchemaRecordWriter(tempRecordFile, idGenerator, false, false);
    writer.writeHeader(12345L);
    writer.flush();
    headerSize = Long.valueOf(tempRecordFile.length()).intValue();
    writer.writeRecord(record);
    writer.flush();
    recordSize = Long.valueOf(tempRecordFile.length()).intValue() - headerSize;
    writer.writeRecord(record2);
    writer.flush();
    recordSize2 = Long.valueOf(tempRecordFile.length()).intValue() - headerSize - recordSize;
    writer.close();

    System.out.println("headerSize =" + headerSize);
    System.out.println("recordSize =" + recordSize);
    System.out.println("recordSize2=" + recordSize2);
}
 

@BeforeClass
public static void findJournalSizes() throws IOException {
    // determine header and record size

    final Map<String, String> attributes = new HashMap<>();
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", "12345678-0000-0000-0000-012345678912");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();
    builder.setComponentId("2345");
    final ProvenanceEventRecord record2 = builder.build();

    final File tempRecordFile = tempFolder.newFile("record.tmp");
    System.out.println("findJournalSizes position 0 = " + tempRecordFile.length());

    final AtomicLong idGenerator = new AtomicLong(0L);
    final RecordWriter writer = RecordWriters.newSchemaRecordWriter(tempRecordFile, idGenerator, false, false);
    writer.writeHeader(12345L);
    writer.flush();
    headerSize = Long.valueOf(tempRecordFile.length()).intValue();
    writer.writeRecord(record);
    writer.flush();
    recordSize = Long.valueOf(tempRecordFile.length()).intValue() - headerSize;
    writer.writeRecord(record2);
    writer.flush();
    recordSize2 = Long.valueOf(tempRecordFile.length()).intValue() - headerSize - recordSize;
    writer.close();

    System.out.println("headerSize =" + headerSize);
    System.out.println("recordSize =" + recordSize);
    System.out.println("recordSize2=" + recordSize2);
}
 

@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());

            // call skipToBlock half the time to ensure that we can; avoid calling it
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }

            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());

            // call skipToBlock half the time to ensure that we can; avoid calling it
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }

            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

RecordWriter[] getWriters() {
    Class klass = PersistentProvenanceRepository.class;
    Field writersField;
    RecordWriter[] writers = null;
    try {
        writersField = klass.getDeclaredField("writers");
        writersField.setAccessible(true);
        writers = (RecordWriter[]) writersField.get(this);
    } catch (NoSuchFieldException | IllegalAccessException e) {
        e.printStackTrace();
    }
    return writers;
}
 

private Map<ProvenanceEventRecord, StorageSummary> addEvents(final Iterable<ProvenanceEventRecord> events, final RecordWriter writer) throws IOException {
    final Map<ProvenanceEventRecord, StorageSummary> locationMap = new HashMap<>();

    try {
        long maxId = -1L;
        int numEvents = 0;
        for (final ProvenanceEventRecord nextEvent : events) {
            final StorageSummary writerSummary = writer.writeRecord(nextEvent);
            final StorageSummary summaryWithIndex = new StorageSummary(writerSummary.getEventId(), writerSummary.getStorageLocation(), this.partitionName,
                writerSummary.getBlockIndex(), writerSummary.getSerializedLength(), writerSummary.getBytesWritten());
            locationMap.put(nextEvent, summaryWithIndex);
            maxId = summaryWithIndex.getEventId();
            numEvents++;
        }

        if (numEvents == 0) {
            return locationMap;
        }

        writer.flush();

        // Update max event id to be equal to be the greater of the current value or the
        // max value just written.
        final long maxIdWritten = maxId;
        this.maxEventId.getAndUpdate(cur -> maxIdWritten > cur ? maxIdWritten : cur);

        if (config.isAlwaysSync()) {
            writer.sync();
        }
    } catch (final Exception e) {
        // We need to set the repoDirty flag before we release the lock for this journal.
        // Otherwise, another thread may write to this journal -- this is a problem because
        // the journal contains part of our record but not all of it. Writing to the end of this
        // journal will result in corruption!
        writer.markDirty();
        throw e;
    }

    return locationMap;
}
 

private RecordWriter createWriter(final File file, final TocWriter tocWriter, final boolean compressed, final int uncompressedBlockSize) throws IOException {
    return new EventIdFirstSchemaRecordWriter(file, new AtomicLong(0L), tocWriter, compressed, uncompressedBlockSize, IdentifierLookup.EMPTY);
}
 

@Test
public void testMergeJournalsBadFirstRecord() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileLife(3, TimeUnit.SECONDS);
    TestablePersistentProvenanceRepository testRepo = new TestablePersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    testRepo.initialize(getEventReporter(), null, null, null);

    final Map<String, String> attributes = new HashMap<>();

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", "12345678-0000-0000-0000-012345678912");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    final ProvenanceEventRecord record = builder.build();

    final ExecutorService exec = Executors.newFixedThreadPool(10);
    final List<Future> futures = new ArrayList<>();
    for (int i = 0; i < 10000; i++) {
        futures.add(exec.submit(new Runnable() {
            @Override
            public void run() {
                testRepo.registerEvent(record);
            }
        }));
    }

    // wait for writers to finish and then corrupt the first record of the first journal file
    for (Future future : futures) {
        while (!future.isDone()) {
            Thread.sleep(10);
        }
    }
    RecordWriter firstWriter = testRepo.getWriters()[0];
    corruptJournalFile(firstWriter.getFile(), headerSize + 15, "RECEIVE", "BADTYPE");

    testRepo.recoverJournalFiles();

    final File storageDir = config.getStorageDirectories().values().iterator().next();
    assertTrue(checkJournalRecords(storageDir, false) < 10000);
}
 

@Override
protected RecordWriter createWriter(File file, TocWriter tocWriter, boolean compressed, int uncompressedBlockSize) throws IOException {
    return new ByteArraySchemaRecordWriter(file, idGenerator, tocWriter, compressed, uncompressedBlockSize);
}
 

@Override
protected RecordWriter createWriter(File file, TocWriter tocWriter, boolean compressed, int uncompressedBlockSize) throws IOException {
    return new StandardRecordWriter(file, idGenerator, tocWriter, compressed, uncompressedBlockSize);
}
 

public RecordWriter getWriter() {
    return writer;
}
 

private synchronized boolean tryRollover(final RecordWriterLease lease) throws IOException {
    if (!Objects.equals(lease, eventWriterLeaseRef.get())) {
        return false;
    }

    final long nextEventId = idGenerator.get();
    final File updatedEventFile = new File(partitionDirectory, nextEventId + ".prov");
    final RecordWriter updatedWriter = recordWriterFactory.createWriter(updatedEventFile, idGenerator, false, true);
    updatedWriter.writeHeader(nextEventId);

    final RecordWriterLease updatedLease = new RecordWriterLease(updatedWriter, config.getMaxEventFileCapacity(), config.getMaxEventFileCount(), config.getMaxEventFileLife(TimeUnit.MILLISECONDS));
    final boolean updated = eventWriterLeaseRef.compareAndSet(lease, updatedLease);

    if (!updated) {
        try {
            updatedWriter.close();
        } catch (final Exception e) {
            logger.warn("Failed to close Record Writer {}; some resources may not be cleaned up properly.", updatedWriter, e);
        }

        updatedEventFile.delete();
        return false;
    }

    if (lease != null) {
        lease.close();
    }

    synchronized (minEventIdToPathMap) {
        minEventIdToPathMap.put(nextEventId, updatedEventFile);
    }

    if (config.isCompressOnRollover() && lease != null && lease.getWriter() != null) {
        boolean offered = false;
        while (!offered && !closed) {
            try {
                offered = filesToCompress.offer(lease.getWriter().getFile(), 1, TimeUnit.SECONDS);
            } catch (final InterruptedException ie) {
                Thread.currentThread().interrupt();
                throw new IOException("Interrupted while waiting to enqueue " + lease.getWriter().getFile() + " for compression");
            }
        }
    }

    return true;
}
 

@Test
public void testFailureToCreateWriterDoesNotPreventSubsequentRollover() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration config = createConfiguration();
    config.setMaxAttributeChars(50);
    config.setMaxEventFileLife(3, TimeUnit.SECONDS);

    // Create a repo that will allow only a single writer to be created.
    final IOException failure = new IOException("Already created writers once. Unit test causing failure.");
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) {
        int iterations = 0;

        @Override
        protected RecordWriter[] createWriters(RepositoryConfiguration config, long initialRecordId) throws IOException {
            if (iterations++ == 1) {
                throw failure;
            } else {
                return super.createWriters(config, initialRecordId);
            }
        }
    };

    // initialize with our event reporter
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    // create some events in the journal files.
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("75chars", "123456789012345678901234567890123456789012345678901234567890123456789012345");

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    attributes.put("uuid", "12345678-0000-0000-0000-012345678912");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");

    for (int i = 0; i < 50; i++) {
        final ProvenanceEventRecord event = builder.build();
        repo.registerEvent(event);
    }

    // Attempt to rollover but fail to create new writers.
    try {
        repo.rolloverWithLock(true);
        Assert.fail("Expected to get IOException when calling rolloverWithLock");
    } catch (final IOException ioe) {
        assertTrue(ioe == failure);
    }

    // Wait for the first rollover to succeed.
    repo.waitForRollover();

    // This time when we rollover, we should not have a problem rolling over.
    repo.rolloverWithLock(true);

    // Ensure that no errors were reported.
    assertEquals(0, reportedEvents.size());
}