org.apache.nifi.provenance.toc.TocUtil Java Examples

@Override
public File execute(final File expiredFile) throws IOException {
    final boolean removed = remove(expiredFile);
    if (removed) {
        logger.info("Removed expired Provenance Event file {}", expiredFile);
    } else {
        logger.warn("Failed to remove old Provenance Event file {}; this file should be cleaned up manually", expiredFile);
    }

    final File tocFile = TocUtil.getTocFile(expiredFile);
    if (remove(tocFile)) {
        logger.info("Removed expired Provenance Table-of-Contents file {}", tocFile);
    } else {
        logger.warn("Failed to remove old Provenance Table-of-Contents file {}; this file should be cleaned up manually", expiredFile);
    }

    return removed ? null : expiredFile;
}
 

private boolean delete(final File file) {
    final long firstEventId = DirectoryUtils.getMinId(file);
    synchronized (minEventIdToPathMap) {
        minEventIdToPathMap.remove(firstEventId);
    }

    if (!file.delete()) {
        logger.warn("Failed to remove Provenance Event file {}; this file should be cleaned up manually", file);
        return false;
    }

    final File tocFile = TocUtil.getTocFile(file);
    if (tocFile.exists() && !tocFile.delete()) {
        logger.warn("Failed to remove Provenance Table-of-Contents file {}; this file should be cleaned up manually", tocFile);
    }

    return true;
}
 

@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);
    final String expectedTransitUri = "nifi://unit-test";
        final int expectedBlockIndex = 0;

    assertRecoveredRecord(journalFile, tocReader, expectedTransitUri, expectedBlockIndex);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
    final IdentifierLookup idLookup) throws IOException {
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup);
    };

    final EventFileManager fileManager = new EventFileManager();
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
        fileManager.obtainReadLock(file);
        try {
            return RecordReaders.newRecordReader(file, logs, maxChars);
        } finally {
            fileManager.releaseReadLock(file);
        }
    };

   init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
}
 

@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Override
public File execute(final File expiredFile) throws IOException {
    final boolean removed = remove(expiredFile);
    if (removed) {
        logger.info("Removed expired Provenance Event file {}", expiredFile);
    } else {
        logger.warn("Failed to remove old Provenance Event file {}; this file should be cleaned up manually", expiredFile);
    }

    final File tocFile = TocUtil.getTocFile(expiredFile);
    if (remove(tocFile)) {
        logger.info("Removed expired Provenance Table-of-Contents file {}", tocFile);
    } else {
        logger.warn("Failed to remove old Provenance Table-of-Contents file {}; this file should be cleaned up manually", expiredFile);
    }

    return removed ? null : expiredFile;
}
 

@Test
public void testInitEmptyFile() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testInitEmptyFile");
    repoConfig.setMaxEventFileCount(5);

    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();

    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };

    final RecordReaderFactory recordReaderFactory = RecordReaders::newRecordReader;

    WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
            recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }

    long maxEventId = partition.getMaxEventId();
    assertTrue(maxEventId > 0);
    partition.close();

    final List<File> fileList = Arrays.asList(storageDirectory.listFiles(DirectoryUtils.EVENT_FILE_FILTER));
    Collections.sort(fileList, DirectoryUtils.LARGEST_ID_FIRST);

    // Create new empty prov file with largest id
    assertTrue(new File(storageDirectory, "1" + fileList.get(0).getName()).createNewFile());

    partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
            recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    partition.initialize();

    assertEquals(maxEventId, partition.getMaxEventId());
}
 

@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());

            // call skipToBlock half the time to ensure that we can; avoid calling it
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }

            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

private boolean delete(final File file) {
    final File activeEventFile = getActiveEventFile();
    if (file.equals(activeEventFile)) {
        logger.debug("Attempting to age off Active Event File {}. Will return without deleting the file.", file);
        return false;
    }

    final long firstEventId = DirectoryUtils.getMinId(file);
    synchronized (minEventIdToPathMap) {
        minEventIdToPathMap.remove(firstEventId);
    }

    eventFileManager.obtainWriteLock(file);
    try {
        if (!file.delete()) {
            logger.warn("Failed to remove Provenance Event file {}; this file should be cleaned up manually", file);
            return false;
        }

        final File tocFile = TocUtil.getTocFile(file);
        if (tocFile.exists() && !tocFile.delete()) {
            logger.warn("Failed to remove Provenance Table-of-Contents file {}; this file should be cleaned up manually", tocFile);
        }

        return true;
    } finally {
        eventFileManager.releaseWriteLock(file);
    }
}
 

@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());

            // call skipToBlock half the time to ensure that we can; avoid calling it
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }

            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory,
    final IdentifierLookup idLookup) throws IOException {
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup);
    };

    final EventFileManager fileManager = new EventFileManager();
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
        fileManager.obtainReadLock(file);
        try {
            return RecordReaders.newRecordReader(file, logs, maxChars);
        } finally {
            fileManager.releaseReadLock(file);
        }
    };

    eventStore = new PartitionedWriteAheadEventStore(config, recordWriterFactory, recordReaderFactory, eventReporter, fileManager);

    final IndexManager indexManager = new SimpleIndexManager(config);
    eventIndex = new LuceneEventIndex(config, indexManager, eventReporter);

    this.eventReporter = eventReporter;
    this.authorizer = authorizer;
    this.resourceFactory = resourceFactory;

    eventStore.initialize();
    eventIndex.initialize(eventStore);

    try {
        eventStore.reindexLatestEvents(eventIndex);
    } catch (final Exception e) {
        logger.error("Failed to re-index some of the Provenance Events. It is possible that some of the latest "
            + "events will not be available from the Provenance Repository when a query is issued.", e);
    }
}
 

@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testContentClaimRemoved() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim(null, null, null, 0L, 0L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertNull(recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertNull(recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertNull(recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertNull(recovered.getContentClaimOffset());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(0L, recovered.getFileSize());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

public static RecordWriter newSchemaRecordWriter(final File file, final AtomicLong idGenerator, final boolean compressed, final boolean createToc,
    final int compressionBlockBytes) throws IOException {
    final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
    return new ByteArraySchemaRecordWriter(file, idGenerator, tocWriter, compressed, compressionBlockBytes);
}
 

@Test
@SuppressWarnings("unchecked")
public void testReindex() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testReindex");
    repoConfig.setMaxEventFileCount(5);

    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();

    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };

    final RecordReaderFactory recordReaderFactory = RecordReaders::newRecordReader;

    final WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
        recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP, Mockito.mock(EventFileManager.class));

    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }

    final Map<ProvenanceEventRecord, StorageSummary> reindexedEvents = new ConcurrentHashMap<>();
    final EventIndex eventIndex = Mockito.mock(EventIndex.class);
    Mockito.doAnswer(new Answer<Object>() {
        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            final Map<ProvenanceEventRecord, StorageSummary> events = invocation.getArgument(0);
            reindexedEvents.putAll(events);
            return null;
        }
    }).when(eventIndex).reindexEvents(Mockito.anyMap());

    Mockito.doReturn(18L).when(eventIndex).getMinimumEventIdToReindex("1");
    partition.reindexLatestEvents(eventIndex);

    final List<Long> eventIdsReindexed = reindexedEvents.values().stream()
        .map(StorageSummary::getEventId)
        .sorted()
        .collect(Collectors.toList());

    assertEquals(82, eventIdsReindexed.size());
    for (int i = 0; i < eventIdsReindexed.size(); i++) {
        assertEquals(18 + i, eventIdsReindexed.get(i).intValue());
    }
}
 

@Before
public void setup() {
    journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testFieldAddedToSchema");
    tocFile = TocUtil.getTocFile(journalFile);
    idGenerator.set(0L);
}
 

@Before
public void setup() {
    journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testEventIdFirstSchemaRecordReaderWriter");
    tocFile = TocUtil.getTocFile(journalFile);
    idGenerator.set(0L);
}
 

@Test
public void testContentClaimUnchanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-1", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-1", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-1", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testContentClaimRemoved() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim(null, null, null, 0L, 0L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertNull(recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertNull(recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertNull(recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertNull(recovered.getContentClaimOffset());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(0L, recovered.getFileSize());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testContentClaimChanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-2", "section-2", "identifier-2", 2L, 2L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-2", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-2", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-2", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testContentClaimAdded() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getContentClaimContainer());
        assertNull(recovered.getPreviousContentClaimContainer());

        assertEquals("section-1", recovered.getContentClaimSection());
        assertNull(recovered.getPreviousContentClaimSection());

        assertEquals("identifier-1", recovered.getContentClaimIdentifier());
        assertNull(recovered.getPreviousContentClaimIdentifier());

        assertEquals(1L, recovered.getContentClaimOffset().longValue());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertEquals(1L, recovered.getFileSize());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testContentClaimUnchanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-1", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-1", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-1", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Before
public void setup() {
    journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testEventIdFirstSchemaRecordReaderWriter");
    tocFile = TocUtil.getTocFile(journalFile);
    idGenerator.set(0L);
}
 

@Test
public void testContentClaimAdded() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getContentClaimContainer());
        assertNull(recovered.getPreviousContentClaimContainer());

        assertEquals("section-1", recovered.getContentClaimSection());
        assertNull(recovered.getPreviousContentClaimSection());

        assertEquals("identifier-1", recovered.getContentClaimIdentifier());
        assertNull(recovered.getPreviousContentClaimIdentifier());

        assertEquals(1L, recovered.getContentClaimOffset().longValue());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertEquals(1L, recovered.getFileSize());
        assertNull(recovered.getPreviousContentClaimOffset());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

@Test
public void testContentClaimChanged() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    builder.setCurrentContentClaim("container-2", "section-2", "identifier-2", 2L, 2L);
    final ProvenanceEventRecord record = builder.build();

    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());

        assertEquals("container-1", recovered.getPreviousContentClaimContainer());
        assertEquals("container-2", recovered.getContentClaimContainer());

        assertEquals("section-1", recovered.getPreviousContentClaimSection());
        assertEquals("section-2", recovered.getContentClaimSection());

        assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
        assertEquals("identifier-2", recovered.getContentClaimIdentifier());

        assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertEquals(1L, recovered.getPreviousFileSize().longValue());
        assertEquals(2L, recovered.getContentClaimOffset().longValue());

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 

public static RecordWriter newSchemaRecordWriter(final File file, final AtomicLong idGenerator, final boolean compressed, final boolean createToc,
    final int compressionBlockBytes) throws IOException {
    final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
    return new ByteArraySchemaRecordWriter(file, idGenerator, tocWriter, compressed, compressionBlockBytes);
}
 

@Test
@SuppressWarnings("unchecked")
public void testReindex() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testReindex");
    repoConfig.setMaxEventFileCount(5);

    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();

    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };

    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> RecordReaders.newRecordReader(file, logs, maxChars);

    final WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory,
        recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP);

    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }

    final Map<ProvenanceEventRecord, StorageSummary> reindexedEvents = new HashMap<>();
    final EventIndex eventIndex = Mockito.mock(EventIndex.class);
    Mockito.doAnswer(new Answer<Object>() {
        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            final Map<ProvenanceEventRecord, StorageSummary> events = invocation.getArgumentAt(0, Map.class);
            reindexedEvents.putAll(events);
            return null;
        }
    }).when(eventIndex).reindexEvents(Mockito.anyMap());

    Mockito.doReturn(18L).when(eventIndex).getMinimumEventIdToReindex("1");
    partition.reindexLatestEvents(eventIndex);

    final List<Long> eventIdsReindexed = reindexedEvents.values().stream()
        .map(StorageSummary::getEventId)
        .sorted()
        .collect(Collectors.toList());

    assertEquals(82, eventIdsReindexed.size());
    for (int i = 0; i < eventIdsReindexed.size(); i++) {
        assertEquals(18 + i, eventIdsReindexed.get(i).intValue());
    }
}