Java Code Examples for org.apache.nifi.provenance.search.QueryResult#getMatchingEvents()
The following examples show how to use
org.apache.nifi.provenance.search.QueryResult#getMatchingEvents() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestLuceneEventIndex.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void addThenQueryWithEmptyQuery() throws InterruptedException { final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new SimpleIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP); final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); final Query query = new Query(UUID.randomUUID().toString()); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(100, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example 2
Source File: TestLuceneEventIndex.java From nifi with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void addThenQueryWithEmptyQuery() throws InterruptedException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new StandardIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP); final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); final Query query = new Query(UUID.randomUUID().toString()); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(4000, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example 3
Source File: TestLuceneEventIndex.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 50000) public void testQuerySpecificField() throws InterruptedException { final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new SimpleIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP); // add 2 events, one of which we will query for. final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L)); // Create a query that searches for the event with the FlowFile UUID equal to the first event's. final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid())); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(100, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example 4
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException { final List<SearchableField> searchableFields = new ArrayList<>(); searchableFields.add(SearchableFields.ComponentID); final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileLife(2, TimeUnit.SECONDS); config.setSearchableFields(searchableFields); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("uuid", UUID.randomUUID().toString()); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); final ProvenanceEventRecord record = builder.build(); for (int i = 0; i < 10; i++) { repo.registerEvent(record); } builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record. attributes.put("uuid", "11111111-1111-1111-1111-111111111111"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.) // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different // log file than the previous one. attributes.put("uuid", "22222222-2222-2222-2222-222222222222"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(2, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example 5
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example 6
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L * 10); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L * 10); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "10000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(2000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example 7
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } // Give time for rollover to happen repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(1000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example 8
Source File: TestPersistentProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 10000) public void testNotAuthorizedQuery() throws IOException, InterruptedException { final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(5, TimeUnit.MINUTES); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setDesiredIndexSize(10); // force new index to be created for each rollover repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) { @Override public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) { return event.getEventId() > 2; } }; repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(10L); // make sure the events are destroyed when we call purge repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query("1234"); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234")); final QuerySubmission submission = repo.submitQuery(query, createUser()); final QueryResult result = submission.getResult(); while (!result.isFinished()) { Thread.sleep(100L); } // Ensure that we gets events with ID's 3 through 10. final List<ProvenanceEventRecord> events = result.getMatchingEvents(); assertEquals(7, events.size()); final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList()); for (int i = 0; i < 7; i++) { Assert.assertEquals(i + 3, eventIds.get(i).intValue()); } }
Example 9
Source File: ControllerFacade.java From localization_nifi with Apache License 2.0 | 4 votes |
/** * Retrieves the results of a provenance query. * * @param provenanceId id * @return the results of a provenance query */ public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) { try { // get the query to the provenance repository final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository(); final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser()); // ensure the query results could be found if (querySubmission == null) { throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged."); } // get the original query and the results final Query query = querySubmission.getQuery(); final QueryResult queryResult = querySubmission.getResult(); // build the response final ProvenanceDTO provenanceDto = new ProvenanceDTO(); final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO(); final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO(); // include the original request and results provenanceDto.setRequest(requestDto); provenanceDto.setResults(resultsDto); // convert the original request requestDto.setStartDate(query.getStartDate()); requestDto.setEndDate(query.getEndDate()); requestDto.setMinimumFileSize(query.getMinFileSize()); requestDto.setMaximumFileSize(query.getMaxFileSize()); requestDto.setMaxResults(query.getMaxResults()); if (query.getSearchTerms() != null) { final Map<String, String> searchTerms = new HashMap<>(); for (final SearchTerm searchTerm : query.getSearchTerms()) { searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue()); } requestDto.setSearchTerms(searchTerms); } // convert the provenance provenanceDto.setId(query.getIdentifier()); provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime()); provenanceDto.setExpiration(queryResult.getExpiration()); provenanceDto.setFinished(queryResult.isFinished()); provenanceDto.setPercentCompleted(queryResult.getPercentComplete()); // convert each event final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults); if (includeResults || queryResult.isFinished()) { final List<ProvenanceEventDTO> events = new ArrayList<>(); for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) { events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize))); } resultsDto.setProvenanceEvents(events); } if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) { resultsDto.setTotalCount(requestDto.getMaxResults().longValue()); resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+"); } else { resultsDto.setTotalCount(queryResult.getTotalHitCount()); resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount())); } // include any errors if (queryResult.getError() != null) { final Set<String> errors = new HashSet<>(); errors.add(queryResult.getError()); resultsDto.setErrors(errors); } // set the generated timestamp final Date now = new Date(); resultsDto.setGenerated(now); resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime())); // get the oldest available event time final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1); if (!firstEvent.isEmpty()) { resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime())); } provenanceDto.setResults(resultsDto); return provenanceDto; } catch (final IOException ioe) { throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe); } }
Example 10
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException { assumeFalse(isWindowsEnvironment()); final List<SearchableField> searchableFields = new ArrayList<>(); searchableFields.add(SearchableFields.ComponentID); final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileCapacity(1024L * 1024L); config.setMaxEventFileLife(2, TimeUnit.SECONDS); config.setSearchableFields(searchableFields); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("uuid", UUID.randomUUID().toString()); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); final ProvenanceEventRecord record = builder.build(); for (int i = 0; i < 10; i++) { repo.registerEvent(record); } builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record. attributes.put("uuid", "11111111-1111-1111-1111-111111111111"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.) // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different // log file than the previous one. attributes.put("uuid", "22222222-2222-2222-2222-222222222222"); builder.fromFlowFile(createFlowFile(11L, 11L, attributes)); repo.registerEvent(builder.build()); repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(2, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example 11
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "000000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } }
Example 12
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentSearch() throws IOException, InterruptedException, ParseException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L * 10); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L * 10); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "10000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); // query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, "00000000-0000-0000-0000*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.Filename, "file-*")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "12?4")); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.TransitURI, "nifi://*")); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(2000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example 13
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test public void testIndexAndCompressOnRolloverAndSubsequentEmptySearch() throws IOException, InterruptedException, ParseException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(30, TimeUnit.SECONDS); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS); repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); attributes.put("uuid", uuid); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); repo.registerEvent(builder.build()); } // Give time for rollover to happen repo.waitForRollover(); final Query query = new Query(UUID.randomUUID().toString()); query.setMaxResults(100); final QueryResult result = repo.queryEvents(query, createUser()); assertEquals(10, result.getMatchingEvents().size()); for (final ProvenanceEventRecord match : result.getMatchingEvents()) { System.out.println(match); } Thread.sleep(2000L); config.setMaxStorageCapacity(100L); config.setMaxRecordLife(500, TimeUnit.MILLISECONDS); repo.purgeOldEvents(); Thread.sleep(1000L); final QueryResult newRecordSet = repo.queryEvents(query, createUser()); assertTrue(newRecordSet.getMatchingEvents().isEmpty()); }
Example 14
Source File: ITestPersistentProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 10000) public void testNotAuthorizedQuery() throws IOException, InterruptedException { assumeFalse(isWindowsEnvironment()); final RepositoryConfiguration config = createConfiguration(); config.setMaxRecordLife(5, TimeUnit.MINUTES); config.setMaxStorageCapacity(1024L * 1024L); config.setMaxEventFileLife(500, TimeUnit.MILLISECONDS); config.setMaxEventFileCapacity(1024L * 1024L); config.setSearchableFields(new ArrayList<>(SearchableFields.getStandardFields())); config.setDesiredIndexSize(10); // force new index to be created for each rollover repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS) { @Override public boolean isAuthorized(ProvenanceEventRecord event, NiFiUser user) { return event.getEventId() > 2; } }; repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY); final String uuid = "00000000-0000-0000-0000-000000000000"; final Map<String, String> attributes = new HashMap<>(); attributes.put("abc", "xyz"); attributes.put("xyz", "abc"); attributes.put("filename", "file-" + uuid); final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder(); builder.setEventTime(System.currentTimeMillis()); builder.setEventType(ProvenanceEventType.RECEIVE); builder.setTransitUri("nifi://unit-test"); builder.fromFlowFile(createFlowFile(3L, 3000L, attributes)); builder.setComponentId("1234"); builder.setComponentType("dummy processor"); for (int i = 0; i < 10; i++) { attributes.put("uuid", "00000000-0000-0000-0000-00000000000" + i); builder.fromFlowFile(createFlowFile(i, 3000L, attributes)); builder.setEventTime(10L); // make sure the events are destroyed when we call purge repo.registerEvent(builder.build()); } repo.waitForRollover(); final Query query = new Query("1234"); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "1234")); final QuerySubmission submission = repo.submitQuery(query, createUser()); final QueryResult result = submission.getResult(); while (!result.isFinished()) { Thread.sleep(100L); } // Ensure that we gets events with ID's 3 through 10. final List<ProvenanceEventRecord> events = result.getMatchingEvents(); assertEquals(7, events.size()); final List<Long> eventIds = events.stream().map(event -> event.getEventId()).sorted().collect(Collectors.toList()); for (int i = 0; i < 7; i++) { Assert.assertEquals(i + 3, eventIds.get(i).intValue()); } }
Example 15
Source File: TestLuceneEventIndex.java From nifi with Apache License 2.0 | 4 votes |
@Test(timeout = 50000) public void testQuerySpecificField() throws InterruptedException { final RepositoryConfiguration repoConfig = createConfig(); final IndexManager indexManager = new StandardIndexManager(repoConfig); final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 2, EventReporter.NO_OP); // add 2 events, one of which we will query for. final ProvenanceEventRecord event = createEvent(); index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L)); index.addEvent(createEvent(), new StorageSummary(2L, "1.prov", "1", 1, 2L, 2L)); // Create a query that searches for the event with the FlowFile UUID equal to the first event's. final Query query = new Query(UUID.randomUUID().toString()); query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.FlowFileUUID, event.getFlowFileUuid())); final ArrayListEventStore eventStore = new ArrayListEventStore(); eventStore.addEvent(event); index.initialize(eventStore); // We don't know how long it will take for the event to be indexed, so keep querying until // we get a result. The test will timeout after 5 seconds if we've still not succeeded. List<ProvenanceEventRecord> matchingEvents = Collections.emptyList(); while (matchingEvents.isEmpty()) { final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user"); assertNotNull(submission); final QueryResult result = submission.getResult(); assertNotNull(result); result.awaitCompletion(4000, TimeUnit.MILLISECONDS); assertTrue(result.isFinished()); assertNull(result.getError()); matchingEvents = result.getMatchingEvents(); assertNotNull(matchingEvents); Thread.sleep(100L); // avoid crushing the CPU } assertEquals(1, matchingEvents.size()); assertEquals(event, matchingEvents.get(0)); }
Example 16
Source File: ControllerFacade.java From nifi with Apache License 2.0 | 4 votes |
/** * Retrieves the results of a provenance query. * * @param provenanceId id * @return the results of a provenance query */ public ProvenanceDTO getProvenanceQuery(String provenanceId, Boolean summarize, Boolean incrementalResults) { try { // get the query to the provenance repository final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository(); final QuerySubmission querySubmission = provenanceRepository.retrieveQuerySubmission(provenanceId, NiFiUserUtils.getNiFiUser()); // ensure the query results could be found if (querySubmission == null) { throw new ResourceNotFoundException("Cannot find the results for the specified provenance requests. Results may have been purged."); } // get the original query and the results final Query query = querySubmission.getQuery(); final QueryResult queryResult = querySubmission.getResult(); // build the response final ProvenanceDTO provenanceDto = new ProvenanceDTO(); final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO(); final ProvenanceResultsDTO resultsDto = new ProvenanceResultsDTO(); // include the original request and results provenanceDto.setRequest(requestDto); provenanceDto.setResults(resultsDto); // convert the original request requestDto.setStartDate(query.getStartDate()); requestDto.setEndDate(query.getEndDate()); requestDto.setMinimumFileSize(query.getMinFileSize()); requestDto.setMaximumFileSize(query.getMaxFileSize()); requestDto.setMaxResults(query.getMaxResults()); if (query.getSearchTerms() != null) { final Map<String, String> searchTerms = new HashMap<>(); for (final SearchTerm searchTerm : query.getSearchTerms()) { searchTerms.put(searchTerm.getSearchableField().getFriendlyName(), searchTerm.getValue()); } requestDto.setSearchTerms(searchTerms); } // convert the provenance provenanceDto.setId(query.getIdentifier()); provenanceDto.setSubmissionTime(querySubmission.getSubmissionTime()); provenanceDto.setExpiration(queryResult.getExpiration()); provenanceDto.setFinished(queryResult.isFinished()); provenanceDto.setPercentCompleted(queryResult.getPercentComplete()); // convert each event final boolean includeResults = incrementalResults == null || Boolean.TRUE.equals(incrementalResults); if (includeResults || queryResult.isFinished()) { final List<ProvenanceEventDTO> events = new ArrayList<>(); for (final ProvenanceEventRecord record : queryResult.getMatchingEvents()) { events.add(createProvenanceEventDto(record, Boolean.TRUE.equals(summarize))); } resultsDto.setProvenanceEvents(events); } if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) { resultsDto.setTotalCount(requestDto.getMaxResults().longValue()); resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+"); } else { resultsDto.setTotalCount(queryResult.getTotalHitCount()); resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount())); } // include any errors if (queryResult.getError() != null) { final Set<String> errors = new HashSet<>(); errors.add(queryResult.getError()); resultsDto.setErrors(errors); } // set the generated timestamp final Date now = new Date(); resultsDto.setGenerated(now); resultsDto.setTimeOffset(TimeZone.getDefault().getOffset(now.getTime())); // get the oldest available event time final List<ProvenanceEventRecord> firstEvent = provenanceRepository.getEvents(0, 1); if (!firstEvent.isEmpty()) { resultsDto.setOldestEvent(new Date(firstEvent.get(0).getEventTime())); } provenanceDto.setResults(resultsDto); return provenanceDto; } catch (final IOException ioe) { throw new NiFiCoreException("An error occurred while searching the provenance events.", ioe); } }