org.apache.nifi.util.RingBuffer Java Examples
The following examples show how to use
org.apache.nifi.util.RingBuffer.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRingBuffer.java From localization_nifi with Apache License 2.0 | 6 votes |
@Test public void testIterateForwards() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } final AtomicInteger countHolder = new AtomicInteger(0); ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { final int expected = values[counter++]; countHolder.incrementAndGet(); assertEquals(expected, value.intValue()); return true; } }, IterationDirection.FORWARD); assertEquals(4, countHolder.get()); }
Example #2
Source File: VolatileBulletinRepository.java From nifi with Apache License 2.0 | 6 votes |
@Override public List<Bulletin> findBulletinsForGroupBySource(final String groupId, final int maxPerComponent) { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); final ConcurrentMap<String, RingBuffer<Bulletin>> componentMap = bulletinStoreMap.get(groupId); if (componentMap == null) { return Collections.<Bulletin>emptyList(); } final List<Bulletin> allComponentBulletins = new ArrayList<>(); for (final RingBuffer<Bulletin> ringBuffer : componentMap.values()) { allComponentBulletins.addAll(ringBuffer.getSelectedElements(new Filter<Bulletin>() { @Override public boolean select(final Bulletin bulletin) { return bulletin.getTimestamp().getTime() >= fiveMinutesAgo; } }, maxPerComponent)); } return allComponentBulletins; }
Example #3
Source File: VolatileProvenanceRepository.java From nifi with Apache License 2.0 | 6 votes |
public VolatileProvenanceRepository(final NiFiProperties nifiProperties) { maxSize = nifiProperties.getIntegerProperty(BUFFER_SIZE, DEFAULT_BUFFER_SIZE); ringBuffer = new RingBuffer<>(maxSize); final String indexedFieldString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_FIELDS); final String indexedAttrString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_ATTRIBUTES); searchableFields = Collections.unmodifiableList(SearchableFieldParser.extractSearchableFields(indexedFieldString, true)); searchableAttributes = Collections.unmodifiableList(SearchableFieldParser.extractSearchableFields(indexedAttrString, false)); final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); queryExecService = Executors.newFixedThreadPool(2, new ThreadFactory() { private final AtomicInteger counter = new AtomicInteger(0); @Override public Thread newThread(final Runnable r) { final Thread thread = defaultThreadFactory.newThread(r); thread.setName("Provenance Query Thread-" + counter.incrementAndGet()); return thread; } }); scheduledExecService = Executors.newScheduledThreadPool(2); }
Example #4
Source File: VolatileProvenanceRepository.java From localization_nifi with Apache License 2.0 | 6 votes |
public VolatileProvenanceRepository(final NiFiProperties nifiProperties) { final int bufferSize = nifiProperties.getIntegerProperty(BUFFER_SIZE, DEFAULT_BUFFER_SIZE); ringBuffer = new RingBuffer<>(bufferSize); final String indexedFieldString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_FIELDS); final String indexedAttrString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_ATTRIBUTES); searchableFields = Collections.unmodifiableList(SearchableFieldParser.extractSearchableFields(indexedFieldString, true)); searchableAttributes = Collections.unmodifiableList(SearchableFieldParser.extractSearchableFields(indexedAttrString, false)); final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); queryExecService = Executors.newFixedThreadPool(2, new ThreadFactory() { private final AtomicInteger counter = new AtomicInteger(0); @Override public Thread newThread(final Runnable r) { final Thread thread = defaultThreadFactory.newThread(r); thread.setName("Provenance Query Thread-" + counter.incrementAndGet()); return thread; } }); scheduledExecService = Executors.newScheduledThreadPool(2); }
Example #5
Source File: TestRingBuffer.java From nifi with Apache License 2.0 | 6 votes |
@Test public void testIterateForwards() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } final AtomicInteger countHolder = new AtomicInteger(0); ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { final int expected = values[counter++]; countHolder.incrementAndGet(); assertEquals(expected, value.intValue()); return true; } }, IterationDirection.FORWARD); assertEquals(4, countHolder.get()); }
Example #6
Source File: VolatileBulletinRepository.java From localization_nifi with Apache License 2.0 | 6 votes |
@Override public List<Bulletin> findBulletinsForGroupBySource(final String groupId, final int maxPerComponent) { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); final ConcurrentMap<String, RingBuffer<Bulletin>> componentMap = bulletinStoreMap.get(groupId); if (componentMap == null) { return Collections.<Bulletin>emptyList(); } final List<Bulletin> allComponentBulletins = new ArrayList<>(); for (final RingBuffer<Bulletin> ringBuffer : componentMap.values()) { allComponentBulletins.addAll(ringBuffer.getSelectedElements(new Filter<Bulletin>() { @Override public boolean select(final Bulletin bulletin) { return bulletin.getTimestamp().getTime() >= fiveMinutesAgo; } }, maxPerComponent)); } return allComponentBulletins; }
Example #7
Source File: TestRingBuffer.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testAsList() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); final List<Integer> emptyList = ringBuffer.asList(); assertTrue(emptyList.isEmpty()); for (int i = 0; i < 3; i++) { ringBuffer.add(i); } List<Integer> list = ringBuffer.asList(); assertEquals(3, list.size()); for (int i = 0; i < 3; i++) { assertEquals(Integer.valueOf(i), list.get(i)); } for (int i = 3; i < 10; i++) { ringBuffer.add(i); } list = ringBuffer.asList(); assertEquals(10, list.size()); for (int i = 0; i < 10; i++) { assertEquals(Integer.valueOf(i), list.get(i)); } }
Example #8
Source File: VolatileBulletinRepository.java From nifi with Apache License 2.0 | 5 votes |
@Override public List<Bulletin> findBulletinsForController(final int max) { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); final Filter<Bulletin> filter = new Filter<Bulletin>() { @Override public boolean select(final Bulletin bulletin) { return bulletin.getTimestamp().getTime() >= fiveMinutesAgo; } }; final List<Bulletin> controllerBulletins = new ArrayList<>(); final ConcurrentMap<String, RingBuffer<Bulletin>> controllerBulletinMap = bulletinStoreMap.get(CONTROLLER_BULLETIN_STORE_KEY); if (controllerBulletinMap != null) { final RingBuffer<Bulletin> buffer = controllerBulletinMap.get(CONTROLLER_BULLETIN_STORE_KEY); if (buffer != null) { controllerBulletins.addAll(buffer.getSelectedElements(filter, max)); } } // We only want the newest bulletin, so we sort based on time and take the top 'max' entries Collections.sort(controllerBulletins); if (controllerBulletins.size() > max) { return controllerBulletins.subList(0, max); } return controllerBulletins; }
Example #9
Source File: LatestEventsPerProcessorQuery.java From nifi with Apache License 2.0 | 5 votes |
@Override public Optional<List<Long>> evaluate(final Query query) { if (query.getMaxResults() > 1000) { // If query max results > 1000 then we know we don't have enough results. So just return empty. return Optional.empty(); } final List<SearchTerm> terms = query.getSearchTerms(); if (terms.size() != 1) { return Optional.empty(); } final SearchTerm term = terms.get(0); if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) { return Optional.empty(); } if (query.getEndDate() != null || query.getStartDate() != null) { return Optional.empty(); } final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue()); if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) { return Optional.empty(); } List<Long> eventIds = ringBuffer.asList(); if (eventIds.size() > query.getMaxResults()) { eventIds = eventIds.subList(0, query.getMaxResults()); } return Optional.of(eventIds); }
Example #10
Source File: TestRingBuffer.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testIterateBackwardsAfterFull() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); for (int i = 0; i < 12; i++) { ringBuffer.add(i); } final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { if (counter < values.length) { final int index = values.length - 1 - counter; final int expected = values[index]; assertEquals(expected, value.intValue()); counter++; } return true; } }, IterationDirection.BACKWARD); }
Example #11
Source File: TestRingBuffer.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testIterateBackwards() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } final AtomicInteger countHolder = new AtomicInteger(0); ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { final int index = values.length - 1 - counter; final int expected = values[index]; countHolder.incrementAndGet(); assertEquals(expected, value.intValue()); counter++; return true; } }, IterationDirection.BACKWARD); assertEquals(4, countHolder.get()); }
Example #12
Source File: TestRingBuffer.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testIterateForwardsAfterFull() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); for (int i = 0; i < 12; i++) { ringBuffer.add(i); } final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { if (counter < 6) { assertEquals(counter + 6, value.intValue()); } else { final int expected = values[counter - 6]; assertEquals(expected, value.intValue()); } counter++; return true; } }, IterationDirection.FORWARD); }
Example #13
Source File: TestRingBuffer.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testAsList() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); final List<Integer> emptyList = ringBuffer.asList(); assertTrue(emptyList.isEmpty()); for (int i = 0; i < 3; i++) { ringBuffer.add(i); } List<Integer> list = ringBuffer.asList(); assertEquals(3, list.size()); for (int i = 0; i < 3; i++) { assertEquals(Integer.valueOf(i), list.get(i)); } for (int i = 3; i < 10; i++) { ringBuffer.add(i); } list = ringBuffer.asList(); assertEquals(10, list.size()); for (int i = 0; i < 10; i++) { assertEquals(Integer.valueOf(i), list.get(i)); } }
Example #14
Source File: TestRingBuffer.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testGetNewestElement() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); for (int i = 0; i < 11; i++) { ringBuffer.add(i); assertEquals(i, ringBuffer.getNewestElement().intValue()); } }
Example #15
Source File: VolatileBulletinRepository.java From localization_nifi with Apache License 2.0 | 5 votes |
@Override public List<Bulletin> findBulletinsForController(final int max) { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); final Filter<Bulletin> filter = new Filter<Bulletin>() { @Override public boolean select(final Bulletin bulletin) { return bulletin.getTimestamp().getTime() >= fiveMinutesAgo; } }; final List<Bulletin> controllerBulletins = new ArrayList<>(); final ConcurrentMap<String, RingBuffer<Bulletin>> controllerBulletinMap = bulletinStoreMap.get(CONTROLLER_BULLETIN_STORE_KEY); if (controllerBulletinMap != null) { final RingBuffer<Bulletin> buffer = controllerBulletinMap.get(CONTROLLER_BULLETIN_STORE_KEY); if (buffer != null) { controllerBulletins.addAll(buffer.getSelectedElements(filter, max)); } } // We only want the newest bulletin, so we sort based on time and take the top 'max' entries Collections.sort(controllerBulletins); if (controllerBulletins.size() > max) { return controllerBulletins.subList(0, max); } return controllerBulletins; }
Example #16
Source File: TestRingBuffer.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testGetNewestElement() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); for (int i = 0; i < 11; i++) { ringBuffer.add(i); assertEquals(i, ringBuffer.getNewestElement().intValue()); } }
Example #17
Source File: TestRingBuffer.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testIterateForwardsAfterFull() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); for (int i = 0; i < 12; i++) { ringBuffer.add(i); } final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { if (counter < 6) { assertEquals(counter + 6, value.intValue()); } else { final int expected = values[counter - 6]; assertEquals(expected, value.intValue()); } counter++; return true; } }, IterationDirection.FORWARD); }
Example #18
Source File: LatestEventsPerProcessorQuery.java From localization_nifi with Apache License 2.0 | 5 votes |
@Override public Optional<List<Long>> evaluate(final Query query) { if (query.getMaxResults() > 1000) { // If query max results > 1000 then we know we don't have enough results. So just return empty. return Optional.empty(); } final List<SearchTerm> terms = query.getSearchTerms(); if (terms.size() != 1) { return Optional.empty(); } final SearchTerm term = terms.get(0); if (!COMPONENT_ID_FIELD_NAME.equals(term.getSearchableField().getSearchableFieldName())) { return Optional.empty(); } if (query.getEndDate() != null || query.getStartDate() != null) { return Optional.empty(); } final RingBuffer<Long> ringBuffer = latestRecords.get(term.getValue()); if (ringBuffer == null || ringBuffer.getSize() < query.getMaxResults()) { return Optional.empty(); } List<Long> eventIds = ringBuffer.asList(); if (eventIds.size() > query.getMaxResults()) { eventIds = eventIds.subList(0, query.getMaxResults()); } return Optional.of(eventIds); }
Example #19
Source File: TestRingBuffer.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testIterateBackwardsAfterFull() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); for (int i = 0; i < 12; i++) { ringBuffer.add(i); } final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { if (counter < values.length) { final int index = values.length - 1 - counter; final int expected = values[index]; assertEquals(expected, value.intValue()); counter++; } return true; } }, IterationDirection.BACKWARD); }
Example #20
Source File: TestRingBuffer.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testIterateBackwards() { final RingBuffer<Integer> ringBuffer = new RingBuffer<>(10); final int[] values = new int[]{3, 5, 20, 7}; for (final int v : values) { ringBuffer.add(v); } final AtomicInteger countHolder = new AtomicInteger(0); ringBuffer.forEach(new ForEachEvaluator<Integer>() { int counter = 0; @Override public boolean evaluate(final Integer value) { final int index = values.length - 1 - counter; final int expected = values[index]; countHolder.incrementAndGet(); assertEquals(expected, value.intValue()); counter++; return true; } }, IterationDirection.BACKWARD); assertEquals(4, countHolder.get()); }
Example #21
Source File: VolatileBulletinRepository.java From nifi with Apache License 2.0 | 4 votes |
@Override public void update(final Bulletin bulletin) { for (final RingBuffer<Bulletin> bulletinBuffer : getBulletinBuffers(bulletin)) { bulletinBuffer.add(bulletin); } }
Example #22
Source File: VolatileBulletinRepository.java From nifi with Apache License 2.0 | 4 votes |
@Override public List<Bulletin> findBulletins(final BulletinQuery bulletinQuery) { final Filter<Bulletin> filter = new Filter<Bulletin>() { @Override public boolean select(final Bulletin bulletin) { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); if (bulletin.getTimestamp().getTime() < fiveMinutesAgo) { return false; } // only include bulletins after the specified id if (bulletinQuery.getAfter() != null && bulletin.getId() <= bulletinQuery.getAfter()) { return false; } // if group pattern was specified see if it should be excluded if (bulletinQuery.getGroupIdPattern() != null) { // exclude if this bulletin doesnt have a group or if it doesnt match if (bulletin.getGroupId() == null || !bulletinQuery.getGroupIdPattern().matcher(bulletin.getGroupId()).find()) { return false; } } // if a message pattern was specified see if it should be excluded if (bulletinQuery.getMessagePattern() != null) { // exclude if this bulletin doesnt have a message or if it doesnt match if (bulletin.getMessage() == null || !bulletinQuery.getMessagePattern().matcher(bulletin.getMessage()).find()) { return false; } } // if a name pattern was specified see if it should be excluded if (bulletinQuery.getNamePattern() != null) { // exclude if this bulletin doesnt have a source name or if it doesnt match if (bulletin.getSourceName() == null || !bulletinQuery.getNamePattern().matcher(bulletin.getSourceName()).find()) { return false; } } // if a source id was specified see if it should be excluded if (bulletinQuery.getSourceIdPattern() != null) { // exclude if this bulletin doesn't have a source id or if it doesn't match if (bulletin.getSourceId() == null || !bulletinQuery.getSourceIdPattern().matcher(bulletin.getSourceId()).find()) { return false; } } // if a source component type was specified see if it should be excluded if (bulletinQuery.getSourceType() != null) { // exclude if this bulletin source type doesn't match if (bulletin.getSourceType() == null || !bulletinQuery.getSourceType().equals(bulletin.getSourceType())) { return false; } } return true; } }; final Set<Bulletin> selected = new TreeSet<>(); int max = bulletinQuery.getLimit() == null ? Integer.MAX_VALUE : bulletinQuery.getLimit(); for (final ConcurrentMap<String, RingBuffer<Bulletin>> componentMap : bulletinStoreMap.values()) { for (final RingBuffer<Bulletin> ringBuffer : componentMap.values()) { final List<Bulletin> bulletinsForComponent = ringBuffer.getSelectedElements(filter, max); selected.addAll(bulletinsForComponent); max -= bulletinsForComponent.size(); if (max <= 0) { break; } } } return new ArrayList<>(selected); }
Example #23
Source File: RingBufferGarbageCollectionLog.java From nifi with Apache License 2.0 | 4 votes |
public RingBufferGarbageCollectionLog(final int eventCount, final long minDurationThreshold) { this.events = new RingBuffer<>(eventCount); this.minDurationThreshold = minDurationThreshold; jvmStartTime = ManagementFactory.getRuntimeMXBean().getStartTime(); }
Example #24
Source File: VolatileComponentStatusRepository.java From nifi with Apache License 2.0 | 4 votes |
public VolatileComponentStatusRepository(final NiFiProperties nifiProperties) { numDataPoints = nifiProperties.getIntegerProperty(NUM_DATA_POINTS_PROPERTY, DEFAULT_NUM_DATA_POINTS); gcStatuses = new RingBuffer<>(numDataPoints); timestamps = new RingBuffer<>(numDataPoints); }
Example #25
Source File: VolatileProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
public ComputeLineageRunnable(final RingBuffer<ProvenanceEventRecord> ringBuffer, final Filter<ProvenanceEventRecord> filter, final AsyncLineageSubmission submission) { this.ringBuffer = ringBuffer; this.filter = filter; this.submission = submission; }
Example #26
Source File: VolatileProvenanceRepository.java From nifi with Apache License 2.0 | 4 votes |
public QueryRunnable(final RingBuffer<ProvenanceEventRecord> ringBuffer, final Filter<ProvenanceEventRecord> filter, final int maxRecords, final AsyncQuerySubmission submission) { this.ringBuffer = ringBuffer; this.filter = filter; this.submission = submission; this.maxRecords = maxRecords; }
Example #27
Source File: LatestEventsPerProcessorQuery.java From nifi with Apache License 2.0 | 4 votes |
@Override public void update(final ProvenanceEventRecord event, final StorageSummary storageSummary) { final String componentId = event.getComponentId(); final RingBuffer<Long> ringBuffer = latestRecords.computeIfAbsent(componentId, id -> new RingBuffer<>(1000)); ringBuffer.add(storageSummary.getEventId()); }
Example #28
Source File: LatestEventsPerProcessorQuery.java From localization_nifi with Apache License 2.0 | 4 votes |
@Override public void update(final ProvenanceEventRecord event, final StorageSummary storageSummary) { final String componentId = event.getComponentId(); final RingBuffer<Long> ringBuffer = latestRecords.computeIfAbsent(componentId, id -> new RingBuffer<>(1000)); ringBuffer.add(storageSummary.getEventId()); }
Example #29
Source File: VolatileProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
public QueryRunnable(final RingBuffer<ProvenanceEventRecord> ringBuffer, final Filter<ProvenanceEventRecord> filter, final int maxRecords, final AsyncQuerySubmission submission) { this.ringBuffer = ringBuffer; this.filter = filter; this.submission = submission; this.maxRecords = maxRecords; }
Example #30
Source File: VolatileProvenanceRepository.java From localization_nifi with Apache License 2.0 | 4 votes |
public ComputeLineageRunnable(final RingBuffer<ProvenanceEventRecord> ringBuffer, final Filter<ProvenanceEventRecord> filter, final AsyncLineageSubmission submission) { this.ringBuffer = ringBuffer; this.filter = filter; this.submission = submission; }