org.apache.nifi.provenance.search.SearchableField Java Examples

public static List<SearchableField> extractSearchableFields(final String indexedFieldString, final boolean predefinedField) {
    final List<SearchableField> searchableFields = new ArrayList<>();
    if (indexedFieldString != null) {
        final String[] split = indexedFieldString.split(",");
        for (String fieldName : split) {
            fieldName = fieldName.trim();
            if (fieldName.isEmpty()) {
                continue;
            }

            final SearchableField searchableField;
            if (predefinedField) {
                searchableField = SearchableFields.getSearchableField(fieldName);
            } else {
                searchableField = SearchableFields.newSearchableAttribute(fieldName);
            }

            if (searchableField == null) {
                throw new RuntimeException("Invalid Configuration: Provenance Repository configured to Index field '" + fieldName + "', but this is not a valid field");
            }
            searchableFields.add(searchableField);
        }
    }

    return searchableFields;
}
 

@Override
public boolean equals(final Object obj) {
    if (obj == this) {
        return true;
    }
    if (obj == null) {
        return false;
    }

    if (!(obj instanceof SearchableField)) {
        return false;
    }

    final SearchableField other = (SearchableField) obj;
    return attribute == other.isAttribute() && this.searchableName.equals(other.getSearchableFieldName());
}
 

public static List<SearchableField> extractSearchableFields(final String indexedFieldString, final boolean predefinedField) {
    final List<SearchableField> searchableFields = new ArrayList<>();
    if (indexedFieldString != null) {
        final String[] split = indexedFieldString.split(",");
        for (String fieldName : split) {
            fieldName = fieldName.trim();
            if (fieldName.isEmpty()) {
                continue;
            }

            final SearchableField searchableField;
            if (predefinedField) {
                searchableField = SearchableFields.getSearchableField(fieldName);
            } else {
                searchableField = SearchableFields.newSearchableAttribute(fieldName);
            }

            if (searchableField == null) {
                throw new RuntimeException("Invalid Configuration: Provenance Repository configured to Index field '" + fieldName + "', but this is not a valid field");
            }
            searchableFields.add(searchableField);
        }
    }

    return searchableFields;
}
 

@Override
public boolean equals(final Object obj) {
    if (obj == this) {
        return true;
    }
    if (obj == null) {
        return false;
    }

    if (!(obj instanceof SearchableField)) {
        return false;
    }

    final SearchableField other = (SearchableField) obj;
    return attribute == other.isAttribute() && this.searchableName.equals(other.getSearchableFieldName());
}
 

@Test
public void testSimpleQueryByComponentID() throws NiFiClientException, IOException, InterruptedException {
    final ProcessorEntity generateFlowFile = getClientUtil().createProcessor("GenerateFlowFile");
    final ProcessorEntity count = getClientUtil().createProcessor("CountEvents");
    getClientUtil().setAutoTerminatedRelationships(count, "success");
    getClientUtil().createConnection(generateFlowFile, count, "success");

    getNifiClient().getProcessorClient().startProcessor(generateFlowFile);
    getNifiClient().getProcessorClient().startProcessor(count);

    final Map<SearchableField, String> searchTerms = Collections.singletonMap(SearchableFields.ComponentID, generateFlowFile.getId());
    ProvenanceEntity provenanceEntity = getClientUtil().queryProvenance(searchTerms, null, null);
    assertEquals(0, provenanceEntity.getProvenance().getResults().getProvenanceEvents().size());

    // Wait for there to be at least 1 event.
    waitForEventCountAtLeast(searchTerms, 1);

    provenanceEntity = getClientUtil().queryProvenance(searchTerms, null, null);

    final List<ProvenanceEventDTO> events = provenanceEntity.getProvenance().getResults().getProvenanceEvents();
    assertEquals(1, events.size());

    final ProvenanceEventDTO firstEvent = events.get(0);
    assertEquals(ProvenanceEventType.CREATE.name(), firstEvent.getEventType());
}
 

private Object getFieldValue(final ProvenanceEventRecord record, final SearchableField field) {
    if (SearchableFields.AlternateIdentifierURI.equals(field)) {
        return record.getAlternateIdentifierUri();
    }
    if (SearchableFields.ComponentID.equals(field)) {
        return record.getComponentId();
    }
    if (SearchableFields.Details.equals(field)) {
        return record.getDetails();
    }
    if (SearchableFields.EventTime.equals(field)) {
        return record.getEventTime();
    }
    if (SearchableFields.EventType.equals(field)) {
        return record.getEventType();
    }
    if (SearchableFields.Filename.equals(field)) {
        return record.getAttributes().get(CoreAttributes.FILENAME.key());
    }
    if (SearchableFields.FileSize.equals(field)) {
        return record.getFileSize();
    }
    if (SearchableFields.FlowFileUUID.equals(field)) {
        return record.getFlowFileUuid();
    }
    if (SearchableFields.LineageStartDate.equals(field)) {
        return record.getLineageStartDate();
    }
    if (SearchableFields.Relationship.equals(field)) {
        return record.getRelationship();
    }
    if (SearchableFields.TransitURI.equals(field)) {
        return record.getTransitUri();
    }

    return null;
}
 

private void waitForEventCount(final Map<SearchableField, String> searchTerms, final Predicate<Integer> predicate) throws InterruptedException {
    // Wait for there to be at least 1000 events for Generate processor
    waitFor(() -> {
        try {
            return predicate.test(getEventCount(searchTerms));
        } catch (final Exception e) {
            return false;
        }
    }, 500L);
}
 

@Test
public void testAgeOffEventsThenAddSomeThenQuery() throws NiFiClientException, IOException, InterruptedException {
    ProcessorEntity generateFlowFile = getClientUtil().createProcessor("GenerateFlowFile");
    generateFlowFile = getClientUtil().updateProcessorProperties(generateFlowFile, Collections.singletonMap("Batch Size", "800"));

    final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile");
    getClientUtil().setAutoTerminatedRelationships(terminate, "success");
    getClientUtil().createConnection(generateFlowFile, terminate, "success");

    generateFlowFile = getNifiClient().getProcessorClient().startProcessor(generateFlowFile);

    final Map<SearchableField, String> generateSearchTerms = Collections.singletonMap(SearchableFields.ComponentID, generateFlowFile.getId());

    // Wait for there to be at least 1000 events for Generate processor and then stop the processor
    waitForEventCountAtLeast(generateSearchTerms, 800);
    getNifiClient().getProcessorClient().stopProcessor(generateFlowFile);

    // Start Terminate proc & wait for at least 600 events to be registered. We do this because each Event File can hold up to 1,000 Events.
    // The GenerateFlowFile would have 800. The first 200 events from Terminate will be in the first Event File, causing that one to
    // roll over and subsequently be aged off. The second Event File will hold the other 600. So we may have 600 or 800 events,
    // depending on when the query is executed.
    getNifiClient().getProcessorClient().startProcessor(terminate);
    final Map<SearchableField, String> terminateSearchTerms = Collections.singletonMap(SearchableFields.ComponentID, terminate.getId());
    waitForEventCountAtLeast(terminateSearchTerms, 600);

    waitForEventCountExactly(generateSearchTerms, 0);

    // Emit 25 more events
    getClientUtil().updateProcessorProperties(generateFlowFile, Collections.singletonMap("Batch Size", "25"));
    getNifiClient().getProcessorClient().startProcessor(generateFlowFile);

    // Wait for those 25 events to be emitted
    waitForEventCountAtLeast(generateSearchTerms, 25);
}
 

public ProvenanceEntity queryProvenance(final Map<SearchableField, String> searchTerms, final Long startTime, final Long endTime) throws NiFiClientException, IOException {
    final Map<String, String> searchTermsAsStrings = searchTerms.entrySet().stream()
        .collect(Collectors.toMap(entry -> entry.getKey().getSearchableFieldName(), Map.Entry::getValue));

    final ProvenanceRequestDTO requestDto = new ProvenanceRequestDTO();
    requestDto.setSearchTerms(searchTermsAsStrings);
    requestDto.setSummarize(false);
    requestDto.setStartDate(startTime == null ? null : new Date(startTime));
    requestDto.setEndDate(endTime == null ? null : new Date(endTime));
    requestDto.setMaxResults(1000);

    final ProvenanceDTO dto = new ProvenanceDTO();
    dto.setRequest(requestDto);
    dto.setSubmissionTime(new Date());

    final ProvenanceEntity entity = new ProvenanceEntity();
    entity.setProvenance(dto);

    ProvenanceEntity responseEntity = nifiClient.getProvenanceClient().submitProvenanceQuery(entity);

    try {
        responseEntity = waitForComplete(responseEntity);
    } catch (final InterruptedException ie) {
        Assert.fail("Interrupted while waiting for Provenance Query to complete");
    }

    nifiClient.getProvenanceClient().deleteProvenanceQuery(responseEntity.getProvenance().getId());
    return responseEntity;
}
 

/**
 * Gets the available options for searching provenance.
 *
 * @return the available options for searching provenance
 */
public ProvenanceOptionsDTO getProvenanceSearchOptions() {
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();

    // create the search options dto
    final ProvenanceOptionsDTO searchOptions = new ProvenanceOptionsDTO();
    final List<ProvenanceSearchableFieldDTO> searchableFieldNames = new ArrayList<>();
    final List<SearchableField> fields = provenanceRepository.getSearchableFields();
    for (final SearchableField field : fields) {
        // we exclude the Event Time because it is always searchable but don't want support querying it this way...
        // we prefer the user queries using startDate and endDate
        if (SearchableFields.EventTime.equals(field)) {
            continue;
        }

        final ProvenanceSearchableFieldDTO searchableField = new ProvenanceSearchableFieldDTO();
        searchableField.setId(field.getIdentifier());
        searchableField.setField(field.getSearchableFieldName());
        searchableField.setLabel(field.getFriendlyName());
        searchableField.setType(field.getFieldType().name());
        searchableFieldNames.add(searchableField);
    }
    final List<SearchableField> searchableAttributes = provenanceRepository.getSearchableAttributes();
    for (final SearchableField searchableAttr : searchableAttributes) {
        final ProvenanceSearchableFieldDTO searchableAttribute = new ProvenanceSearchableFieldDTO();
        searchableAttribute.setId(searchableAttr.getIdentifier());
        searchableAttribute.setField(searchableAttr.getSearchableFieldName());
        searchableAttribute.setLabel(searchableAttr.getFriendlyName());
        searchableAttribute.setType(searchableAttr.getFieldType().name());
        searchableFieldNames.add(searchableAttribute);
    }
    searchOptions.setSearchableFields(searchableFieldNames);
    return searchOptions;
}
 

@Override
public List<SearchableField> getSearchableFields() {
    final List<SearchableField> searchableFields = new ArrayList<>(configuration.getSearchableFields());
    // we exclude the Event Time because it is always searchable and is a bit special in its handling
    // because it dictates in some cases which index files we look at
    searchableFields.remove(SearchableFields.EventTime);
    return searchableFields;
}
 

private void addField(final Document doc, final SearchableField field, final String value, final Store store) {
    if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) {
        return;
    }

    doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), store));
}
 

private void addField(final Document doc, final SearchableField field, final String value) {
    if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) {
        return;
    }

    doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), Store.NO));
}
 

private void addField(final Document doc, final SearchableField field, final String value, final Store store) {
    if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) {
        return;
    }

    doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), store));
}
 

private void addField(final Document doc, final SearchableField field, final String value) {
    if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) {
        return;
    }

    doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), Store.NO));
}
 

/**
 * Gets the available options for searching provenance.
 *
 * @return the available options for searching provenance
 */
public ProvenanceOptionsDTO getProvenanceSearchOptions() {
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();

    // create the search options dto
    final ProvenanceOptionsDTO searchOptions = new ProvenanceOptionsDTO();
    final List<ProvenanceSearchableFieldDTO> searchableFieldNames = new ArrayList<>();
    final List<SearchableField> fields = provenanceRepository.getSearchableFields();
    for (final SearchableField field : fields) {
        final ProvenanceSearchableFieldDTO searchableField = new ProvenanceSearchableFieldDTO();
        searchableField.setId(field.getIdentifier());
        searchableField.setField(field.getSearchableFieldName());
        searchableField.setLabel(field.getFriendlyName());
        searchableField.setType(field.getFieldType().name());
        searchableFieldNames.add(searchableField);
    }
    final List<SearchableField> searchableAttributes = provenanceRepository.getSearchableAttributes();
    for (final SearchableField searchableAttr : searchableAttributes) {
        final ProvenanceSearchableFieldDTO searchableAttribute = new ProvenanceSearchableFieldDTO();
        searchableAttribute.setId(searchableAttr.getIdentifier());
        searchableAttribute.setField(searchableAttr.getSearchableFieldName());
        searchableAttribute.setLabel(searchableAttr.getFriendlyName());
        searchableAttribute.setType(searchableAttr.getFieldType().name());
        searchableFieldNames.add(searchableAttribute);
    }
    searchOptions.setSearchableFields(searchableFieldNames);
    return searchOptions;
}
 

private Object getFieldValue(final ProvenanceEventRecord record, final SearchableField field) {
    if (SearchableFields.AlternateIdentifierURI.equals(field)) {
        return record.getAlternateIdentifierUri();
    }
    if (SearchableFields.ComponentID.equals(field)) {
        return record.getComponentId();
    }
    if (SearchableFields.Details.equals(field)) {
        return record.getDetails();
    }
    if (SearchableFields.EventTime.equals(field)) {
        return record.getEventTime();
    }
    if (SearchableFields.EventType.equals(field)) {
        return record.getEventType();
    }
    if (SearchableFields.Filename.equals(field)) {
        return record.getAttributes().get(CoreAttributes.FILENAME.key());
    }
    if (SearchableFields.FileSize.equals(field)) {
        return record.getFileSize();
    }
    if (SearchableFields.FlowFileUUID.equals(field)) {
        return record.getFlowFileUuid();
    }
    if (SearchableFields.LineageStartDate.equals(field)) {
        return record.getLineageStartDate();
    }
    if (SearchableFields.Relationship.equals(field)) {
        return record.getRelationship();
    }
    if (SearchableFields.TransitURI.equals(field)) {
        return record.getTransitUri();
    }

    return null;
}
 

@Test
public void testAddToMultipleLogsAndRecover() throws IOException, InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final List<SearchableField> searchableFields = new ArrayList<>();
    searchableFields.add(SearchableFields.ComponentID);

    final RepositoryConfiguration config = createConfiguration();
    config.setMaxEventFileCapacity(1024L * 1024L);
    config.setMaxEventFileLife(2, TimeUnit.SECONDS);
    config.setSearchableFields(searchableFields);
    repo = new PersistentProvenanceRepository(config, DEFAULT_ROLLOVER_MILLIS);
    repo.initialize(getEventReporter(), null, null, IdentifierLookup.EMPTY);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("abc", "xyz");
    attributes.put("xyz", "abc");
    attributes.put("uuid", UUID.randomUUID().toString());

    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    final ProvenanceEventRecord record = builder.build();

    for (int i = 0; i < 10; i++) {
        repo.registerEvent(record);
    }

    builder.setComponentId("XXXX"); // create a different component id so that we can make sure we query this record.

    attributes.put("uuid", "11111111-1111-1111-1111-111111111111");

    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());

    repo.waitForRollover();
    Thread.sleep(500L); // Give the repo time to shutdown (i.e., close all file handles, etc.)

    // Create a new repo and add another record with component id XXXX so that we can ensure that it's added to a different
    // log file than the previous one.
    attributes.put("uuid", "22222222-2222-2222-2222-222222222222");
    builder.fromFlowFile(createFlowFile(11L, 11L, attributes));
    repo.registerEvent(builder.build());
    repo.waitForRollover();

    final Query query = new Query(UUID.randomUUID().toString());
    query.addSearchTerm(SearchTerms.newSearchTerm(SearchableFields.ComponentID, "XXXX"));
    query.setMaxResults(100);

    final QueryResult result = repo.queryEvents(query, createUser());
    assertEquals(2, result.getMatchingEvents().size());
    for (final ProvenanceEventRecord match : result.getMatchingEvents()) {
        System.out.println(match);
    }
}
 

public IndexingAction(final List<SearchableField> searchableEventFields, final List<SearchableField> searchableAttributes) {
    this.searchableEventFields = Collections.unmodifiableSet(new HashSet<>(searchableEventFields));
    this.searchableAttributeFields = Collections.unmodifiableSet(new HashSet<>(searchableAttributes));
}
 

/**
 * Submits a provenance query.
 *
 * @param provenanceDto dto
 * @return provenance info
 */
public ProvenanceDTO submitProvenance(ProvenanceDTO provenanceDto) {
    final ProvenanceRequestDTO requestDto = provenanceDto.getRequest();

    // create the query
    final Query query = new Query(provenanceDto.getId());

    // if the request was specified
    if (requestDto != null) {
        // add each search term specified
        final Map<String, String> searchTerms = requestDto.getSearchTerms();
        if (searchTerms != null) {
            for (final Map.Entry<String, String> searchTerm : searchTerms.entrySet()) {
                SearchableField field;

                field = SearchableFields.getSearchableField(searchTerm.getKey());
                if (field == null) {
                    field = SearchableFields.newSearchableAttribute(searchTerm.getKey());
                }
                query.addSearchTerm(SearchTerms.newSearchTerm(field, searchTerm.getValue()));
            }
        }

        // specify the start date if specified
        if (requestDto.getStartDate() != null) {
            query.setStartDate(requestDto.getStartDate());
        }

        // ensure an end date is populated
        if (requestDto.getEndDate() != null) {
            query.setEndDate(requestDto.getEndDate());
        }

        // set the min/max file size
        query.setMinFileSize(requestDto.getMinimumFileSize());
        query.setMaxFileSize(requestDto.getMaximumFileSize());

        // set the max results desired
        query.setMaxResults(requestDto.getMaxResults());
    }

    // submit the query to the provenance repository
    final ProvenanceRepository provenanceRepository = flowController.getProvenanceRepository();
    final QuerySubmission querySubmission = provenanceRepository.submitQuery(query, NiFiUserUtils.getNiFiUser());

    // return the query with the results populated at this point
    return getProvenanceQuery(querySubmission.getQueryIdentifier(), requestDto.getSummarize(), requestDto.getIncrementalResults());
}
 

@Override
public List<SearchableField> getSearchableAttributes() {
    return new ArrayList<>(configuration.getSearchableAttributes());
}
 

@Override
public List<SearchableField> getSearchableFields() {
    return new ArrayList<>(configuration.getSearchableFields());
}
 

public static RepositoryConfiguration create(final NiFiProperties nifiProperties) {
    final Map<String, Path> storageDirectories = nifiProperties.getProvenanceRepositoryPaths();
    if (storageDirectories.isEmpty()) {
        storageDirectories.put("provenance_repository", Paths.get("provenance_repository"));
    }
    final String storageTime = nifiProperties.getProperty(NiFiProperties.PROVENANCE_MAX_STORAGE_TIME, "24 hours");
    final String storageSize = nifiProperties.getProperty(NiFiProperties.PROVENANCE_MAX_STORAGE_SIZE, "1 GB");
    final String rolloverTime = nifiProperties.getProperty(NiFiProperties.PROVENANCE_ROLLOVER_TIME, "5 mins");
    final String rolloverSize = nifiProperties.getProperty(NiFiProperties.PROVENANCE_ROLLOVER_SIZE, "100 MB");
    final int rolloverEventCount = nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_ROLLOVER_EVENT_COUNT, Integer.MAX_VALUE);
    final String shardSize = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEX_SHARD_SIZE, "500 MB");
    final int queryThreads = nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_QUERY_THREAD_POOL_SIZE, 2);
    final int indexThreads = nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_INDEX_THREAD_POOL_SIZE, 2);
    final int journalCount = nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_JOURNAL_COUNT, 16);
    final int concurrentMergeThreads = nifiProperties.getIntegerProperty(CONCURRENT_MERGE_THREADS, 2);
    final String warmCacheFrequency = nifiProperties.getProperty(WARM_CACHE_FREQUENCY);
    final String maintenanceFrequency = nifiProperties.getProperty(MAINTENACE_FREQUENCY);
    final long storageMillis = FormatUtils.getTimeDuration(storageTime, TimeUnit.MILLISECONDS);
    final long maxStorageBytes = DataUnit.parseDataSize(storageSize, DataUnit.B).longValue();
    final long rolloverMillis = FormatUtils.getTimeDuration(rolloverTime, TimeUnit.MILLISECONDS);
    final long rolloverBytes = DataUnit.parseDataSize(rolloverSize, DataUnit.B).longValue();

    final boolean compressOnRollover = Boolean.parseBoolean(nifiProperties.getProperty(NiFiProperties.PROVENANCE_COMPRESS_ON_ROLLOVER));
    final String indexedFieldString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_FIELDS);
    final String indexedAttrString = nifiProperties.getProperty(NiFiProperties.PROVENANCE_INDEXED_ATTRIBUTES);

    final Boolean alwaysSync = Boolean.parseBoolean(nifiProperties.getProperty("nifi.provenance.repository.always.sync", "false"));

    final int defaultMaxAttrChars = 65536;
    final String maxAttrLength = nifiProperties.getProperty("nifi.provenance.repository.max.attribute.length", String.valueOf(defaultMaxAttrChars));
    int maxAttrChars;
    try {
        maxAttrChars = Integer.parseInt(maxAttrLength);
        // must be at least 36 characters because that's the length of the uuid attribute,
        // which must be kept intact
        if (maxAttrChars < 36) {
            maxAttrChars = 36;
            logger.warn("Found max attribute length property set to " + maxAttrLength + " but minimum length is 36; using 36 instead");
        }
    } catch (final Exception e) {
        maxAttrChars = defaultMaxAttrChars;
    }

    final List<SearchableField> searchableFields = SearchableFieldParser.extractSearchableFields(indexedFieldString, true);
    final List<SearchableField> searchableAttributes = SearchableFieldParser.extractSearchableFields(indexedAttrString, false);

    // We always want to index the Event Time.
    if (!searchableFields.contains(SearchableFields.EventTime)) {
        searchableFields.add(SearchableFields.EventTime);
    }

    final RepositoryConfiguration config = new RepositoryConfiguration();
    for (final Map.Entry<String, Path> entry : storageDirectories.entrySet()) {
        config.addStorageDirectory(entry.getKey(), entry.getValue().toFile());
    }
    config.setCompressOnRollover(compressOnRollover);
    config.setSearchableFields(searchableFields);
    config.setSearchableAttributes(searchableAttributes);
    config.setMaxEventFileCapacity(rolloverBytes);
    config.setMaxEventFileCount(rolloverEventCount);
    config.setMaxEventFileLife(rolloverMillis, TimeUnit.MILLISECONDS);
    config.setMaxRecordLife(storageMillis, TimeUnit.MILLISECONDS);
    config.setMaxStorageCapacity(maxStorageBytes);
    config.setQueryThreadPoolSize(queryThreads);
    config.setIndexThreadPoolSize(indexThreads);
    config.setJournalCount(journalCount);
    config.setMaxAttributeChars(maxAttrChars);
    config.setConcurrentMergeThreads(concurrentMergeThreads);

    if (warmCacheFrequency != null && !warmCacheFrequency.trim().equals("")) {
        config.setWarmCacheFrequencyMinutes((int) FormatUtils.getTimeDuration(warmCacheFrequency, TimeUnit.MINUTES));
    }
    if (shardSize != null) {
        config.setDesiredIndexSize(DataUnit.parseDataSize(shardSize, DataUnit.B).longValue());
    }
    if (maintenanceFrequency != null && !maintenanceFrequency.trim().equals("")) {
        final long millis = FormatUtils.getTimeDuration(maintenanceFrequency.trim(), TimeUnit.MILLISECONDS);
        config.setMaintenanceFrequency(millis, TimeUnit.MILLISECONDS);
    }

    config.setAlwaysSync(alwaysSync);

    config.setDebugFrequency(nifiProperties.getIntegerProperty(NiFiProperties.PROVENANCE_REPO_DEBUG_FREQUENCY, config.getDebugFrequency()));

    // TODO: Check for multiple key loading (NIFI-6617)
    // Encryption values may not be present but are only required for EncryptedWriteAheadProvenanceRepository
    final String implementationClassName = nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS);
    if (EncryptedWriteAheadProvenanceRepository.class.getName().equals(implementationClassName)) {
        config.setEncryptionKeys(nifiProperties.getProvenanceRepoEncryptionKeys());
        config.setKeyId(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_ID));
        config.setKeyProviderImplementation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_IMPLEMENTATION_CLASS));
        config.setKeyProviderLocation(nifiProperties.getProperty(NiFiProperties.PROVENANCE_REPO_ENCRYPTION_KEY_PROVIDER_LOCATION));
    }

    return config;
}
 

/**
 * @param searchableAttributes the FlowFile attributes to index
 */
public void setSearchableAttributes(final List<SearchableField> searchableAttributes) {
    this.searchableAttributes = new ArrayList<>(searchableAttributes);
}
 

/**
 * @return the FlowFile attributes that should be indexed
 */
public List<SearchableField> getSearchableAttributes() {
    return Collections.unmodifiableList(searchableAttributes);
}
 

/**
 * @param searchableFields the fields to index
 */
public void setSearchableFields(final List<SearchableField> searchableFields) {
    this.searchableFields = new ArrayList<>(searchableFields);
}
 

/**
 * @return the fields that should be indexed
 */
public List<SearchableField> getSearchableFields() {
    return Collections.unmodifiableList(searchableFields);
}
 

private int getEventCount(final Map<SearchableField, String> searchTerms) throws NiFiClientException, IOException {
    ProvenanceEntity provEntity = getClientUtil().queryProvenance(searchTerms, null, null);
    return provEntity.getProvenance().getResults().getProvenanceEvents().size();
}
 

@Override
public List<SearchableField> getSearchableAttributes() {
    return Collections.emptyList();
}
 

public static SearchableField newSearchableAttribute(final String attributeName) {
    return new NamedSearchableField(attributeName, attributeName, attributeName, true);
}