Java Code Examples for org.apache.nifi.util.Tuple#getKey()

public void releaseWriteLock(final File file) {
    final String key = getMapKey(file);

    boolean updated = false;
    while (!updated) {
        final Tuple<ReadWriteLock, Integer> tuple = lockMap.get(key);
        if (tuple == null) {
            throw new IllegalMonitorStateException("Lock is not owned");
        }

        // If this is the only reference to the lock, remove it from the map and then unlock.
        if (tuple.getValue() <= 1) {
            updated = lockMap.remove(key, tuple);
            if (updated) {
                tuple.getKey().writeLock().unlock();
            }
        } else {
            final Tuple<ReadWriteLock, Integer> updatedTuple = new Tuple<>(tuple.getKey(), tuple.getValue() - 1);
            updated = lockMap.replace(key, tuple, updatedTuple);
            if (updated) {
                tuple.getKey().writeLock().unlock();
            }
        }
    }
}
 

public void releaseWriteLock(final File file) {
    final String key = getMapKey(file);

    boolean updated = false;
    while (!updated) {
        final Tuple<ReadWriteLock, Integer> tuple = lockMap.get(key);
        if (tuple == null) {
            throw new IllegalMonitorStateException("Lock is not owned");
        }

        // If this is the only reference to the lock, remove it from the map and then unlock.
        if (tuple.getValue() <= 1) {
            updated = lockMap.remove(key, tuple);
            if (updated) {
                tuple.getKey().writeLock().unlock();
            }
        } else {
            final Tuple<ReadWriteLock, Integer> updatedTuple = new Tuple<>(tuple.getKey(), tuple.getValue() - 1);
            updated = lockMap.replace(key, tuple, updatedTuple);
            if (updated) {
                tuple.getKey().writeLock().unlock();
            }
        }
    }
}
 

private Tuple<ProcessGroupStatus, String> getNextProcessGroupStatus() {
    Tuple<Iterator<ProcessGroupStatus>, String> top = iteratorBreadcrumb.peek();
    if (top == null) {
        return null;
    }
    Iterator<ProcessGroupStatus> i = top.getKey();
    String parentId = top.getValue();

    if (i.hasNext()) {
        ProcessGroupStatus nextPG = i.next();
        iteratorBreadcrumb.push(new Tuple<>(nextPG.getProcessGroupStatus().iterator(), nextPG.getId()));
        return new Tuple<>(nextPG, parentId);
    } else {
        // No more child PGs, remove it from the breadcrumb trail and try again
        Tuple<Iterator<ProcessGroupStatus>, String> pop = iteratorBreadcrumb.pop();
        return getNextProcessGroupStatus();
    }
}
 

public static Set<ConfigurableComponent> getRestrictedComponents(final VersionedProcessGroup group, final NiFiServiceFacade serviceFacade) {
    final Set<ConfigurableComponent> restrictedComponents = new HashSet<>();

    final Set<Tuple<String, BundleCoordinate>> componentTypes = new HashSet<>();
    populateComponentTypes(group, componentTypes);

    for (final Tuple<String, BundleCoordinate> tuple : componentTypes) {
        final ConfigurableComponent component = serviceFacade.getTempComponent(tuple.getKey(), tuple.getValue());
        if (component == null) {
            throw new NiFiCoreException("Could not create an instance of component " + tuple.getKey() + " using bundle coordinates " + tuple.getValue());
        }

        final boolean isRestricted = component.getClass().isAnnotationPresent(Restricted.class);
        if (isRestricted) {
            restrictedComponents.add(component);
        }
    }

    return restrictedComponents;
}
 

private ReadWriteLock updateCount(final File file, final Function<Integer, Integer> update) {
    final String key = getMapKey(file);
    boolean updated = false;

    Tuple<ReadWriteLock, Integer> updatedTuple = null;
    while (!updated) {
        final Tuple<ReadWriteLock, Integer> tuple = lockMap.computeIfAbsent(key, k -> new Tuple<>(new ReentrantReadWriteLock(), 0));
        final Integer updatedCount = update.apply(tuple.getValue());
        updatedTuple = new Tuple<>(tuple.getKey(), updatedCount);
        updated = lockMap.replace(key, tuple, updatedTuple);
    }

    return updatedTuple.getKey();
}
 

private ReadWriteLock updateCount(final File file, final Function<Integer, Integer> update) {
    final String key = getMapKey(file);
    boolean updated = false;

    Tuple<ReadWriteLock, Integer> updatedTuple = null;
    while (!updated) {
        final Tuple<ReadWriteLock, Integer> tuple = lockMap.computeIfAbsent(key, k -> new Tuple<>(new ReentrantReadWriteLock(), 0));
        final Integer updatedCount = update.apply(tuple.getValue());
        updatedTuple = new Tuple<>(tuple.getKey(), updatedCount);
        updated = lockMap.replace(key, tuple, updatedTuple);
    }

    return updatedTuple.getKey();
}
 

private SchemaVersionInfo getSchemaVersionInfo(final SchemaRegistryClient client, final SchemaVersionKey key) throws org.apache.nifi.schema.access.SchemaNotFoundException {
    try {
        // Try to fetch the SchemaVersionInfo from the cache.
        final Tuple<SchemaVersionInfo, Long> timestampedVersionInfo = schemaVersionByKeyCache.get(key);

        // Determine if the timestampedVersionInfo is expired
        boolean fetch = false;
        if (timestampedVersionInfo == null) {
            fetch = true;
        } else {
            final long minTimestamp = System.nanoTime() - versionInfoCacheNanos;
            fetch = timestampedVersionInfo.getValue() < minTimestamp;
        }

        // If not expired, use what we got from the cache
        if (!fetch) {
            return timestampedVersionInfo.getKey();
        }

        // schema version info was expired or not found in cache. Fetch from schema registry
        final SchemaVersionInfo versionInfo = client.getSchemaVersionInfo(key);
        if (versionInfo == null) {
            throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with name '" + key.getSchemaName() + "' and version " + key.getVersion());
        }

        // Store new version in cache.
        final Tuple<SchemaVersionInfo, Long> tuple = new Tuple<>(versionInfo, System.nanoTime());
        schemaVersionByKeyCache.put(key, tuple);
        return versionInfo;
    } catch (final SchemaNotFoundException e) {
        throw new org.apache.nifi.schema.access.SchemaNotFoundException(e);
    }
}
 

private SchemaVersionInfo getLatestSchemaVersionInfo(final SchemaRegistryClient client, final String schemaName, final String branchName)
        throws org.apache.nifi.schema.access.SchemaNotFoundException {
    try {
        // Try to fetch the SchemaVersionInfo from the cache.
        final Tuple<String,String> nameAndBranch = new Tuple<>(schemaName, branchName);
        final Tuple<SchemaVersionInfo, Long> timestampedVersionInfo = schemaVersionByNameCache.get(nameAndBranch);

        // Determine if the timestampedVersionInfo is expired
        boolean fetch = false;
        if (timestampedVersionInfo == null) {
            fetch = true;
        } else {
            final long minTimestamp = System.nanoTime() - versionInfoCacheNanos;
            fetch = timestampedVersionInfo.getValue() < minTimestamp;
        }

        // If not expired, use what we got from the cache
        if (!fetch) {
            return timestampedVersionInfo.getKey();
        }

        // schema version info was expired or not found in cache. Fetch from schema registry
        final SchemaVersionInfo versionInfo;
        if (StringUtils.isBlank(branchName)) {
            versionInfo = client.getLatestSchemaVersionInfo(schemaName);
        } else {
            versionInfo = client.getLatestSchemaVersionInfo(branchName, schemaName);
        }

        if (versionInfo == null) {
            throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with name '" + schemaName + "'");
        }

        // Store new version in cache.
        final Tuple<SchemaVersionInfo, Long> tuple = new Tuple<>(versionInfo, System.nanoTime());
        schemaVersionByNameCache.put(nameAndBranch, tuple);
        return versionInfo;
    } catch (final SchemaNotFoundException e) {
        throw new org.apache.nifi.schema.access.SchemaNotFoundException(e);
    }
}
 

@Override
public void processEvent(AnalysisContext analysisContext, NiFiFlow nifiFlow, ProvenanceEventRecord event) {
    if (!ProvenanceEventType.DROP.equals(event.getEventType())) {
        return;
    }
    final ComputeLineageResult lineage = analysisContext.queryLineage(event.getEventId());

    // Construct a tree model to traverse backwards.
    final Map<String, List<LineageNode>> lineageTree = new HashMap<>();
    analyzeLineageTree(lineage, lineageTree);

    final LineagePath lineagePath = new LineagePath();
    extractLineagePaths(analysisContext, lineageTree, lineagePath, event);

    analyzeLineagePath(analysisContext, lineagePath);

    // Input and output data set are both required to report lineage.
    List<Tuple<NiFiFlowPath, DataSetRefs>> createdFlowPaths = new ArrayList<>();
    if (lineagePath.isComplete()) {
        createCompleteFlowPath(nifiFlow, lineagePath, createdFlowPaths);
        for (Tuple<NiFiFlowPath, DataSetRefs> createdFlowPath : createdFlowPaths) {
            final NiFiFlowPath flowPath = createdFlowPath.getKey();
            // NOTE 1: FlowPath creation and DataSet references should be reported separately
            // ------------------------------------------------------------------------------
            // For example, with following provenance event inputs:
            //   CREATE(F1), FORK (F1 -> F2, F3), DROP(F1), SEND (F2), SEND(F3), DROP(F2), DROP(F3),
            // there is no guarantee that DROP(F2) and DROP(F3) are processed within the same cycle.
            // If DROP(F3) is processed in different cycle, it needs to be added to the existing FlowPath
            // that contains F1 -> F2, to be F1 -> F2, F3.
            // Execution cycle 1: Path1 (source of F1 -> ForkA), ForkA_queue (F1 -> F2), Path2 (ForkA -> dest of F2)
            // Execution cycle 2: Path1 (source of F1 -> ForkB), ForkB_queue (F1 -> F3), Path3 (ForkB -> dest of F3)

            // NOTE 2: Both FlowPath creation and FlowPath update messages are required
            // ------------------------------------------------------------------------
            // For the 1st time when a lineage is found, nifi_flow_path and referred DataSets are created.
            // If we notify these entities by a create 3 entities message (Path1, DataSet1, DataSet2)
            // followed by 1 partial update message to add lineage (DataSet1 -> Path1 -> DataSet2), then
            // the update message may arrive at Atlas earlier than the create message gets processed.
            // If that happens, lineage among these entities will be missed.
            // But as written in NOTE1, there is a case where existing nifi_flow_paths need to be updated.
            // Also, we don't know if this is the 1st time or 2nd or later.
            // So, we need to notify entity creation and also partial update messages.

            // Create flow path entity with DataSet refs.
            final Referenceable flowPathRef = toReferenceable(flowPath, nifiFlow);
            final DataSetRefs dataSetRefs = createdFlowPath.getValue();
            addDataSetRefs(dataSetRefs.getInputs(), flowPathRef, ATTR_INPUTS);
            addDataSetRefs(dataSetRefs.getOutputs(), flowPathRef, ATTR_OUTPUTS);
            createEntity(flowPathRef);
            // Also, sending partial update message to update existing flow_path.
            addDataSetRefs(nifiFlow, Collections.singleton(flowPath), createdFlowPath.getValue());

        }
        createdFlowPaths.clear();
    }
}