Java Code Examples for org.apache.nifi.provenance.ProvenanceEventRecord#getComponentId()

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {
    if (event.getTransitUri() == null) {
        return null;
    }

    final URI uri = parseUri(event.getTransitUri());
    final String namespace = context.getNamespaceResolver().fromHostNames(uri.getHost());
    final Set<Tuple<String, String>> outputTables = parseTableNames(null, event.getAttribute(ATTR_OUTPUT_TABLES));
    if (outputTables.isEmpty()) {
        return null;
    }

    final DataSetRefs refs = new DataSetRefs(event.getComponentId());
    outputTables.forEach(tableName -> {
        final Referenceable ref = createTableRef(namespace, tableName);
        refs.addOutput(ref);
    });
    return refs;
}
 

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    if (!ProvenanceEventType.SEND.equals(event.getEventType())
            && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) {
        return null;
    }

    final boolean isInputPort = event.getComponentType().equals("Input Port");
    final String type = isInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT;
    final String rootPortId = event.getComponentId();

    final S2SPort s2SPort = analyzeS2SPort(event, context.getNamespaceResolver());

    // Find connections connecting to/from the remote port.
    final List<ConnectionStatus> connections = isInputPort
            ? context.findConnectionFrom(rootPortId)
            : context.findConnectionTo(rootPortId);
    if (connections == null || connections.isEmpty()) {
        logger.warn("Connection was not found: {}", new Object[]{event});
        return null;
    }

    // The name of the port can be retrieved from any connection, use the first one.
    final ConnectionStatus connection = connections.get(0);
    final Referenceable ref = new Referenceable(type);
    ref.set(ATTR_NAME, isInputPort ? connection.getSourceName() : connection.getDestinationName());
    ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.namespace, rootPortId));

    return singleDataSetRef(event.getComponentId(), event.getEventType(), ref);
}
 

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    // Check if this component is a processor that generates data.
    final String componentId = event.getComponentId();
    final List<ConnectionStatus> incomingConnections = context.findConnectionTo(componentId);
    if (incomingConnections != null && !incomingConnections.isEmpty()) {
        return null;
    }

    return super.analyze(context, event);
}
 

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    final String componentId = event.getComponentId();
    final DataSetRefs refs = new DataSetRefs(componentId);
    final Referenceable ref = createDataSetRef(context, event);
    refs.addOutput(ref);

    return refs;
}
 

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    final String componentId = event.getComponentId();
    final DataSetRefs refs = new DataSetRefs(componentId);
    final Referenceable ref = createDataSetRef(context, event);
    refs.addInput(ref);

    return refs;
}
 

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    if (!ProvenanceEventType.SEND.equals(event.getEventType())
            && !ProvenanceEventType.RECEIVE.equals(event.getEventType())) {
        return null;
    }

    final boolean isRemoteInputPort = event.getComponentType().equals("Remote Input Port");
    final String type = isRemoteInputPort ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT;

    final S2SPort s2SPort = analyzeS2SPort(event, context.getNamespaceResolver());

    // Find connections that connects to/from the remote port.
    final String componentId = event.getComponentId();
    final List<ConnectionStatus> connections = isRemoteInputPort
            ? context.findConnectionTo(componentId)
            : context.findConnectionFrom(componentId);
    if (connections == null || connections.isEmpty()) {
        logger.warn("Connection was not found: {}", new Object[]{event});
        return null;
    }

    // The name of remote port can be retrieved from any connection, use the first one.
    final ConnectionStatus connection = connections.get(0);
    final Referenceable ref = new Referenceable(type);
    ref.set(ATTR_NAME, isRemoteInputPort ? connection.getDestinationName() : connection.getSourceName());
    ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(s2SPort.namespace, s2SPort.targetPortId));

    return singleDataSetRef(event.getComponentId(), event.getEventType(), ref);
}
 

@Override
public void update(final ProvenanceEventRecord event, final StorageSummary storageSummary) {
    final String componentId = event.getComponentId();
    final RingBuffer<Long> ringBuffer = latestRecords.computeIfAbsent(componentId, id -> new RingBuffer<>(1000));
    ringBuffer.add(storageSummary.getEventId());
}
 

@Override
public void update(final ProvenanceEventRecord event, final StorageSummary storageSummary) {
    final String componentId = event.getComponentId();
    final RingBuffer<Long> ringBuffer = latestRecords.computeIfAbsent(componentId, id -> new RingBuffer<>(1000));
    ringBuffer.add(storageSummary.getEventId());
}
 

@Override
protected String getRawProtocolPortId(ProvenanceEventRecord event) {
    return event.getComponentId();
}
 

@Override
public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) {

    // Replace the colon so that the schema in the URI can be parsed correctly.
    final String transitUri = event.getTransitUri();
    if (transitUri == null) {
        return null;
    }

    final Matcher uriMatcher = URI_PATTERN.matcher(transitUri);
    if (!uriMatcher.matches()) {
        logger.warn("Unexpected transit URI: {}", new Object[]{transitUri});
        return null;
    }

    final String namespace = context.getNamespaceResolver().fromHostNames(splitHostNames(uriMatcher.group(1)));
    String connectedDatabaseName = null;
    if (uriMatcher.groupCount() > 1) {
        // Try to find connected database name from connection parameters.
        final String[] connectionParams = uriMatcher.group(2).split(";");
        connectedDatabaseName = connectionParams[0];
    }

    if (StringUtils.isEmpty(connectedDatabaseName)) {
        // If not found, then use "default".
        connectedDatabaseName = "default";
    }

    final Set<Tuple<String, String>> inputTables = parseTableNames(connectedDatabaseName, event.getAttribute(ATTR_INPUT_TABLES));
    final Set<Tuple<String, String>> outputTables = parseTableNames(connectedDatabaseName, event.getAttribute(ATTR_OUTPUT_TABLES));

    if (inputTables.isEmpty() && outputTables.isEmpty()) {
        // If input/output tables are unknown, create database level lineage.
        // Handle case insensitivity of database and table names in Hive: send names uniformly in lower case
        return getDatabaseRef(event.getComponentId(), event.getEventType(), namespace, connectedDatabaseName.toLowerCase());
    }

    final DataSetRefs refs = new DataSetRefs(event.getComponentId());
    addRefs(refs, true, namespace, inputTables);
    addRefs(refs, false, namespace, outputTables);
    return refs;
}