Java Code Examples for org.apache.nifi.registry.flow.VersionedFlowSnapshot#getFlowContents()

@Test
public void testDeserializeVer3() throws IOException {
    final String file = "/serialization/ver3.snapshot";
    try (final InputStream is = this.getClass().getResourceAsStream(file)) {
        final Integer version = serializer.readDataModelVersion(is);
        assertNotNull(version);
        assertEquals(3, version.intValue());
        assertFalse(serializer.isProcessGroupVersion(version));

        final FlowContent flowContent = serializer.deserializeFlowContent(version, is);
        assertNotNull(flowContent);

        final VersionedFlowSnapshot flowSnapshot = flowContent.getFlowSnapshot();
        assertNotNull(flowSnapshot);

        final VersionedProcessGroup processGroup = flowSnapshot.getFlowContents();
        assertNotNull(processGroup);
        assertNotNull(processGroup.getProcessors());
        assertEquals(1, processGroup.getProcessors().size());
    }
}
 

public static void assertFlowSnapshotsEqual(VersionedFlowSnapshot expected, VersionedFlowSnapshot actual, boolean checkServerSetFields) {

        assertNotNull(actual);

        if (expected.getSnapshotMetadata() != null) {
            assertFlowSnapshotMetadataEqual(expected.getSnapshotMetadata(), actual.getSnapshotMetadata(), checkServerSetFields);
        }

        if (expected.getFlowContents() != null) {
            assertVersionedProcessGroupsEqual(expected.getFlowContents(), actual.getFlowContents());
        }

        if (checkServerSetFields) {
            assertFlowsEqual(expected.getFlow(), actual.getFlow(), false); // false because if we are checking a newly created snapshot, the versionsCount won't match
            assertBucketsEqual(expected.getBucket(), actual.getBucket(), true);
        }

    }
 

/**
 * Authorize read/write permissions for the given user on every component of the given flow in support of flow update.
 *
 * @param lookup        A lookup instance to use for retrieving components for authorization purposes
 * @param user          the user to authorize
 * @param groupId       the id of the process group being evaluated
 * @param flowSnapshot  the new flow contents to examine for restricted components
 */
protected void authorizeFlowUpdate(final AuthorizableLookup lookup, final NiFiUser user, final String groupId,
                                   final VersionedFlowSnapshot flowSnapshot) {
    // Step 2: Verify READ and WRITE permissions for user, for every component.
    final ProcessGroupAuthorizable groupAuthorizable = lookup.getProcessGroup(groupId);
    authorizeProcessGroup(groupAuthorizable, authorizer, lookup, RequestAction.READ, true,
            false, true, true, true);
    authorizeProcessGroup(groupAuthorizable, authorizer, lookup, RequestAction.WRITE, true,
            false, true, true, false);

    final VersionedProcessGroup groupContents = flowSnapshot.getFlowContents();
    final Set<ConfigurableComponent> restrictedComponents = FlowRegistryUtils.getRestrictedComponents(groupContents, serviceFacade);
    restrictedComponents.forEach(restrictedComponent -> {
        final ComponentAuthorizable restrictedComponentAuthorizable = lookup.getConfigurableComponent(restrictedComponent);
        authorizeRestrictions(authorizer, restrictedComponentAuthorizable);
    });

    final Map<String, VersionedParameterContext> parameterContexts = flowSnapshot.getParameterContexts();
    if (parameterContexts != null) {
        parameterContexts.values().forEach(
                context -> AuthorizeParameterReference.authorizeParameterContextAddition(context, serviceFacade, authorizer, lookup, user)
        );
    }
}
 

private Set<FlowDifference> getLocalModifications(final ProcessGroup processGroup, final VersionedFlowSnapshot versionedFlowSnapshot) {
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(getFlowController().getExtensionManager());
    final VersionedProcessGroup localGroup = mapper.mapProcessGroup(processGroup, getFlowController().getControllerServiceProvider(), getFlowController().getFlowRegistryClient(), true);
    final VersionedProcessGroup registryGroup = versionedFlowSnapshot.getFlowContents();

    final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localGroup);
    final ComparableDataFlow registryFlow = new StandardComparableDataFlow("Versioned Flow", registryGroup);

    final Set<String> ancestorServiceIds = processGroup.getAncestorServiceIds();
    final FlowComparator flowComparator = new StandardFlowComparator(registryFlow, localFlow, ancestorServiceIds, new ConciseEvolvingDifferenceDescriptor());
    final FlowComparison flowComparison = flowComparator.compare();
    final Set<FlowDifference> differences = flowComparison.getDifferences().stream()
        .filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED)
        .filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS)
        .filter(FlowDifferenceFilters.FILTER_PUBLIC_PORT_NAME_CHANGES)
        .filter(FlowDifferenceFilters.FILTER_IGNORABLE_VERSIONED_FLOW_COORDINATE_CHANGES)
        .collect(Collectors.toCollection(HashSet::new));

    return differences;
}
 

@Override
public ConfigSchema apply(final VersionedFlowSnapshot versionedFlowSnapshot) {
    Map<String, Object> map = new HashMap<>();
    map.put(CommonPropertyKeys.FLOW_CONTROLLER_PROPS_KEY, flowControllerSchemaFunction.apply(versionedFlowSnapshot).toMap());

    VersionedProcessGroup versionedProcessGroup = versionedFlowSnapshot.getFlowContents();
    addVersionedProcessGroup(map, versionedProcessGroup);

    return new ConfigSchema(map);
}
 

public VersionedFlowSnapshot getFlowContents(final String bucketId, final String flowId, final int version, final boolean fetchRemoteFlows, final NiFiUser user)
    throws IOException, NiFiRegistryException {

    final FlowSnapshotClient snapshotClient = getFlowSnapshotClient(user);
    final VersionedFlowSnapshot flowSnapshot = snapshotClient.get(bucketId, flowId, version);

    if (fetchRemoteFlows) {
        final VersionedProcessGroup contents = flowSnapshot.getFlowContents();
        for (final VersionedProcessGroup child : contents.getProcessGroups()) {
            populateVersionedContentsRecursively(child, user);
        }
    }

    return flowSnapshot;
}
 

private void populateVersionedContentsRecursively(final VersionedProcessGroup group, final NiFiUser user) throws NiFiRegistryException, IOException {
    if (group == null) {
        return;
    }

    final VersionedFlowCoordinates coordinates = group.getVersionedFlowCoordinates();
    if (coordinates != null) {
        final String registryUrl = coordinates.getRegistryUrl();
        final String bucketId = coordinates.getBucketId();
        final String flowId = coordinates.getFlowId();
        final int version = coordinates.getVersion();

        final RegistryUtil subFlowUtil = new RegistryUtil(registryUrl, sslContext);
        final VersionedFlowSnapshot snapshot = subFlowUtil.getFlowByID(bucketId, flowId, version);
        final VersionedProcessGroup contents = snapshot.getFlowContents();

        group.setComments(contents.getComments());
        group.setConnections(contents.getConnections());
        group.setControllerServices(contents.getControllerServices());
        group.setFunnels(contents.getFunnels());
        group.setInputPorts(contents.getInputPorts());
        group.setLabels(contents.getLabels());
        group.setOutputPorts(contents.getOutputPorts());
        group.setProcessGroups(contents.getProcessGroups());
        group.setProcessors(contents.getProcessors());
        group.setRemoteProcessGroups(contents.getRemoteProcessGroups());
        group.setVariables(contents.getVariables());
        coordinates.setLatest(snapshot.isLatest());
    }

    for (final VersionedProcessGroup child : group.getProcessGroups()) {
        populateVersionedContentsRecursively(child, user);
    }
}
 

/**
 * Returns the differences between two specified versions of a flow.
 *
 * @param bucketIdentifier the id of the bucket the flow exists in
 * @param flowIdentifier the flow to be examined
 * @param versionA the first version of the comparison
 * @param versionB the second version of the comparison
 * @return The differences between two specified versions, grouped by component.
 */
public VersionedFlowDifference getFlowDiff(final String bucketIdentifier, final String flowIdentifier,
                                           final Integer versionA, final Integer versionB) {
    if (StringUtils.isBlank(bucketIdentifier)) {
        throw new IllegalArgumentException("Bucket identifier cannot be null or blank");
    }

    if (StringUtils.isBlank(flowIdentifier)) {
        throw new IllegalArgumentException("Flow identifier cannot be null or blank");
    }

    if (versionA == null || versionB == null) {
        throw new IllegalArgumentException("Version cannot be null or blank");
    }
    // older version is always the lower, regardless of the order supplied
    final Integer older = Math.min(versionA, versionB);
    final Integer newer = Math.max(versionA, versionB);

    // Get the content for both versions of the flow
    final byte[] serializedSnapshotA = flowPersistenceProvider.getFlowContent(bucketIdentifier, flowIdentifier, older);
    if (serializedSnapshotA == null || serializedSnapshotA.length == 0) {
        throw new IllegalStateException("No serialized content found for snapshot with flow identifier "
                + flowIdentifier + " and version " + older);
    }

    final byte[] serializedSnapshotB = flowPersistenceProvider.getFlowContent(bucketIdentifier, flowIdentifier, newer);
    if (serializedSnapshotB == null || serializedSnapshotB.length == 0) {
        throw new IllegalStateException("No serialized content found for snapshot with flow identifier "
                + flowIdentifier + " and version " + newer);
    }

    // deserialize the contents
    final InputStream inputA = new ByteArrayInputStream(serializedSnapshotA);
    final VersionedFlowSnapshot snapshotA = deserializeFlowContent(inputA);
    final VersionedProcessGroup flowContentsA = snapshotA.getFlowContents();

    final InputStream inputB = new ByteArrayInputStream(serializedSnapshotB);
    final VersionedFlowSnapshot snapshotB = deserializeFlowContent(inputB);
    final VersionedProcessGroup flowContentsB = snapshotB.getFlowContents();

    final ComparableDataFlow comparableFlowA = new StandardComparableDataFlow(String.format("Version %d", older), flowContentsA);
    final ComparableDataFlow comparableFlowB = new StandardComparableDataFlow(String.format("Version %d", newer), flowContentsB);

    // Compare the two versions of the flow
    final FlowComparator flowComparator = new StandardFlowComparator(comparableFlowA, comparableFlowB,
            null, new ConciseEvolvingDifferenceDescriptor());
    final FlowComparison flowComparison = flowComparator.compare();

    final VersionedFlowDifference result = new VersionedFlowDifference();
    result.setBucketId(bucketIdentifier);
    result.setFlowId(flowIdentifier);
    result.setVersionA(older);
    result.setVersionB(newer);

    final Set<ComponentDifferenceGroup> differenceGroups = getStringComponentDifferenceGroupMap(flowComparison.getDifferences());
    result.setComponentDifferenceGroups(differenceGroups);

    return result;
}
 

@Test
public void testSerializeDeserializeFlowContent() {
    final VersionedProcessor processor1 = new VersionedProcessor();
    processor1.setIdentifier("processor1");
    processor1.setName("My Processor 1");

    final VersionedProcessGroup processGroup1 = new VersionedProcessGroup();
    processGroup1.setIdentifier("pg1");
    processGroup1.setName("My Process Group");
    processGroup1.getProcessors().add(processor1);

    final VersionedFlowSnapshot snapshot = new VersionedFlowSnapshot();
    snapshot.setFlowContents(processGroup1);

    final FlowContent flowContent = new FlowContent();
    flowContent.setFlowSnapshot(snapshot);

    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    serializer.serializeFlowContent(flowContent, out);

    //final String json = new String(out.toByteArray(), StandardCharsets.UTF_8);
    //System.out.println(json);

    final ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());

    // make sure we can read the version from the input stream and it should be the current version
    final Integer version = serializer.readDataModelVersion(in);
    assertEquals(serializer.getCurrentDataModelVersion(), version);
    assertEquals(false, serializer.isProcessGroupVersion(version));

    // make sure we can deserialize back to FlowContent
    final FlowContent deserializedFlowContent = serializer.deserializeFlowContent(version, in);
    assertNotNull(deserializedFlowContent);

    final VersionedFlowSnapshot deserializedSnapshot = deserializedFlowContent.getFlowSnapshot();
    assertNotNull(deserializedSnapshot);

    final VersionedProcessGroup deserializedProcessGroup1 = deserializedSnapshot.getFlowContents();
    assertNotNull(deserializedProcessGroup1);
    assertEquals(processGroup1.getIdentifier(), deserializedProcessGroup1.getIdentifier());
    assertEquals(processGroup1.getName(), deserializedProcessGroup1.getName());

    assertEquals(1, deserializedProcessGroup1.getProcessors().size());

    final VersionedProcessor deserializedProcessor1 = deserializedProcessGroup1.getProcessors().iterator().next();
    assertEquals(processor1.getIdentifier(), deserializedProcessor1.getIdentifier());
    assertEquals(processor1.getName(), deserializedProcessor1.getName());
}
 

@Test
public void testSerializeDeserializeWithExternalServices() throws SerializationException {
    final VersionedProcessGroup processGroup1 = new VersionedProcessGroup();
    processGroup1.setIdentifier("pg1");
    processGroup1.setName("My Process Group");

    final ExternalControllerServiceReference serviceReference1 = new ExternalControllerServiceReference();
    serviceReference1.setIdentifier("1");
    serviceReference1.setName("Service 1");

    final ExternalControllerServiceReference serviceReference2 = new ExternalControllerServiceReference();
    serviceReference2.setIdentifier("2");
    serviceReference2.setName("Service 2");

    final Map<String,ExternalControllerServiceReference> serviceReferences = new HashMap<>();
    serviceReferences.put(serviceReference1.getIdentifier(), serviceReference1);
    serviceReferences.put(serviceReference2.getIdentifier(), serviceReference2);

    final VersionedFlowSnapshot snapshot = new VersionedFlowSnapshot();
    snapshot.setFlowContents(processGroup1);
    snapshot.setExternalControllerServices(serviceReferences);

    final FlowContent flowContent = new FlowContent();
    flowContent.setFlowSnapshot(snapshot);

    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    serializer.serializeFlowContent(flowContent, out);

    final ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());

    // make sure we can read the version from the input stream and it should be the current version
    final Integer version = serializer.readDataModelVersion(in);
    assertEquals(serializer.getCurrentDataModelVersion(), version);

    // make sure we can deserialize back to FlowContent
    final FlowContent deserializedFlowContent = serializer.deserializeFlowContent(version, in);
    assertNotNull(deserializedFlowContent);

    final VersionedFlowSnapshot deserializedSnapshot = deserializedFlowContent.getFlowSnapshot();
    assertNotNull(deserializedSnapshot);

    final VersionedProcessGroup deserializedProcessGroup = deserializedSnapshot.getFlowContents();
    assertEquals(processGroup1.getIdentifier(), deserializedProcessGroup.getIdentifier());
    assertEquals(processGroup1.getName(), deserializedProcessGroup.getName());

    final Map<String,ExternalControllerServiceReference> deserializedServiceReferences = deserializedSnapshot.getExternalControllerServices();
    assertNotNull(deserializedServiceReferences);
    assertEquals(2, deserializedServiceReferences.size());

    final ExternalControllerServiceReference deserializedServiceReference1 = deserializedServiceReferences.get(serviceReference1.getIdentifier());
    assertNotNull(deserializedServiceReference1);
    assertEquals(serviceReference1.getIdentifier(), deserializedServiceReference1.getIdentifier());
    assertEquals(serviceReference1.getName(), deserializedServiceReference1.getName());
}
 

@GET
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.APPLICATION_JSON)
@Path("process-groups/{id}/download")
@ApiOperation(
    value = "Gets the latest version of a Process Group for download",
    response = String.class,
    authorizations = {
        @Authorization(value = "Read - /process-groups/{uuid}")
    }
)
@ApiResponses(value = {
    @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
    @ApiResponse(code = 401, message = "Client could not be authenticated."),
    @ApiResponse(code = 403, message = "Client is not authorized to make this request."),
    @ApiResponse(code = 404, message = "The specified resource could not be found."),
    @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.")
})
public Response exportFlowVersion(@ApiParam(value = "The process group id.", required = true) @PathParam("id") final String groupId) {
    // authorize access
    serviceFacade.authorizeAccess(lookup -> {
        final ProcessGroupAuthorizable groupAuthorizable = lookup.getProcessGroup(groupId);
        // ensure access to process groups (nested), encapsulated controller services and referenced parameter contexts
        authorizeProcessGroup(groupAuthorizable, authorizer, lookup, RequestAction.READ, true,
                false, true, false, true);
    });

    // get the versioned flow
    final VersionedFlowSnapshot versionedFlowSnapshot = serviceFacade.getVersionedFlowSnapshotByGroupId(groupId);

    final VersionedProcessGroup versionedProcessGroup = versionedFlowSnapshot.getFlowContents();
    final String flowName = versionedProcessGroup.getName();
    final int flowVersion = versionedFlowSnapshot.getSnapshotMetadata().getVersion();

    // clear top-level registry data which doesn't belong in versioned flow download
    versionedFlowSnapshot.setFlow(null);
    versionedFlowSnapshot.setBucket(null);
    versionedFlowSnapshot.setSnapshotMetadata(null);

    // clear nested process group registry data which doesn't belong in versioned flow download
    sanitizeRegistryInfo(versionedProcessGroup);

    // determine the name of the attachment - possible issues with spaces in file names
    final String filename = flowName.replaceAll("\\s", "_") + "_" + flowVersion + ".json";

    return generateOkResponse(versionedFlowSnapshot).header(HttpHeaders.CONTENT_DISPOSITION, String.format("attachment; filename=\"%s\"", filename)).build();
}