Java Code Examples for org.apache.nifi.controller.ProcessorNode#setAutoTerminatedRelationships()

@Test
public void testNoEventsIfExplicitlyRemoved() throws ExecutionException, InterruptedException, IOException {
    final ProcessorNode createProcessor = createProcessorNode((context, session) -> {
        FlowFile flowFile = session.create();

        final Map<String, String> attrs = new HashMap<>();
        attrs.put("test", "integration");
        attrs.put("integration", "true");

        flowFile = session.putAllAttributes(flowFile, attrs);
        session.remove(flowFile);
    }, REL_SUCCESS);

    createProcessor.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));

    triggerOnce(createProcessor);

    // There should be exactly 1 event.
    final ProvenanceEventRepository provRepo = getProvenanceRepository();
    assertEquals(-1L, provRepo.getMaxEventId().longValue());
    assertTrue(provRepo.getEvents(0L, 1000).isEmpty());
}
 

@Test
public void testSensitiveParameters() throws ExecutionException, InterruptedException {
    final ProcessorNode procNode = createProcessorNode(WriteText.class);
    procNode.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));

    final ParameterContext parameterContext = new StandardParameterContext(UUID.randomUUID().toString(), "testEscapedParameterReference", ParameterReferenceManager.EMPTY, null);
    getRootGroup().setParameterContext(parameterContext);

    final Map<String, Parameter> parameters = new HashMap<>();
    parameters.put("foo", new Parameter(new ParameterDescriptor.Builder().name("foo").build(), "bar"));
    parameters.put("sensitive", new Parameter(new ParameterDescriptor.Builder().name("sensitive").sensitive(true).build(), "*password*"));
    parameterContext.setParameters(parameters);

    verifyCannotSetParameter(procNode, "#{sensitive}", null);
    verifyCannotSetParameter(procNode, "abc#{sensitive}foo", null);
    verifyCannotSetParameter(procNode, "#{foo}", "#{foo}");
    verifyCannotSetParameter(procNode, null, "#{sensitive}#{sensitive}");
    verifyCannotSetParameter(procNode, null, "#{sensitive}123");
    verifyCannotSetParameter(procNode, null, "123#{sensitive}");
    verifyCannotSetParameter(procNode, null, "#{foo}");

    verifyText(procNode, "#{foo}", "#{sensitive}", "bar*password*");
    verifyText(procNode, "#{foo}", "##{sensitive}##{sensitive}##{sensitive}", "bar#{sensitive}#{sensitive}#{sensitive}");
}
 

/**
 * Validates the processors start/stop sequence where the order of
 * operations can only be @OnScheduled, @OnUnscheduled, @OnStopped.
 */
@Test
@Ignore
public void validateSuccessfullAndOrderlyShutdown() throws Exception {
    fc = this.buildFlowControllerForTest();
    ProcessGroup testGroup = fc.createProcessGroup(UUID.randomUUID().toString());
    this.setControllerRootGroup(fc, testGroup);
    ProcessorNode testProcNode = fc.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();

    // sets the scenario for the processor to run
    int randomDelayLimit = 3000;
    this.randomOnTriggerDelay(testProcessor, randomDelayLimit);

    testProcNode.setMaxConcurrentTasks(4);
    testProcNode.setScheduldingPeriod("500 millis");
    testProcNode.setAutoTerminatedRelationships(Collections.singleton(new Relationship.Builder().name("success").build()));

    testGroup.addProcessor(testProcNode);

    fc.startProcessGroup(testGroup.getIdentifier());
    Thread.sleep(2000); // let it run for a while
    assertTrue(testProcNode.getScheduledState() == ScheduledState.RUNNING);

    fc.stopAllProcessors();

    Thread.sleep(randomDelayLimit); // up to randomDelayLimit, otherwise next assertion may fail as the processor still executing

    // validates that regardless of how many running tasks, lifecycle
    // operation are invoked atomically (once each).
    assertTrue(testProcNode.getScheduledState() == ScheduledState.STOPPED);
    // . . . hence only 3 operations must be in the list
    assertEquals(3, testProcessor.operationNames.size());
    // . . . and ordered as @OnScheduled, @OnUnscheduled, @OnStopped
    assertEquals("@OnScheduled", testProcessor.operationNames.get(0));
    assertEquals("@OnUnscheduled", testProcessor.operationNames.get(1));
    assertEquals("@OnStopped", testProcessor.operationNames.get(2));
}
 

@Test
public void testDropEventIfRoutedToAutoTerminatedRelationship() throws ExecutionException, InterruptedException, IOException {
    final ProcessorNode createProcessor = createProcessorNode((context, session) -> {
        FlowFile flowFile = session.create();

        final Map<String, String> attrs = new HashMap<>();
        attrs.put("test", "integration");
        attrs.put("integration", "true");

        flowFile = session.putAllAttributes(flowFile, attrs);
        session.transfer(flowFile, REL_SUCCESS);
    }, REL_SUCCESS);

    createProcessor.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));

    triggerOnce(createProcessor);

    // There should be exactly 1 event.
    final ProvenanceEventRepository provRepo = getProvenanceRepository();
    assertEquals(1L, provRepo.getMaxEventId().longValue());

    final ProvenanceEventRecord firstEvent = provRepo.getEvent(0L);
    assertEquals(ProvenanceEventType.CREATE, firstEvent.getEventType());
    assertEquals("integration", firstEvent.getAttribute("test"));
    assertEquals("true", firstEvent.getAttribute("integration"));

    final ProvenanceEventRecord secondEvent = provRepo.getEvent(1L);
    assertEquals(ProvenanceEventType.DROP, secondEvent.getEventType());
    assertEquals("integration", secondEvent.getAttribute("test"));
    assertEquals("true", secondEvent.getAttribute("integration"));
}
 

@Test
public void testCallingControllerService() throws ExecutionException, InterruptedException {
    final ProcessorNode counter = createProcessorNode(ControllerServiceReferencingProcessor.class.getName());

    final ControllerServiceNode serviceNode = createControllerServiceNode(CounterControllerService.class.getName());
    assertSame(ValidationStatus.VALID, serviceNode.performValidation());
    getFlowController().getControllerServiceProvider().enableControllerService(serviceNode).get();

    counter.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));
    counter.setProperties(Collections.singletonMap("Counter Service", serviceNode.getIdentifier()));

    triggerOnce(counter);

    assertEquals(1, ((Counter) serviceNode.getControllerServiceImplementation()).getValue());
}
 

@Test
public void testEscapedParameterReference() throws ExecutionException, InterruptedException {
    final ProcessorNode procNode = createProcessorNode(WriteText.class);
    procNode.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));

    verifyText(procNode, "hello", "hello");
    verifyText(procNode, "##{foo}", "#{foo}");
    verifyText(procNode, "####{foo}", "##{foo}");
    verifyText(procNode, "## hello ##{foo} ##{bar}", "## hello #{foo} #{bar}");
}
 

@Test
public void testProperReferences() throws ExecutionException, InterruptedException {
    final ProcessorNode procNode = createProcessorNode(WriteText.class);
    procNode.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));

    final ParameterContext parameterContext = new StandardParameterContext(UUID.randomUUID().toString(), "testEscapedParameterReference", ParameterReferenceManager.EMPTY, null);
    getRootGroup().setParameterContext(parameterContext);

    final Map<String, Parameter> parameters = new HashMap<>();
    parameters.put("foo", new Parameter(new ParameterDescriptor.Builder().name("foo").build(), "bar"));
    parameters.put("sensitive", new Parameter(new ParameterDescriptor.Builder().name("sensitive").sensitive(true).build(), "*password*"));
    parameterContext.setParameters(parameters);

    verifyText(procNode, "hello", "hello");
    verifyText(procNode, "##{foo}", "#{foo}");
    verifyText(procNode, "#{foo}", "bar");
    verifyText(procNode, "####{foo}", "##{foo}");
    verifyText(procNode, "#####{foo}", "##bar");
    verifyText(procNode, "## hello #{foo} ##{bar}", "## hello bar #{bar}");

    try {
        verifyText(procNode, "#{bar}", "ISE");
    } catch (final IllegalStateException expected) {
        // Expect IllegalStateException because processor is not valid because it references a non-existent parameter.
    }

    verifyText(procNode, "#{foo}", "password", "barpassword");
    verifyText(procNode, "#{foo}", "#{sensitive}", "bar*password*");
}
 

/**
 * Validates the processors start/stop sequence where the order of
 * operations can only be @OnScheduled, @OnUnscheduled, @OnStopped.
 */
@Test
@Ignore
public void validateSuccessfulAndOrderlyShutdown() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest();
    flowManager = fcsb.getFlowManager();
    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();

    // sets the scenario for the processor to run
    int randomDelayLimit = 3000;
    this.randomOnTriggerDelay(testProcessor, randomDelayLimit);

    testProcNode.setMaxConcurrentTasks(4);
    testProcNode.setScheduldingPeriod("500 millis");
    testProcNode.setAutoTerminatedRelationships(Collections.singleton(new Relationship.Builder().name("success").build()));

    testGroup.addProcessor(testProcNode);

    flowManager.getGroup(testGroup.getIdentifier()).startProcessing();
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), SHORT_DELAY_TOLERANCE);

    flowManager.getRootGroup().stopProcessing();

    Thread.sleep(randomDelayLimit); // up to randomDelayLimit, otherwise next assertion may fail as the processor still executing

    // validates that regardless of how many running tasks, lifecycle
    // operation are invoked atomically (once each).
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState(), SHORT_DELAY_TOLERANCE);
    // . . . hence only 3 operations must be in the list
    assertCondition(() -> testProcessor.operationNames.size() == 3, SHORT_DELAY_TOLERANCE);
    // . . . and ordered as @OnScheduled, @OnUnscheduled, @OnStopped
    assertEquals("@OnScheduled", testProcessor.operationNames.get(0));
    assertEquals("@OnUnscheduled", testProcessor.operationNames.get(1));
    assertEquals("@OnStopped", testProcessor.operationNames.get(2));
}
 

private void testJoin(final boolean emitJoinEventExplicitly) throws ExecutionException, InterruptedException, IOException {
    final ProcessorNode createProcessor = createGenerateProcessor(0);

    final ProcessorNode joinProcessor = createProcessorNode((context, session) -> {
        final List<FlowFile> originals = new ArrayList<>();
        FlowFile flowFile;
        while ((flowFile = session.get()) != null) {
            originals.add(flowFile);
        }

        FlowFile merged = session.create(originals);

        final Map<String, String> attrs = new HashMap<>();
        attrs.put("test", "integration");
        attrs.put("integration", "true");

        merged = session.putAllAttributes(merged, attrs);
        merged = session.write(merged, out -> out.write('A'));

        if (emitJoinEventExplicitly) {
            session.getProvenanceReporter().join(originals, merged);
        }

        session.remove(originals);
        session.transfer(merged, REL_SUCCESS);
        session.getProvenanceReporter().route(merged, REL_SUCCESS);
    }, REL_SUCCESS);

    connect(createProcessor, joinProcessor, REL_SUCCESS);
    joinProcessor.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));

    for (int i=0; i < 3; i++) {
        triggerOnce(createProcessor);
    }

    triggerOnce(joinProcessor);

    final ProvenanceEventRepository provRepo = getProvenanceRepository();
    assertEquals(8L, provRepo.getMaxEventId().longValue());

    // Crete events are from the first 'generate' processor.
    for (int i=0; i < 3; i++) {
        assertEquals(ProvenanceEventType.CREATE, provRepo.getEvent(i).getEventType());
    }

    // Any FORK/JOIN events will occur first in the Process Session to ensure that any other events that reference the FlowFile
    // that is created as a result have a FlowFile to actually reference.
    final ProvenanceEventRecord joinEvent = provRepo.getEvent(3);
    assertEquals(ProvenanceEventType.JOIN, joinEvent.getEventType());
    assertEquals("integration", joinEvent.getAttribute("test"));
    assertEquals("true", joinEvent.getAttribute("integration"));
    assertEquals(3, joinEvent.getParentUuids().size());
    assertEquals(1, joinEvent.getChildUuids().size());
    assertEquals(joinEvent.getFlowFileUuid(), joinEvent.getChildUuids().get(0));

    // Next event to occur in the Processor is the DORP event
    for (int i=4; i < 7; i++) {
        assertEquals(ProvenanceEventType.DROP, provRepo.getEvent(i).getEventType());
    }

    // Finally Processor will ROUTE the FlowFile
    final ProvenanceEventRecord routeEvent = provRepo.getEvent(7);
    assertEquals(ProvenanceEventType.ROUTE, routeEvent.getEventType());
    assertEquals("success", routeEvent.getRelationship());
    assertEquals("integration", routeEvent.getAttribute("test"));
    assertEquals("true", routeEvent.getAttribute("integration"));

    // Merged FlowFile is then auto-terminated.
    final ProvenanceEventRecord dropJoinedEvent = provRepo.getEvent(8);
    assertEquals(ProvenanceEventType.DROP, dropJoinedEvent.getEventType());
    assertEquals("integration", dropJoinedEvent.getAttribute("test"));
    assertEquals("true", dropJoinedEvent.getAttribute("integration"));
}
 

@Test
public void testImportFlowWithProcessorAndControllerService() throws ExecutionException, InterruptedException {
    // Build a Versioned Flow that consists of a Controller Service and a Processor
    // that references that Controller Service.
    final ControllerServiceNode controllerService = createControllerServiceNode(LongValidatingControllerService.class);
    controllerService.setProperties(Collections.singletonMap(LongValidatingControllerService.DELAY.getName(), "250 millis"));

    final ProcessorNode processor = createProcessorNode(NopServiceReferencingProcessor.class);
    processor.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));
    processor.setProperties(Collections.singletonMap(NopServiceReferencingProcessor.SERVICE.getName(), controllerService.getIdentifier()));

    final VersionedFlowSnapshot proposedFlow = createFlowSnapshot(Collections.singletonList(controllerService), Collections.singletonList(processor), null);

    // Create an Inner Process Group and update it to match the Versioned Flow.
    final ProcessGroup innerGroup = getFlowController().getFlowManager().createProcessGroup("inner-group-id");
    innerGroup.setName("Inner Group");
    getRootGroup().addProcessGroup(innerGroup);

    innerGroup.updateFlow(proposedFlow, null, false, true, false);

    // Ensure that the controller service is valid and enable it.
    final Set<ControllerServiceNode> serviceNodes = innerGroup.findAllControllerServices();
    assertEquals(1, serviceNodes.size());

    final ControllerServiceNode serviceNode = serviceNodes.iterator().next();
    final ValidationStatus validationStatus = serviceNode.performValidation();
    assertEquals(ValidationStatus.VALID, validationStatus);
    getFlowController().getControllerServiceProvider().enableControllerService(serviceNode).get();
    assertTrue(serviceNode.isActive());

    // Ensure that the processor is valid.
    final List<ProcessorNode> processorNodes = innerGroup.findAllProcessors();
    assertEquals(1, processorNodes.size());

    final ProcessorNode procNode = processorNodes.get(0);
    final ValidationStatus procValidationStatus = procNode.performValidation();
    final Collection<ValidationResult> validationErrors = procNode.getValidationErrors();
    System.out.println(validationErrors);
    assertEquals(Collections.emptyList(), validationErrors);
    assertEquals(ValidationStatus.VALID, procValidationStatus);

    // Ensure that the reference to the controller service was properly updated
    final String referencedServiceId = procNode.getEffectivePropertyValue(NopServiceReferencingProcessor.SERVICE);
    assertEquals(serviceNode.getIdentifier(), referencedServiceId);
    assertNotEquals("service-id", referencedServiceId);
}