Java Code Examples for org.apache.nifi.controller.ProcessorNode#performValidation()

@Test
public void validateDisableOperation() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest();
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    final ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(),
            UUID.randomUUID().toString(), fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState());
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getPhysicalScheduledState());
    // validates idempotency
    for (int i = 0; i < 2; i++) {
        testProcNode.disable();
    }
    assertCondition(() -> ScheduledState.DISABLED == testProcNode.getScheduledState());
    assertCondition(() -> ScheduledState.DISABLED == testProcNode.getPhysicalScheduledState());

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    assertCondition(() -> ScheduledState.DISABLED == testProcNode.getPhysicalScheduledState());
}
 

/**
 * Will validate the idempotent nature of processor start operation which
 * can be called multiple times without any side-effects.
 */
@Test
public void validateIdempotencyOfProcessorStartOperation() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest();
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    final ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();

    // sets the scenario for the processor to run
    this.noop(testProcessor);

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    processScheduler.startProcessor(testProcNode, true);
    processScheduler.startProcessor(testProcNode, true);

    Thread.sleep(500);
    assertCondition(() -> testProcessor.operationNames.size() == 1);
    assertEquals("@OnScheduled", testProcessor.operationNames.get(0));
}
 

/**
 * Validates that Processor is eventually started once invocation of
 *
 * @OnSchedule stopped throwing exceptions.
 */
@Test
public void validateProcessScheduledAfterAdministrativeDelayDueToTheOnScheduledException() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest();
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();

    // sets the scenario for the processor to run
    this.noop(testProcessor);
    testProcessor.generateExceptionOnScheduled = true;
    testProcessor.keepFailingOnScheduledTimes = 2;

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), LONG_DELAY_TOLERANCE);
    processScheduler.stopProcessor(testProcNode);
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState(), SHORT_DELAY_TOLERANCE);
}
 

/**
 * Validates that Processor can be stopped when @OnScheduled constantly
 * fails. Basically validates that the re-try loop breaks if user initiated
 * stopProcessor.
 */
@Test
public void validateProcessorCanBeStoppedWhenOnScheduledConstantlyFails() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest();
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();

    // sets the scenario for the processor to run
    this.longRunningOnUnschedule(testProcessor, 100);
    testProcessor.generateExceptionOnScheduled = true;
    testProcessor.keepFailingOnScheduledTimes = Integer.MAX_VALUE;

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), SHORT_DELAY_TOLERANCE);
    processScheduler.stopProcessor(testProcNode);
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState(), SHORT_DELAY_TOLERANCE);
}
 

/**
 * Validates that the Processor can be stopped when @OnScheduled blocks
 * indefinitely but written to react to thread interrupts
 */
@Test
public void validateProcessorCanBeStoppedWhenOnScheduledBlocksIndefinitelyInterruptable() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest(NiFiProperties.PROCESSOR_SCHEDULING_TIMEOUT, "5 sec");
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();
    // sets the scenario for the processor to run
    this.blockingInterruptableOnUnschedule(testProcessor);

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), SHORT_DELAY_TOLERANCE);
    processScheduler.stopProcessor(testProcNode);
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState(), MEDIUM_DELAY_TOLERANCE);
}
 

/**
 * Validates that the Processor can be stopped when @OnScheduled blocks
 * indefinitely and written to ignore thread interrupts
 */
@Test
public void validateProcessorCanBeStoppedWhenOnScheduledBlocksIndefinitelyUninterruptable() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest(NiFiProperties.PROCESSOR_SCHEDULING_TIMEOUT, "1 sec");
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();
    // sets the scenario for the processor to run
    this.blockingUninterruptableOnUnschedule(testProcessor);

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), MEDIUM_DELAY_TOLERANCE);
    processScheduler.stopProcessor(testProcNode);
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState(), MEDIUM_DELAY_TOLERANCE);
}
 

/**
 * Validates that processor can be stopped if onTrigger() keeps throwing
 * exceptions.
 */
@Test
public void validateProcessorCanBeStoppedWhenOnTriggerThrowsException() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest();
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();

    // sets the scenario for the processor to run
    this.noop(testProcessor);
    testProcessor.generateExceptionOnTrigger = true;

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), LONG_DELAY_TOLERANCE);
    processScheduler.disableProcessor(testProcNode);
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), LONG_DELAY_TOLERANCE);
    processScheduler.stopProcessor(testProcNode);
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState(), LONG_DELAY_TOLERANCE);
}
 

@Test(timeout = 10000)
public void testProcessorThrowsExceptionOnScheduledRetry() throws InterruptedException {
    final FailOnScheduledProcessor proc = new FailOnScheduledProcessor();
    proc.setDesiredFailureCount(3);

    proc.initialize(new StandardProcessorInitializationContext(UUID.randomUUID().toString(), null, null, null, KerberosConfig.NOT_CONFIGURED));
    final ReloadComponent reloadComponent = Mockito.mock(ReloadComponent.class);
    final LoggableComponent<Processor> loggableComponent = new LoggableComponent<>(proc, systemBundle.getBundleDetails().getCoordinate(), null);

    final ProcessorNode procNode = new StandardProcessorNode(loggableComponent, UUID.randomUUID().toString(),
        new StandardValidationContextFactory(serviceProvider, variableRegistry),
        scheduler, serviceProvider, new StandardComponentVariableRegistry(VariableRegistry.EMPTY_REGISTRY), reloadComponent, extensionManager, new SynchronousValidationTrigger());

    procNode.performValidation();
    rootGroup.addProcessor(procNode);

    scheduler.startProcessor(procNode, true);
    while (!proc.isSucceess()) {
        Thread.sleep(5L);
    }

    assertEquals(3, proc.getOnScheduledInvocationCount());
}
 

protected final Future<Void> start(final ProcessorNode procNode) {
    final ValidationStatus validationStatus = procNode.performValidation();
    if (validationStatus != ValidationStatus.VALID) {
        throw new IllegalStateException("Processor is invalid: " + procNode + ": " + procNode.getValidationErrors());
    }

    return rootProcessGroup.startProcessor(procNode, true);
}
 

/**
 * Validates that processor can be stopped before start sequence finished.
 */
@Test
public void validateProcessorUnscheduledAndStoppedWhenStopIsCalledBeforeProcessorFullyStarted() throws Exception {
    final FlowManagerAndSystemBundle fcsb = this.buildFlowControllerForTest();
    flowManager = fcsb.getFlowManager();

    ProcessGroup testGroup = flowManager.createProcessGroup(UUID.randomUUID().toString());
    ProcessorNode testProcNode = flowManager.createProcessor(TestProcessor.class.getName(), UUID.randomUUID().toString(),
            fcsb.getSystemBundle().getBundleDetails().getCoordinate());
    testProcNode.setProperties(properties);
    TestProcessor testProcessor = (TestProcessor) testProcNode.getProcessor();

    // sets the scenario for the processor to run
    int delay = 200;
    this.longRunningOnSchedule(testProcessor, delay);

    testProcNode.performValidation();
    processScheduler.startProcessor(testProcNode, true);
    assertCondition(() -> ScheduledState.RUNNING == testProcNode.getScheduledState(), MEDIUM_DELAY_TOLERANCE);

    processScheduler.stopProcessor(testProcNode);
    assertCondition(() -> ScheduledState.STOPPED == testProcNode.getScheduledState(), MEDIUM_DELAY_TOLERANCE);
    assertCondition(() -> testProcessor.operationNames.size() == 3, LONG_DELAY_TOLERANCE);
    assertEquals("@OnScheduled", testProcessor.operationNames.get(0));
    assertEquals("@OnUnscheduled", testProcessor.operationNames.get(1));
    assertEquals("@OnStopped", testProcessor.operationNames.get(2));
}
 

@Test(timeout = 60000)
public void testDisableControllerServiceWithProcessorTryingToStartUsingIt() throws InterruptedException, ExecutionException {
    final String uuid = UUID.randomUUID().toString();
    final Processor proc = new ServiceReferencingProcessor();
    proc.initialize(new StandardProcessorInitializationContext(uuid, null, null, null, KerberosConfig.NOT_CONFIGURED));

    final ReloadComponent reloadComponent = Mockito.mock(ReloadComponent.class);

    final ControllerServiceNode service = flowManager.createControllerService(NoStartServiceImpl.class.getName(), "service",
            systemBundle.getBundleDetails().getCoordinate(), null, true, true);

    rootGroup.addControllerService(service);

    final LoggableComponent<Processor> loggableComponent = new LoggableComponent<>(proc, systemBundle.getBundleDetails().getCoordinate(), null);
    final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, variableRegistry);
    final ProcessorNode procNode = new StandardProcessorNode(loggableComponent, uuid, validationContextFactory, scheduler,
        serviceProvider, new StandardComponentVariableRegistry(VariableRegistry.EMPTY_REGISTRY), reloadComponent, extensionManager, new SynchronousValidationTrigger());

    rootGroup.addProcessor(procNode);

    Map<String, String> procProps = new HashMap<>();
    procProps.put(ServiceReferencingProcessor.SERVICE_DESC.getName(), service.getIdentifier());
    procNode.setProperties(procProps);

    service.performValidation();
    scheduler.enableControllerService(service);

    procNode.performValidation();
    scheduler.startProcessor(procNode, true);

    Thread.sleep(25L);

    scheduler.stopProcessor(procNode);
    assertTrue(service.isActive());
    assertSame(service.getState(), ControllerServiceState.ENABLING);
    scheduler.disableControllerService(service).get();
    assertFalse(service.isActive());
    assertSame(service.getState(), ControllerServiceState.DISABLED);
}
 

@Test(timeout = 10000)
public void testProcessorTimeOutRespondsToInterrupt() throws InterruptedException {
    final FailOnScheduledProcessor proc = new FailOnScheduledProcessor();
    proc.setDesiredFailureCount(0);
    proc.setOnScheduledSleepDuration(20, TimeUnit.MINUTES, true, 1);

    proc.initialize(new StandardProcessorInitializationContext(UUID.randomUUID().toString(), null, null, null, KerberosConfig.NOT_CONFIGURED));
    final ReloadComponent reloadComponent = Mockito.mock(ReloadComponent.class);
    final LoggableComponent<Processor> loggableComponent = new LoggableComponent<>(proc, systemBundle.getBundleDetails().getCoordinate(), null);

    final ProcessorNode procNode = new StandardProcessorNode(loggableComponent, UUID.randomUUID().toString(),
        new StandardValidationContextFactory(serviceProvider, variableRegistry),
        scheduler, serviceProvider, new StandardComponentVariableRegistry(VariableRegistry.EMPTY_REGISTRY), reloadComponent, extensionManager, new SynchronousValidationTrigger());

    rootGroup.addProcessor(procNode);

    procNode.performValidation();
    scheduler.startProcessor(procNode, true);
    while (!proc.isSucceess()) {
        Thread.sleep(5L);
    }

    // The first time that the processor's @OnScheduled method is called, it will sleep for 20 minutes. The scheduler should interrupt
    // that thread and then try again. The second time, the Processor will not sleep because setOnScheduledSleepDuration was called
    // above with iterations = 1
    assertEquals(2, proc.getOnScheduledInvocationCount());
}
 

@Test(timeout = 10000)
public void testProcessorTimeOutNoResponseToInterrupt() throws InterruptedException {
    final FailOnScheduledProcessor proc = new FailOnScheduledProcessor();
    proc.setDesiredFailureCount(0);
    proc.setOnScheduledSleepDuration(20, TimeUnit.MINUTES, false, 1);

    proc.initialize(new StandardProcessorInitializationContext(UUID.randomUUID().toString(), null, null, null, KerberosConfig.NOT_CONFIGURED));
    final ReloadComponent reloadComponent = Mockito.mock(ReloadComponent.class);
    final LoggableComponent<Processor> loggableComponent = new LoggableComponent<>(proc, systemBundle.getBundleDetails().getCoordinate(), null);

    final ProcessorNode procNode = new StandardProcessorNode(loggableComponent, UUID.randomUUID().toString(),
        new StandardValidationContextFactory(serviceProvider, variableRegistry),
        scheduler, serviceProvider, new StandardComponentVariableRegistry(VariableRegistry.EMPTY_REGISTRY), reloadComponent, extensionManager, new SynchronousValidationTrigger());

    rootGroup.addProcessor(procNode);

    procNode.performValidation();
    scheduler.startProcessor(procNode, true);

    while(proc.getOnScheduledInvocationCount() < 1){
        Thread.sleep(100L);
    }
    assertEquals(1, proc.getOnScheduledInvocationCount());
    Thread.sleep(100L);
    assertEquals(1, proc.getOnScheduledInvocationCount());

    // Allow test to complete.
    proc.setAllowSleepInterrupt(true);
}
 

@Test
public void testImportFlowWithProcessorAndControllerService() throws ExecutionException, InterruptedException {
    // Build a Versioned Flow that consists of a Controller Service and a Processor
    // that references that Controller Service.
    final ControllerServiceNode controllerService = createControllerServiceNode(LongValidatingControllerService.class);
    controllerService.setProperties(Collections.singletonMap(LongValidatingControllerService.DELAY.getName(), "250 millis"));

    final ProcessorNode processor = createProcessorNode(NopServiceReferencingProcessor.class);
    processor.setAutoTerminatedRelationships(Collections.singleton(REL_SUCCESS));
    processor.setProperties(Collections.singletonMap(NopServiceReferencingProcessor.SERVICE.getName(), controllerService.getIdentifier()));

    final VersionedFlowSnapshot proposedFlow = createFlowSnapshot(Collections.singletonList(controllerService), Collections.singletonList(processor), null);

    // Create an Inner Process Group and update it to match the Versioned Flow.
    final ProcessGroup innerGroup = getFlowController().getFlowManager().createProcessGroup("inner-group-id");
    innerGroup.setName("Inner Group");
    getRootGroup().addProcessGroup(innerGroup);

    innerGroup.updateFlow(proposedFlow, null, false, true, false);

    // Ensure that the controller service is valid and enable it.
    final Set<ControllerServiceNode> serviceNodes = innerGroup.findAllControllerServices();
    assertEquals(1, serviceNodes.size());

    final ControllerServiceNode serviceNode = serviceNodes.iterator().next();
    final ValidationStatus validationStatus = serviceNode.performValidation();
    assertEquals(ValidationStatus.VALID, validationStatus);
    getFlowController().getControllerServiceProvider().enableControllerService(serviceNode).get();
    assertTrue(serviceNode.isActive());

    // Ensure that the processor is valid.
    final List<ProcessorNode> processorNodes = innerGroup.findAllProcessors();
    assertEquals(1, processorNodes.size());

    final ProcessorNode procNode = processorNodes.get(0);
    final ValidationStatus procValidationStatus = procNode.performValidation();
    final Collection<ValidationResult> validationErrors = procNode.getValidationErrors();
    System.out.println(validationErrors);
    assertEquals(Collections.emptyList(), validationErrors);
    assertEquals(ValidationStatus.VALID, procValidationStatus);

    // Ensure that the reference to the controller service was properly updated
    final String referencedServiceId = procNode.getEffectivePropertyValue(NopServiceReferencingProcessor.SERVICE);
    assertEquals(serviceNode.getIdentifier(), referencedServiceId);
    assertNotEquals("service-id", referencedServiceId);
}