Java Code Examples for java.util.concurrent.ScheduledExecutorService#awaitTermination()

public void stop() throws Exception {
  if (!stopped.getAndSet(true)) {
    executorPools.forEach(ScheduledExecutorService::shutdown);

    long timeoutLeftInMillis = timeoutInMillis;

    for (ScheduledExecutorService service : executorPools) {
      final long start = System.currentTimeMillis();

      if (!service.awaitTermination(timeoutLeftInMillis, TimeUnit.MILLISECONDS)) {
        return;
      }

      timeoutLeftInMillis -= (System.currentTimeMillis() - start);
    }
  }
}
 

public void stop() {
    final ScheduledFuture<?> future = this.future.getAndSet(null);
    if (future != null
            && !future.isDone() && !future.isCancelled()
            && !future.cancel(false)) {
        Logger.getLogger(Pool.class.getName()).log(Level.WARNING, "Pool scheduler task termination timeout expired");
    }

    final ScheduledExecutorService scheduler = this.scheduler.getAndSet(null);
    if (scheduler != null) {
        scheduler.shutdown();
        try {
            if (!scheduler.awaitTermination(10, SECONDS)) { // should last something like 0s max since we killed the task
                Logger.getLogger(Pool.class.getName()).log(Level.WARNING, "Pool scheduler termination timeout expired");
            }
        } catch (final InterruptedException e) {
            //Ignore
        }
    }
}
 

@Test
public void test1 () throws Exception
{
    final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor ();

    final Input input = new UrlConnectionInput ( executor, new URL ( "http://google.com" ), true, 100 );

    final TestListener listener = new TestListener ();

    input.addInputListener ( listener );

    input.start ();
    Thread.sleep ( 100 );
    input.stop ();
    input.dispose ();

    executor.shutdown ();
    executor.awaitTermination ( Long.MAX_VALUE, TimeUnit.MINUTES );

    dumpData ( listener );

    // TODO: test
}
 

@Test
public void testDelay() throws InterruptedException {
    ScheduledExecutorService ses =
        Executors.newSingleThreadScheduledExecutor();

    SingletonTask st1 = new SingletonTask(ses, new Runnable() {
        @Override
        public void run() {
            ran += 1;
            time = System.nanoTime();
        }
    });
    long start = System.nanoTime();
    st1.reschedule(10, TimeUnit.MILLISECONDS);
    assertFalse("Check that task hasn't run yet", ran > 0);

    ses.shutdown();
    ses.awaitTermination(5, TimeUnit.SECONDS);

    assertEquals("Check that task ran", 1, ran);
    assertTrue("Check that time passed appropriately",
               (time - start) >= TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS));
}
 

public static void killTasks(final String name) {
    final ScheduledExecutorService executorService = getBackgroundExecutor(name);
    executorService.shutdownNow();
    try {
        executorService.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        Log.wtf(TAG, "Failed to shut down: " + name);
    }
    if (executorService == sExecutorServiceForTests) {
        // Don't do anything to the test service.
        return;
    }
    switch (name) {
        case KEYBOARD:
            sKeyboardExecutorService = newExecutorService(KEYBOARD);
            break;
        case SPELLING:
            sSpellingExecutorService = newExecutorService(SPELLING);
            break;
        default:
            throw new IllegalArgumentException("Invalid executor: " + name);
    }
}
 

@Test
public void shouldNotShutdownExecutorServicesSuppliedToGremlinExecutor() throws Exception {
    final ScheduledExecutorService service = Executors.newScheduledThreadPool(4, testingThreadFactory);
    final GremlinExecutor gremlinExecutor = GremlinExecutor.build()
            .executorService(service)
            .scheduledExecutorService(service).create();

    gremlinExecutor.close();
    assertFalse(service.isShutdown());
    service.shutdown();
    service.awaitTermination(30000, TimeUnit.MILLISECONDS);
}
 

/**
 * Runs a number of threads which generate metrics.
 */
public static void generateMetrics(final MetricRegistry metrics, final long metricsToGenerate, final int period,
        final TimeUnit periodTimeUnit, HadoopMetrics2Reporter metrics2Reporter, int numThreads) throws Exception {
    final ScheduledExecutorService pool = Executors.newScheduledThreadPool(numThreads);
    final CountDownLatch latch = new CountDownLatch(numThreads);

    for (int i = 0; i < numThreads; i++) {
        final int id = i;
        final int halfPeriod = (period / 2);
        Runnable task = new Runnable() {
            private long executions = 0;
            final Random r = new Random();

            @Override
            public void run() {
                if (executions >= metricsToGenerate) {
                    return;
                }
                metrics.counter("foo counter thread" + id).inc();
                executions++;
                if (executions < metricsToGenerate) {
                    pool.schedule(this, period + r.nextInt(halfPeriod), periodTimeUnit);
                } else {
                    latch.countDown();
                }
            }
        };
        pool.schedule(task, period, periodTimeUnit);
    }

    while (!latch.await(2, TimeUnit.SECONDS)) {
        metrics2Reporter.printQueueDebugMessage();
    }

    pool.shutdown();
    pool.awaitTermination(5000, TimeUnit.SECONDS);
}
 

public static void shutdown(ScheduledExecutorService executor, long timeoutMillis) {
	if (executor == null) {
		return;
	}
	executor.shutdownNow();
	boolean cleanlyTerminated;
	try {
		cleanlyTerminated = executor.awaitTermination(timeoutMillis, TimeUnit.MILLISECONDS);
	} catch (InterruptedException e) {
		Thread.currentThread().interrupt();
		cleanlyTerminated = executor.isTerminated();
	}
	if (!cleanlyTerminated) {
		String threadpoolName;
		if (executor instanceof ScheduledThreadPoolExecutor) {
			ThreadFactory factory = ((ScheduledThreadPoolExecutor) executor).getThreadFactory();
			if (factory instanceof NamingThreadFactory) {
				NamingThreadFactory namingFactory = (NamingThreadFactory) factory;
				threadpoolName = namingFactory.getPrefix();
			} else {
				threadpoolName = "unknown[" + factory.getClass().getSimpleName() + "]";
			}
		} else {
			threadpoolName = "unknown[" + executor.getClass().getSimpleName() + "]";
		}
		LOG.error("executor did not terminate in the specified time: {}", threadpoolName);
	}
}
 

@Test
public void sameIP() throws InterruptedException {
    List<Throttle> throttles = Collections.singletonList(
            new Throttle("IP", new LimitRate(1, Duration.ofMillis(500L)), HttpRequest::getRemoteAddr)
    );

    ThrottlingMiddleware<HttpResponse> middleware = builder(new ThrottlingMiddleware<HttpResponse>())
            .set(ThrottlingMiddleware::setThrottles, throttles)
            .build();

    DefaultHttpRequest req = new DefaultHttpRequest();
    req.setRemoteAddr("127.0.0.1");
    MiddlewareChain<HttpRequest, HttpResponse, ?, ?> chain = new DefaultMiddlewareChain<>(new AnyPredicate<>(), null,
            (Endpoint<HttpRequest, HttpResponse>) request -> HttpResponse.of(""));
    final AtomicInteger count200 = new AtomicInteger(0);
    ScheduledExecutorService service = Executors.newScheduledThreadPool(3);
    service.scheduleAtFixedRate(() -> {
        HttpResponse res = middleware.handle(req, chain);
        if (res.getStatus() == 200) {
            count200.addAndGet(1);
        }
    }, 0, 250, TimeUnit.MILLISECONDS);

    service.schedule(service::shutdown, 4, TimeUnit.SECONDS);
    service.awaitTermination(5, TimeUnit.SECONDS);

    assertThat(count200.get()).isGreaterThan(3);
}
 

/**
 * Test to make sure we only execute the task once
 * no matter how many times we schedule it. 
 */
public void testExecuteOnlyOnce() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
 
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  ex.submit(new Callable() {

    public Object call() throws Exception {
      latch.await();
      return null;
    }
  });
  
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  for(int i = 0; i < 50; i++) {
    decorator.schedule(increment, 0, TimeUnit.SECONDS);
  }
  
  assertEquals(0, counter.get());
  latch.countDown();
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  assertEquals(1, counter.get());
  assertEquals(49, listener.getDropCount());
}
 

/**
 * Test to make sure we reschedule the task for execution 
 * if the new requested execution is earlier than the previous one
 */
public void testRescheduleForEarlierTime() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  decorator.schedule(increment, 120, TimeUnit.SECONDS);
  decorator.schedule(increment, 10, TimeUnit.MILLISECONDS);
  
  long start = System.nanoTime();
  
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  long elapsed = System.nanoTime() - start;
  assertEquals(1, counter.get());
  assertEquals(1, listener.getDropCount());
  assertTrue(elapsed < TimeUnit.SECONDS.toNanos(120));
}
 

/**
 * Test to make sure we reschedule the task for execution 
 * if the new requested execution is earlier than the previous one
 */
public void testRescheduleForEarlierTime() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  decorator.schedule(increment, 120, TimeUnit.SECONDS);
  decorator.schedule(increment, 10, TimeUnit.MILLISECONDS);
  
  long start = System.nanoTime();
  
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  long elapsed = System.nanoTime() - start;
  assertEquals(1, counter.get());
  assertEquals(1, listener.getDropCount());
  assertTrue(elapsed < TimeUnit.SECONDS.toNanos(120));
}
 

public void executeRequest(final AutoScalingGroup autoScalingGroup, AwsAutoScalingDeployUtils awsDeployUtils, InstanceStatus instanceStatus) {
    final AtomicInteger waitFor = new AtomicInteger(1);
    final AtomicBoolean inService = new AtomicBoolean(false);
    final AtomicBoolean found = new AtomicBoolean(false);


    log.info("Waiting for new instance in asGroup to come in service...");
    ScheduledExecutorService exec = Executors.newSingleThreadScheduledExecutor();

    exec.scheduleAtFixedRate(() -> {
        log.debug("existing instances : " + Arrays.toString(autoScalingGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));
        AutoScalingGroup updatedGroup = awsDeployUtils.getAutoScalingGroup();
        log.debug("Updated instances : " + Arrays.toString(updatedGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));

        if (updatedGroup.getInstances().equals(autoScalingGroup.getInstances())) {
            log.info("no new instance found in auto scaling group.");
        }

        if (newInstance == null) {
            newInstance = findNewInstance(autoScalingGroup, updatedGroup);
            if (newInstance != null && !found.get()) {
                found.set(true);
                log.info("Found new instance with id " + newInstance.getInstanceId());
            }
        }
        if (newInstance != null && instanceStatus.inService(newInstance)) {
            waitFor.decrementAndGet();
        }
    }, 30, 30, TimeUnit.SECONDS);

    try {
        while (waitFor.intValue() > 0 && System.currentTimeMillis() <= timeout) {
            Thread.sleep(3000);
        }
        log.info("Shutting down executor");
        exec.shutdown();
        log.info("awaiting termination of executor");
        exec.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        log.error(e.getMessage());
        Thread.currentThread().interrupt();
        inService.get();
    } catch (Exception t) {
        log.error("Throwable: ", t);
    }
}
 

/**
 * 检查消费
 *
 * @param nameSvr
 * @param address
 * @throws MQClientException
 * @throws NoSuchFieldException
 * @throws SecurityException
 * @throws IllegalArgumentException
 * @throws IllegalAccessException
 * @throws InterruptedException
 * @throws RemotingException
 * @throws MQBrokerException
 */
public static long checkReceive(String cluster, String nameSvr, String address)
        throws MQClientException, NoSuchFieldException, SecurityException, IllegalArgumentException,
        IllegalAccessException, InterruptedException, RemotingException, MQBrokerException {

    DefaultMQPullConsumer consumer = getReceiveCheckConsumer(nameSvr, cluster, address);
    Field f1 = DefaultMQPullConsumerImpl.class.getDeclaredField("mQClientFactory");
    f1.setAccessible(true);

    MQClientInstance instance = (MQClientInstance) f1.get(consumer.getDefaultMQPullConsumerImpl());

    Field f = MQClientInstance.class.getDeclaredField("brokerAddrTable");
    f.setAccessible(true);

    Field f2 = MQClientInstance.class.getDeclaredField("scheduledExecutorService");
    f2.setAccessible(true);

    ScheduledExecutorService service = (ScheduledExecutorService) f2.get(instance);
    service.shutdown();
    service.awaitTermination(1000, TimeUnit.SECONDS);

    ConcurrentHashMap<String, HashMap<Long, String>> map = (ConcurrentHashMap<String, HashMap<Long, String>>) f.get(instance);
    HashMap<Long, String> addresses = new HashMap<>();
    addresses.put(0L, address);
    map.put("rmqmonitor_" + address, addresses);

    MessageQueue queue = new MessageQueue("SELF_TEST_TOPIC", "rmqmonitor_" + address, 0);

    boolean pullOk = false;
    long maxOffset = -1;
    for (int i = 0; i < 2; ++i) {
        try {
            maxOffset = consumer.getDefaultMQPullConsumerImpl().maxOffset(queue);
            PullResult result = consumer.pull(queue, "*", maxOffset > 100 ? maxOffset - 10 : 0, 1);
            if (result.getPullStatus() == PullStatus.FOUND) {
                pullOk = true;
                break;
            } else if(result.getPullStatus() == PullStatus.NO_NEW_MSG) {
                checkSend(cluster, nameSvr, address);
                continue;
            }

            logger.warn("pull result failed, PullResult={}, cluster={}, namesvr={}, address={}", result, cluster, nameSvr, address);
        } catch (Throwable e) {
            logger.error("pull exception, cluster={}, namesvr={}, address={}", cluster, nameSvr, address, e);
        }
        Thread.sleep(1000);
    }
    if (!pullOk) {
        logger.error(String.format("[AlarmPullErr] cluster=%s, broker=%s", cluster, address));
    } else {
        logger.info("AlarmPullCheck cluster={}, broker={}", cluster, address);
    }
    return maxOffset;
}
 

private void finishDoStop(CompletableFuture<Void> future) {
    serverChannels.clear();

    if (config.shutdownBlockingTaskExecutorOnStop()) {
        final ScheduledExecutorService executor;
        final ScheduledExecutorService blockingTaskExecutor = config.blockingTaskExecutor();
        if (blockingTaskExecutor instanceof UnstoppableScheduledExecutorService) {
            executor =
                    ((UnstoppableScheduledExecutorService) blockingTaskExecutor).getExecutorService();
        } else {
            executor = blockingTaskExecutor;
        }

        try {
            executor.shutdown();
            while (!executor.isTerminated()) {
                try {
                    executor.awaitTermination(1, TimeUnit.DAYS);
                } catch (InterruptedException ignore) {
                    // Do nothing.
                }
            }
        } catch (Exception e) {
            logger.warn("Failed to shutdown the blockingTaskExecutor: {}", executor, e);
        }
    }

    final Builder<AccessLogWriter> builder = ImmutableSet.builder();
    config.virtualHosts()
          .stream()
          .filter(VirtualHost::shutdownAccessLogWriterOnStop)
          .forEach(virtualHost -> builder.add(virtualHost.accessLogWriter()));
    config.serviceConfigs()
          .stream()
          .filter(ServiceConfig::shutdownAccessLogWriterOnStop)
          .forEach(serviceConfig -> builder.add(serviceConfig.accessLogWriter()));

    final Set<AccessLogWriter> writers = builder.build();
    final List<CompletableFuture<Void>> completionFutures = new ArrayList<>(writers.size());
    writers.forEach(accessLogWriter -> {
        final CompletableFuture<Void> shutdownFuture = accessLogWriter.shutdown();
        shutdownFuture.exceptionally(cause -> {
            logger.warn("Failed to close the {}:", AccessLogWriter.class.getSimpleName(), cause);
            return null;
        });
        completionFutures.add(shutdownFuture);
    });

    CompletableFutures.successfulAsList(completionFutures, cause -> null)
                      .thenRunAsync(() -> future.complete(null), config.startStopExecutor());
}
 

private static void multiInstanceListen(ObjectMapper objectMapper, Listener<SalesOrderPlaced> listener) throws IOException {
    final HikariConfig hikariConfig = new HikariConfig();
    hikariConfig.setJdbcUrl(JDBC_URL);
    hikariConfig.setUsername(JDBC_USERNAME);
    hikariConfig.setPassword(JDBC_PASSWORD);

    final DataSource dataSource = new HikariDataSource(hikariConfig);

    final ZignAccessTokenProvider accessTokenProvider = new ZignAccessTokenProvider();

    final AtomicInteger name = new AtomicInteger();
    final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(16);

    for (int i = 0; i < 12; i++) {
        final String instanceName = "consumer-" + name.getAndIncrement();
        final JdbcPartitionManager partitionManager = new JdbcPartitionManager(dataSource, "fahrschein-demo");
        final JdbcCursorManager cursorManager = new JdbcCursorManager(dataSource, "fahrschein-demo");

        final NakadiClient nakadiClient = NakadiClient.builder(NAKADI_URI)
                .withAccessTokenProvider(accessTokenProvider)
                .withCursorManager(cursorManager)
                .build();

        final List<Partition> partitions = nakadiClient.getPartitions(SALES_ORDER_SERVICE_ORDER_PLACED);

        final IORunnable instance = () -> {

            final IORunnable runnable = () -> {
                final Optional<Lock> optionalLock = partitionManager.lockPartitions(SALES_ORDER_SERVICE_ORDER_PLACED, partitions, instanceName);

                if (optionalLock.isPresent()) {
                    final Lock lock = optionalLock.get();
                    try {
                        nakadiClient.stream(SALES_ORDER_SERVICE_ORDER_PLACED)
                                .withLock(lock)
                                .withObjectMapper(objectMapper)
                                .withStreamParameters(new StreamParameters().withStreamLimit(10))
                                .withBackoffStrategy(new NoBackoffStrategy())
                                .listen(SalesOrderPlaced.class, listener);
                    } finally {
                        partitionManager.unlockPartitions(lock);
                    }
                }
            };

            scheduledExecutorService.scheduleWithFixedDelay(runnable.unchecked(), 0, 1, TimeUnit.SECONDS);
        };
        scheduledExecutorService.submit(instance.unchecked());
    }

    try {
        Thread.sleep(60L*1000);
        scheduledExecutorService.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
}
 

@Test
public void testPutHeadChunkRunnableFileGrowth() throws Exception {
    File file = File.createTempFile("test", "");

    try (final FileOutputStream outputStream = new FileOutputStream(file)) {
        outputStream.write(new byte[]{0x65});

        UUID transferId = UUID.randomUUID();
        BlobTransferTarget blobTransferTarget = mock(BlobTransferTarget.class);
        TransportService transportService = mock(TransportService.class);
        DiscoveryNode discoveryNode = mock(DiscoveryNode.class);
        DigestBlob digestBlob = mock(DigestBlob.class);
        when(digestBlob.file()).thenReturn(file);

        ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(EsExecutors.daemonThreadFactory("blob-head"));
        try {
            scheduledExecutor.schedule(new Runnable() {
                @Override
                public void run() {
                    try {
                        outputStream.write(new byte[]{0x66, 0x67, 0x68, 0x69});
                    } catch (IOException ex) {
                        //pass
                    }
                }
            }, 800, TimeUnit.MILLISECONDS);
            PutHeadChunkRunnable runnable = new PutHeadChunkRunnable(
                digestBlob, 5, transportService, blobTransferTarget, discoveryNode, transferId
            );

            @SuppressWarnings("unchecked")
            TransportFuture<TransportResponse.Empty> result = mock(TransportFuture.class);

            when(transportService.submitRequest(
                eq(discoveryNode),
                eq(BlobHeadRequestHandler.Actions.PUT_BLOB_HEAD_CHUNK),
                any(TransportRequest.class),
                any(TransportRequestOptions.class),
                eq(EmptyTransportResponseHandler.INSTANCE_SAME)
            )).thenReturn(result);

            runnable.run();

            verify(blobTransferTarget).putHeadChunkTransferFinished(transferId);
            verify(transportService, times(2)).submitRequest(
                eq(discoveryNode),
                eq(BlobHeadRequestHandler.Actions.PUT_BLOB_HEAD_CHUNK),
                any(TransportRequest.class),
                any(TransportRequestOptions.class),
                eq(EmptyTransportResponseHandler.INSTANCE_SAME)
            );
        } finally {
            scheduledExecutor.awaitTermination(1, TimeUnit.SECONDS);
            scheduledExecutor.shutdownNow();
        }
    }
}
 

@Test
public void testAsyncDirFinderShuffledDirsCreatedPeriodically() throws Exception {
  String testDataDir = "target" + File.separatorChar + UUID.randomUUID().toString();
  Files.createDirectories(Paths.get(testDataDir));
  final Collection<Path> paths = Arrays.asList(
      Paths.get(testDataDir + File.separatorChar + "d1"),
      Paths.get(testDataDir + File.separatorChar + "d2"),
      Paths.get(testDataDir + File.separatorChar + "d3"),
      Paths.get(testDataDir + File.separatorChar + "d4"),
      Paths.get(testDataDir + File.separatorChar + "d5"),
      Paths.get(testDataDir + File.separatorChar + "d6"),
      Paths.get(testDataDir + File.separatorChar + "d7"),
      Paths.get(testDataDir + File.separatorChar + "d8"),

      Paths.get(testDataDir + File.separatorChar + "d1" + File.separatorChar + "d11"),
      Paths.get(testDataDir + File.separatorChar + "d1" + File.separatorChar + "d12"),

      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d21" + File.separatorChar + "d211"),
      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d22" + File.separatorChar + "d221"),
      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d23" + File.separatorChar + "d231"),

      Paths.get(testDataDir + File.separatorChar + "d3" + File.separatorChar + "d31"),

      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4112"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41112"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41111" + File.separatorChar + "d411111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41112" + File.separatorChar + "d411111")

  );
  DirectoryPathCreationWatcher watcher = new DirectoryPathCreationWatcher(paths, 1);

  ScheduledExecutorService directoryCreationExecutor = new SafeScheduledExecutorService(1, "Directory Creator");
  DirectoryCreationWorker worker = new DirectoryCreationWorker(paths, true);
  directoryCreationExecutor.scheduleAtFixedRate(worker, 0, 100, TimeUnit.MILLISECONDS);

  Set<Path> foundPaths = new HashSet<Path>();
  while (!worker.getRemainingPathsToCreate().isEmpty()) {
    Set<Path> foundPathsTillNow = watcher.find();
    foundPaths.addAll(foundPathsTillNow);
    Thread.sleep(100);
  }

  directoryCreationExecutor.shutdownNow();
  directoryCreationExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS);

  Thread.sleep(paths.size() * 100 + 1000);

  //Just to find the last set of detected directories.
  foundPaths.addAll(watcher.find());

  checkCorrectness(paths, foundPaths, (ScheduledExecutorService)Whitebox.getInternalState(watcher, "executor"));
}
 

/**
 * Test that a streaming task will work correctly when an extractor is continuously producing records and encounters
 * an error in the writer.
 *
 * The task should exit in a failed state.
 *
 * No converters
 * Identity fork
 * One writer
 * @throws Exception
 */
@Test
public void testContinuousTaskError()
    throws Exception {

  for (Boolean taskExecutionSync: new Boolean[]{true, false}) {
    ArrayList<Object> recordCollector = new ArrayList<>(100);
    long perRecordExtractLatencyMillis = 1000; // 1 second per record

    ContinuousExtractor continuousExtractor = new ContinuousExtractor(perRecordExtractLatencyMillis);

    TaskContext mockTaskContext = getMockTaskContext(recordCollector, continuousExtractor, taskExecutionSync, 5);

    // Create a mock TaskStateTracker
    TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class);

    // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread
    TaskExecutor taskExecutor = new TaskExecutor(new Properties());

    // Create the Task
    Task task = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch>absent());

    ScheduledExecutorService taskRunner = new ScheduledThreadPoolExecutor(1, ExecutorsUtils.newThreadFactory(Optional.of(log)));

    taskRunner.execute(task);

    // Let the task run for 10 seconds
    int sleepIterations = 10;
    int currentIteration = 0;

    while (currentIteration < sleepIterations) {
      Thread.sleep(1000);
      currentIteration++;
      Map<String, CheckpointableWatermark> externalWatermarkStorage = mockTaskContext.getWatermarkStorage()
          .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default"));
      if (!externalWatermarkStorage.isEmpty()) {
        for (CheckpointableWatermark watermark : externalWatermarkStorage.values()) {
          log.info("Observed committed watermark: {}", watermark);
        }
        log.info("Task progress: {}", task.getProgress());
        // Ensure that watermarks seem reasonable at each step
        Assert.assertTrue(continuousExtractor.validateWatermarks(false, externalWatermarkStorage));
      }
    }

    boolean success = task.awaitShutdown(30000);
    Assert.assertTrue(success, "Task should shutdown in 30 seconds");
    log.info("Task done waiting to shutdown {}", success);

    // Shutdown on error, so don't check for the exact watermark
    Assert.assertTrue(continuousExtractor.validateWatermarks(false, mockTaskContext.getWatermarkStorage()
        .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default"))));

    task.commit();

    // Task should be in failed state from extractor error
    Assert.assertEquals(mockTaskContext.getTaskState().getWorkingState(), WorkUnitState.WorkingState.FAILED);
    // Shutdown the executor
    taskRunner.shutdown();
    taskRunner.awaitTermination(100, TimeUnit.MILLISECONDS);
  }
}
 

@Override
public void stop(final boolean force) {
    writeLock.lock();
    try {

        if (!isRunning()) {
            return;
        }

        running.set(false);

        if (clusterCoordinator != null) {
            final Thread shutdownClusterCoordinator = new Thread(clusterCoordinator::shutdown);

            shutdownClusterCoordinator.setDaemon(true);
            shutdownClusterCoordinator.setName("Shutdown Cluster Coordinator");
            shutdownClusterCoordinator.start();
        }

        if (!controller.isTerminated()) {
            controller.shutdown(force);
        }

        if (configuredForClustering && senderListener != null) {
            try {
                senderListener.stop();
            } catch (final IOException ioe) {
                logger.warn("Protocol sender/listener did not stop gracefully due to: " + ioe);
            }
        }

        final ScheduledExecutorService executorService = executor.get();
        if (executorService != null) {
            if (force) {
                executorService.shutdownNow();
            } else {
                executorService.shutdown();
            }

            boolean graceful;
            try {
                graceful = executorService.awaitTermination(gracefulShutdownSeconds, TimeUnit.SECONDS);
            } catch (final InterruptedException e) {
                graceful = false;
            }

            if (!graceful) {
                logger.warn("Scheduling service did not gracefully shutdown within configured " + gracefulShutdownSeconds + " second window");
            }
        }
    } finally {
        writeLock.unlock();
    }
}