Java Code Examples for java.util.concurrent.ExecutorService#shutdownNow()

private static boolean shutdownExecutor(ExecutorService executor) {
    executor.shutdown();
    try {
        if (!executor.awaitTermination(TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
            executor.shutdownNow();

            return executor.awaitTermination(TIMEOUT_MS, TimeUnit.MILLISECONDS);
        }

        return true;
    } catch (InterruptedException ex) {
        executor.shutdownNow();
        Thread.currentThread().interrupt();
        logger.severe(() -> "Exception: " + ex);
    }
    return false;
}
 

@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
   	try {
		ServletContext servletContext = servletContextEvent.getServletContext();
		WebApplicationContext webApplicationContext = WebApplicationContextUtils.getWebApplicationContext(servletContext);
		ExecutorService workerExecutorService = (ExecutorService) webApplicationContext.getBean("WorkerExecutorService");
		Thread.sleep(1000);
		logger.info("Shutting down WorkerExecutorService");
		int retryCount = 0;
		while (!workerExecutorService.isTerminated() && retryCount < 10) {
			if (retryCount > 0) {
				logger.error("WorkerExecutorService shutdown retryCount: " + retryCount);
				Thread.sleep(1000);
			}
			workerExecutorService.shutdownNow();
			retryCount++;
		}
   	} catch (Exception e) {
		logger.error("Cannot shutdown WorkerExecutorService: " + e.getMessage(), e);
	}
}
 

private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3prefix = params.getRequired("s3prefix");
	final String s3filePrefix = params.get("s3filePrefix", "");
	int parallelism = params.getInt("parallelism", 10);

	List<String> files = listByFullPathPrefix(bucket, s3prefix);

	ExecutorService executor = Executors.newFixedThreadPool(parallelism);
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	List<CompletableFuture<Integer>> requests =
		submitLineCountingRequestsForFilesAsync(executor, s3client, bucket, files, s3filePrefix);
	int count = waitAndComputeTotalLineCountResult(requests);

	executor.shutdownNow();
	s3client.shutdown();
	System.out.print(count);
}
 

private static boolean shutdownExecutor(ExecutorService executor) {
    executor.shutdown();
    try {
        if (!executor.awaitTermination(TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
            executor.shutdownNow();

            return executor.awaitTermination(TIMEOUT_MS, TimeUnit.MILLISECONDS);
        }

        return true;
    } catch (InterruptedException ex) {
        executor.shutdownNow();
        Thread.currentThread().interrupt();
        logger.severe(() -> "Exception: " + ex);
    }
    return false;
}
 

/**
 * Immediate shutdown. Any transfer in process will be interrupted. It is
 * safe to invoke this method whether or not the service is running.
 * <p>
 * Note: This class exposes its synchronization mechanism to
 * {@link HAReceiveService}.
 */
synchronized public void terminate() {
    if (log.isInfoEnabled())
        log.info(toString() + " : stopping.");
    final ExecutorService tmp = executorRef.getAndSet(null);
    if (tmp == null) {
        // Not running.
        if (log.isInfoEnabled())
            log.info("Service was not running.");
        return;
    }
    try {
        closeSocketChannelNoBlock();
    } finally {
        // shutdown executor.
        tmp.shutdownNow();
        // clear address.
        addrNext.set(null);
        if (log.isInfoEnabled())
            log.info(toString() + " : stopped.");
    }
}
 

private Map<String, TableMetaData> parallelLoadTables(final DatabaseType databaseType, final Map<String, DataSource> dataSourceMap, final DataNodes dataNodes, 
                                                      final String tableName, final int maxConnectionsSizePerQuery) {
    Map<String, List<DataNode>> dataNodeGroups = dataNodes.getDataNodeGroups(tableName);
    Map<String, TableMetaData> result = new HashMap<>(dataNodeGroups.size(), 1);
    Map<String, Future<Optional<TableMetaData>>> tableFutureMap = new HashMap<>(dataNodeGroups.size(), 1);
    ExecutorService executorService = Executors.newFixedThreadPool(Math.min(CPU_CORES * 2, dataNodeGroups.size() * maxConnectionsSizePerQuery));
    for (Entry<String, List<DataNode>> entry : dataNodeGroups.entrySet()) {
        for (DataNode each : entry.getValue()) {
            Future<Optional<TableMetaData>> futures = executorService.submit(() -> loadTableByDataNode(each, databaseType, dataSourceMap));
            tableFutureMap.put(each.getTableName(), futures);
        }
    }
    tableFutureMap.forEach((key, value) -> {
        try {
            getTableMetaData(value).ifPresent(tableMetaData -> result.put(key, tableMetaData));
        } catch (final InterruptedException | ExecutionException | TimeoutException ex) {
            throw new IllegalStateException(String.format("Error while fetching tableMetaData with key= %s and Value=%s", key, value), ex);
        }
    });
    executorService.shutdownNow();
    return result;
}
 

public static void main(String[] args) throws InterruptedException, IOException {
    String master = "https://localhost:8443/";
    String podName = null;

    if (args.length == 2) {
        master = args[0];
        podName = args[1];
    }
    if (args.length == 1) {
        podName = args[0];
    }

    Config config = new ConfigBuilder().withMasterUrl(master).build();
    ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (
            KubernetesClient client = new DefaultKubernetesClient(config);
            ExecWatch watch = client.pods().withName(podName)
                    .redirectingInput()
                    .redirectingOutput()
                    .redirectingError()
                    .redirectingErrorChannel()
                    .exec();
            InputStreamPumper pump = new InputStreamPumper(watch.getOutput(), new SystemOutCallback()))
    {

        executorService.submit(pump);
        watch.getInput().write("ls -al\n".getBytes());
        Thread.sleep(5 * 1000);
    } catch (Exception e) {
        throw KubernetesClientException.launderThrowable(e);
    } finally {
        executorService.shutdownNow();
    }
}
 

@Test
public void testFutureResult() throws Exception {

    final FutureTask<Integer> futureTask = new FutureTask<Integer>(new Callable<Integer>() {
        @Override
        public Integer call() throws Exception {
            return 11;
        }
    });

    LastOperationMonitor opMonitor = new LastOperationMonitor();
    FutureOperationalResultImpl<Integer> futureResult =
            new FutureOperationalResultImpl<Integer>("test", futureTask, System.currentTimeMillis(), opMonitor);

    ExecutorService threadPool = Executors.newSingleThreadExecutor();

    threadPool.submit(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            Thread.sleep(400);
            futureTask.run();
            return null;
        }
    });

    OperationResult<Integer> opResult = futureResult.get();
    int integerResult = opResult.getResult();
    long latency = opResult.getLatency();

    Assert.assertEquals(11, integerResult);
    Assert.assertTrue(latency >= 400);

    threadPool.shutdownNow();
}
 

private void shutDownExecutor(final ExecutorService executorService) {
    executorService.shutdown();
    try {
        executorService.awaitTermination(2, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        LOG.warn("Timeout exceeded waiting for saga lib orderly executor service shutdown, forcing shutdown.", e);
        executorService.shutdownNow();
    }
}
 

private static void assertConcurrent(final String message, final List<? extends Runnable> runnables, final int maxTimeoutSeconds) throws InterruptedException {
    final int numThreads = runnables.size();
    final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<>());
    final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);
    try {
        final CountDownLatch allExecutorThreadsReady = new CountDownLatch(numThreads);
        final CountDownLatch afterInitBlocker = new CountDownLatch(1);
        final CountDownLatch allDone = new CountDownLatch(numThreads);
        for (final Runnable submittedTestRunnable : runnables) {
            threadPool.submit(() -> {
                allExecutorThreadsReady.countDown();
                try {
                    afterInitBlocker.await();
                    submittedTestRunnable.run();
                } catch (final Throwable e) {
                    exceptions.add(e);
                } finally {
                    allDone.countDown();
                }
            });
        }
        // wait until all threads are ready
        assertTrue("Timeout initializing threads! Perform long lasting initializations before passing runnables to assertConcurrent", allExecutorThreadsReady.await(runnables.size() * 10, TimeUnit.MILLISECONDS));
        // start all test runners
        afterInitBlocker.countDown();
        assertTrue(message + " timeout! More than" + maxTimeoutSeconds + "seconds", allDone.await(maxTimeoutSeconds, TimeUnit.SECONDS));
    } finally {
        threadPool.shutdownNow();
    }
    assertTrue(message + "failed with exception(s)" + exceptions, exceptions.isEmpty());
}
 

@Test(groups="Integration") // because takes 1 sec
public void testCreateWithMaxConcurrentCallsAppliesToSubLocations() throws Exception {
    final int numCalls = 4;
    final int maxConcurrentCreations = 2;
    ConcurrencyTracker interceptor = new ConcurrencyTracker();
    ExecutorService executor = Executors.newCachedThreadPool();

    try {
        final BailOutJcloudsLocation jcloudsLocation = BailOutJcloudsLocation.newBailOutJcloudsLocation(
                managementContext, ImmutableMap.of(
                        BailOutJcloudsLocation.BUILD_TEMPLATE_INTERCEPTOR, interceptor,
                        MAX_CONCURRENT_MACHINE_CREATIONS, maxConcurrentCreations));

        for (int i = 0; i < numCalls; i++) {
            final BailOutJcloudsLocation subLocation = (BailOutJcloudsLocation) jcloudsLocation.newSubLocation(MutableMap.of());
            executor.execute(new Runnable() {
                @Override
                public void run() {
                    subLocation.tryObtain();
                }
            });
        }

        interceptor.assertCallCountEventually(maxConcurrentCreations);
        interceptor.assertCallCountContinually(maxConcurrentCreations);

        interceptor.unblock();
        interceptor.assertCallCountEventually(numCalls);
        executor.shutdown();
        executor.awaitTermination(10, TimeUnit.SECONDS);

    } finally {
        executor.shutdownNow();
    }
}
 

/**
 * Tests the get() method if waiting for the initialization is interrupted.
 */
public void testGetInterruptedException() throws Exception {
    ExecutorService exec = Executors.newSingleThreadExecutor();
    final BackgroundInitializerTestImpl init = new BackgroundInitializerTestImpl(
            exec);
    final CountDownLatch latch1 = new CountDownLatch(1);
    init.shouldSleep = true;
    init.start();
    final AtomicReference<InterruptedException> iex = new AtomicReference<InterruptedException>();
    Thread getThread = new Thread() {
        @Override
        public void run() {
            try {
                init.get();
            } catch (ConcurrentException cex) {
                if (cex.getCause() instanceof InterruptedException) {
                    iex.set((InterruptedException) cex.getCause());
                }
            } finally {
                assertTrue("Thread not interrupted", isInterrupted());
                latch1.countDown();
            }
        }
    };
    getThread.start();
    getThread.interrupt();
    latch1.await();
    exec.shutdownNow();
    exec.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
    assertNotNull("No interrupted exception", iex.get());
}
 

public static void main(String[] args) throws Exception {
    List<NetworkInterface> toTest = getNetworkInterfacesAsStream()
                    .filter(hasHardwareAddress)
                    .collect(Collectors.toList());

    ExecutorService executor = Executors.newFixedThreadPool(NUM_THREADS);

    for (NetworkInterface ni : toTest) {
        Phaser startingGate = new Phaser(NUM_THREADS);
        System.out.println("Testing: " + ni.getName());
        List<Callable<Exception>> list = new ArrayList<>();
        for (int i = 0; i < NUM_THREADS; i++)
            list.add(new GetMacAddress(ni, ni.getName() + "-Thread-" + i, startingGate));
        List<Future<Exception>> futures = executor.invokeAll(list);
        for (Future<Exception> f : futures) {
            if (f.get() != null)
                f.get().printStackTrace(System.out);
        }
        if (failed)
            break;
    }
    executor.shutdownNow();
    if (!failed) {
        System.out.println("PASSED - Finished all threads");
    } else {
        throw new RuntimeException("Failed");
    }
}
 

protected void shutdownHeartbeat( ExecutorService heartbeat ) {

    if ( heartbeat != null ) {

      try {
        heartbeat.shutdownNow(); // prevents waiting tasks from starting and attempts to stop currently executing ones

      } catch ( Throwable t ) {
        /* do nothing */
      }
    }
  }
 

private void shutdownNowAndAwaitTermination(ExecutorService executorService) {
	executorService.shutdownNow();
	try {
		if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) {
			LOG.warn("More than 10s to terminate the service executor");
		}
	} catch (InterruptedException e) {
		LOG.warn("Unable to interrupt the service executor", e);
		Thread.currentThread().interrupt();
	}
}
 

@Test(timeOut = 2000)
public void testAsyncProducerAndConsumerWithZeroQueueSize() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int totalMsg = 100;
    final Set<String> produceMsgs = Sets.newHashSet();
    final Set<String> consumeMsgs = Sets.newHashSet();

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();

    // produce message
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        produceMsgs.add(message);
    }

    log.info(" start receiving messages :");
    CountDownLatch latch = new CountDownLatch(totalMsg);
    // receive messages
    ExecutorService executor = Executors.newFixedThreadPool(1);
    receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);

    latch.await();

    // verify message produced correctly
    assertEquals(produceMsgs.size(), totalMsg);
    // verify produced and consumed messages must be exactly same
    produceMsgs.removeAll(consumeMsgs);
    assertTrue(produceMsgs.isEmpty());

    producer.close();
    consumer.close();
    executor.shutdownNow();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testConcurrentAccessToSystemCredentials(){
  final Map<ApplicationId, ByteBuffer> testCredentials = new HashMap<>();
  ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[300]);
  ApplicationId applicationId = ApplicationId.newInstance(123456, 120);
  testCredentials.put(applicationId, byteBuffer);

  final List<Throwable> exceptions = Collections.synchronizedList(new
      ArrayList<Throwable>());

  final int NUM_THREADS = 10;
  final CountDownLatch allDone = new CountDownLatch(NUM_THREADS);
  final ExecutorService threadPool = Executors.newFixedThreadPool(
      NUM_THREADS);

  final AtomicBoolean stop = new AtomicBoolean(false);

  try {
    for (int i = 0; i < NUM_THREADS; i++) {
      threadPool.submit(new Runnable() {
        @Override
        public void run() {
          try {
            for (int i = 0; i < 100 && !stop.get(); i++) {
              NodeHeartbeatResponse nodeHeartBeatResponse =
                  newNodeHeartbeatResponse(0, NodeAction.NORMAL,
                      null, null, null, null, 0);
              nodeHeartBeatResponse.setSystemCredentialsForApps(
                  testCredentials);
              NodeHeartbeatResponseProto proto =
                  ((NodeHeartbeatResponsePBImpl)nodeHeartBeatResponse)
                      .getProto();
              Assert.assertNotNull(proto);
            }
          } catch (Throwable t) {
            exceptions.add(t);
            stop.set(true);
          } finally {
            allDone.countDown();
          }
        }
      });
    }

    int testTimeout = 2;
    Assert.assertTrue("Timeout waiting for more than " + testTimeout + " " +
            "seconds",
        allDone.await(testTimeout, TimeUnit.SECONDS));
  } catch (InterruptedException ie) {
    exceptions.add(ie);
  } finally {
    threadPool.shutdownNow();
  }
  Assert.assertTrue("Test failed with exception(s)" + exceptions,
      exceptions.isEmpty());
}
 

private static void doConcurrentPurgesAndUpdatesTest() throws Exception {

    final CountDownLatch startUpdating = new CountDownLatch(1);
    final CountDownLatch finishUpdating = new CountDownLatch(1);

    try (final Monitor monitor = new Monitor(ANALYZER)) {
      Runnable updaterThread = () -> {
        try {
          startUpdating.await();
          for (int i = 200; i < 400; i++) {
            monitor.register(newMonitorQuery(i));
          }
          finishUpdating.countDown();
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      };

      ExecutorService executor = Executors.newFixedThreadPool(1, new NamedThreadFactory("updaters"));
      try {
        executor.submit(updaterThread);

        for (int i = 0; i < 200; i++) {
          monitor.register(newMonitorQuery(i));
        }
        for (int i = 20; i < 80; i++) {
          monitor.deleteById(Integer.toString(i));
        }

        assertEquals(200, monitor.getQueryCacheStats().cachedQueries);

        startUpdating.countDown();
        monitor.purgeCache();
        finishUpdating.await();

        assertEquals(340, monitor.getQueryCacheStats().cachedQueries);
        Document doc = new Document();
        doc.add(newTextField("field", "test", Field.Store.NO));
        MatchingQueries<QueryMatch> matcher = monitor.match(doc, QueryMatch.SIMPLE_MATCHER);
        assertEquals(0, matcher.getErrors().size());
        assertEquals(340, matcher.getMatchCount());
      } finally {
        executor.shutdownNow();
      }
    }
  }
 

/**
 * [FLINK-6020]
 * Tests that concurrent put operations will only upload the file once to the {@link BlobStore}
 * and that the files are not corrupt at any time.
 *
 * @param jobId
 * 		job ID to use (or <tt>null</tt> if job-unrelated)
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 */
private void testConcurrentPutOperations(
		@Nullable final JobID jobId, final BlobKey.BlobType blobType)
		throws IOException, InterruptedException, ExecutionException {
	final Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	BlobStore blobStore = mock(BlobStore.class);
	int concurrentPutOperations = 2;
	int dataSize = 1024;

	final CountDownLatch countDownLatch = new CountDownLatch(concurrentPutOperations);
	final byte[] data = new byte[dataSize];

	ArrayList<CompletableFuture<BlobKey>> allFutures = new ArrayList<>(concurrentPutOperations);

	ExecutorService executor = Executors.newFixedThreadPool(concurrentPutOperations);

	try (final BlobServer server = new BlobServer(config, blobStore)) {

		server.start();

		for (int i = 0; i < concurrentPutOperations; i++) {
			CompletableFuture<BlobKey> putFuture = CompletableFuture
				.supplyAsync(
					() -> {
						try {
							BlockingInputStream inputStream =
								new BlockingInputStream(countDownLatch, data);
							BlobKey uploadedKey = put(server, jobId, inputStream, blobType);
							// check the uploaded file's contents (concurrently)
							verifyContents(server, jobId, uploadedKey, data);
							return uploadedKey;
						} catch (IOException e) {
							throw new CompletionException(new FlinkException(
								"Could not upload blob.", e));
						}
					},
					executor);

			allFutures.add(putFuture);
		}

		FutureUtils.ConjunctFuture<Collection<BlobKey>> conjunctFuture = FutureUtils.combineAll(allFutures);

		// wait until all operations have completed and check that no exception was thrown
		Collection<BlobKey> blobKeys = conjunctFuture.get();

		Iterator<BlobKey> blobKeyIterator = blobKeys.iterator();

		assertTrue(blobKeyIterator.hasNext());

		BlobKey blobKey = blobKeyIterator.next();

		// make sure that all blob keys are the same
		while (blobKeyIterator.hasNext()) {
			verifyKeyDifferentHashEquals(blobKey, blobKeyIterator.next());
		}

		// check the uploaded file's contents
		verifyContents(server, jobId, blobKey, data);

		// check that we only uploaded the file once to the blob store
		if (blobType == PERMANENT_BLOB) {
			verify(blobStore, times(1)).put(any(File.class), eq(jobId), eq(blobKey));
		} else {
			// can't really verify much in the other cases other than that the put operations should
			// work and not corrupt files
			verify(blobStore, times(0)).put(any(File.class), eq(jobId), eq(blobKey));
		}
	} finally {
		executor.shutdownNow();
	}
}
 

@Override
public void build(boolean parallel, List<V> list, DistanceMetric dm)
{
    setDistanceMetric(dm);
    if(!dm.isSubadditive())
        throw new RuntimeException("VPTree only supports metrics that support the triangle inequality");
    Random rand = RandomUtil.getRandom();

    this.size = list.size();
    this.allVecs = list;
    distCache = dm.getAccelerationCache(allVecs, parallel);
    //Use simple list so both halves can be modified simultaniously
    List<Pair<Double, Integer>> tmpList = new SimpleList<>(list.size());
    for(int i = 0; i < allVecs.size(); i++)
        tmpList.add(new Pair<>(-1.0, i));
    if(!parallel)
        this.root = makeVPTree(tmpList);
    else
    {
        ExecutorService threadpool = ParallelUtils.getNewExecutor(parallel);
        ModifiableCountDownLatch mcdl = new ModifiableCountDownLatch(1);
        this.root = makeVPTree(tmpList, threadpool, mcdl);
        mcdl.countDown();
        try
        {
            mcdl.await();
        }
        catch (InterruptedException ex)
        {
            Logger.getLogger(SVPTree.class.getName()).log(Level.SEVERE, null, ex);
            System.err.println("Falling back to single threaded VPTree constructor");
            tmpList.clear();
            for(int i = 0; i < list.size(); i++)
                tmpList.add(new Pair<>(-1.0, i));
            this.root = makeVPTree(tmpList);
        }
        finally
        {
            threadpool.shutdownNow();
        }
    }
}