Java Code Examples for java.util.concurrent.CompletionService#submit()

@Test
@Timeout(30)
void testConcurrentlyCreateProxyClasses() throws Exception {
	ExecutorService executorService = Executors.newFixedThreadPool(4);
	try {
		CompletionService<Void> completionService = new ExecutorCompletionService<>(executorService);
		for (int i = 0; i < 4; i++) {
			completionService.submit(() -> {
				for (int r = 0; r < 100; r++) {
					PropertyUtils.getPropertyDescriptor(TestEntity.class, TestEntity::getNumber);
					PropertyUtils.clearCache();
				}
				return null;
			});
		}
		for (int i = 0; i < 4; i++) {
			completionService.take().get();
		}
	} finally {
		executorService.shutdown();
	}
}
 

/**
 * 调用服务端
 * 
 * @param port
 * @param multiSize
 *            并发数
 * @param invokeNum
 *            总请求数
 * @param size
 *            batch请求的数据内含的list数量
 * @param textLength
 *            batch请求数据中随机字符串的长度
 * @throws Exception
 */
public void run(int port, int multiSize, int invokeNum, int size, int textLength)
        throws Exception {
    PbrpcClient client = PbrpcClientFactory.buildPooledConnection(new PooledConfiguration(),
            "127.0.0.1", port, 60000);
    ExecutorService pool = Executors.newFixedThreadPool(multiSize);
    CompletionService<DemoBatchResponse> completionService = new ExecutorCompletionService<DemoBatchResponse>(
            pool);

    BatchInvoker invoker = new BatchInvoker(client, size,
            RandomUtils.generateString(textLength));
    long time = System.currentTimeMillis();
    for (int i = 0; i < invokeNum; i++) {
        completionService.submit(invoker);
    }

    for (int i = 0; i < invokeNum; i++) {
        completionService.take().get();
    }

    long timetook = System.currentTimeMillis() - time;
    LOG.info("Send " + invokeNum + " requests using " + timetook + "ms");
    LOG.info("QPS:" + 1000f / ((timetook) / (1.0f * invokeNum)));
}
 

/**
 * poll returns non-null when the returned task is completed
 */
public void testPoll1()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    assertNull(cs.poll());
    cs.submit(new StringTask());

    long startTime = System.nanoTime();
    Future f;
    while ((f = cs.poll()) == null) {
        if (millisElapsedSince(startTime) > LONG_DELAY_MS)
            fail("timed out");
        Thread.yield();
    }
    assertTrue(f.isDone());
    assertSame(TEST_STRING, f.get());
}
 

/**
 * GET urls in parallel using the executor service.
 * @param urls absolute URLs to GET
 * @param timeoutMs timeout in milliseconds for each GET request
 * @return instance of CompletionService. Completion service will provide
 *   results as they arrive. The order is NOT same as the order of URLs
 */
public CompletionService<GetMethod> execute(List<String> urls, int timeoutMs) {
  HttpClientParams clientParams = new HttpClientParams();
  clientParams.setConnectionManagerTimeout(timeoutMs);
  HttpClient client = new HttpClient(clientParams, _connectionManager);

  CompletionService<GetMethod> completionService = new ExecutorCompletionService<>(_executor);
  for (String url : urls) {
    completionService.submit(() -> {
      try {
        GetMethod getMethod = new GetMethod(url);
        getMethod.getParams().setSoTimeout(timeoutMs);
        client.executeMethod(getMethod);
        return getMethod;
      } catch (Exception e) {
        // Log only exception type and message instead of the whole stack trace
        LOGGER.warn("Caught '{}' while executing GET on URL: {}", e.toString(), url);
        throw e;
      }
    });
  }
  return completionService;
}
 

/**
 * successful and failed tasks are both returned
 */
public void testTaskAssortment()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    ArithmeticException ex = new ArithmeticException();
    for (int i = 0; i < 2; i++) {
        cs.submit(new StringTask());
        cs.submit(callableThrowing(ex));
        cs.submit(runnableThrowing(ex), null);
    }
    int normalCompletions = 0;
    int exceptionalCompletions = 0;
    for (int i = 0; i < 3 * 2; i++) {
        try {
            if (cs.take().get() == TEST_STRING)
                normalCompletions++;
        }
        catch (ExecutionException expected) {
            assertTrue(expected.getCause() instanceof ArithmeticException);
            exceptionalCompletions++;
        }
    }
    assertEquals(2 * 1, normalCompletions);
    assertEquals(2 * 2, exceptionalCompletions);
    assertNull(cs.poll());
}
 

@Override
public ImmutableList<Profile> findAllByName(Iterable<String> names) throws IOException, InterruptedException {
    CompletionService<List<Profile>> completion = new ExecutorCompletionService<>(executorService);
    int count = 0;
    for (final List<String> partition : Iterables.partition(names, getEffectiveProfilesPerJob())) {
        count++;
        completion.submit(() -> resolver.findAllByName(partition));
    }

    Builder<Profile> builder = ImmutableList.builder();
    for (int i = 0; i < count; i++) {
        try {
            builder.addAll(completion.take().get());
        } catch (ExecutionException e) {
            if (e.getCause() instanceof IOException) {
                throw (IOException) e.getCause();
            } else {
                throw new RuntimeException("Error occurred during the operation", e);
            }
        }
    }
    return builder.build();
}
 

@Test
public void testCompletionServiceRunnableCaptures() throws InterruptedException, Exception {
    // Setup
    ExecutorService executor = Executors.newCachedThreadPool();
    CompletionService<Object> delegate = new ExecutorCompletionService<>(executor);
    CompletionService<Object> cs = StateCapture.capturingDecorator(delegate);

    CapturedState mockCapturedState = mock(CapturedState.class);
    Runnable mockRunnable = mock(Runnable.class);
    ThreadLocalStateCaptor.THREAD_LOCAL.set(mockCapturedState);
    Object result = new Object();
    Future<Object> futureResult = cs.submit(mockRunnable, result);
    assertThat("Expected the delegate response to return",
        result, sameInstance(futureResult.get()));
    executor.shutdown();

    verifyStandardCaptures(mockCapturedState, mockRunnable);
}
 

public void testPool() throws Exception {
    PbrpcClient client = PbrpcClientFactory.buildPooledConnection(new PooledConfiguration(),
            "127.0.0.1", 8088, 4000);

    PbrpcMsg msg;
    msg = new PbrpcMsg();
    msg.setServiceId(100);
    msg.setProvider("beidou");
    msg.setData(getData(1));
    DemoResponse res = client.asyncTransport(DemoResponse.class, msg).get();
    System.out.println(res);

    int multiSize = 12;
    int totalRequestSize = 100000;
    ExecutorService pool = Executors.newFixedThreadPool(multiSize);
    CompletionService<DemoResponse> completionService = new ExecutorCompletionService<DemoResponse>(
            pool);

    Invoker invoker = new Invoker(client);
    long time = System.currentTimeMillis();
    for (int i = 0; i < totalRequestSize; i++) {
        completionService.submit(invoker);
    }

    for (int i = 0; i < totalRequestSize; i++) {
        completionService.take().get();
    }

    long timetook = System.currentTimeMillis() - time;
    LOG.info("Total using " + timetook + "ms");
    LOG.info("QPS:" + 1000f / ((timetook) / (1.0f * totalRequestSize)));
}
 

public static void main(String[] args) throws Exception{
   ExecutorService service=Executors.newCachedThreadPool();
   CompletionService<String> completion=new ExecutorCompletionService<String>(service);
   for(int i=0;i<10;i++){
    completion.submit(new MyCompletionService(i));
   }
   for(int i=0;i<10;i++){
    System.out.println(completion.take().get());
   }
   service.shutdown();
}
 

/**
 * Take returns the same future object returned by submit
 */
public void testTake2() throws InterruptedException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    Future f1 = cs.submit(new StringTask());
    Future f2 = cs.take();
    assertSame(f1, f2);
}
 

protected Boolean executeByServiceThreadPool(CompletionService<Boolean> cService, I context, Collection<IProcessor<I, Boolean>> processors) throws InterruptedException, ExecutionException {
	Boolean results = true;
	for (IProcessor<I, Boolean> processor : processors) {
		cService.submit(new Callable<Boolean>() {
			@Override
			public Boolean call() throws Exception {
				if (processor.isEnabled()) {
					try {
						return processor.process(context);
					} catch (Exception ex) {
						logger.error("executeByServiceThreadPool", ex);
						return false;
					}
				} else {
					logger.info(String.format("processor:%s,%s", processor.getClass().getName(), processor.isEnabled()));
				}
				return true;
			}
		});
	}
	for (int i = 0; i < processors.size(); i++) {
		results = results && cService.take().get();
		if (!results) {
			break;
		}
	}
	return results;
}
 

protected void parallelRequests(final AmazonS3 s3,
                                final String bucket,
                                final String key,
                                final Supplier<IOFunction<String, List<PartETag>>> operations)
{
  InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(bucket, key);
  String uploadId = s3.initiateMultipartUpload(initiateRequest).getUploadId();

  CompletionService<List<PartETag>> completionService = new ExecutorCompletionService<>(executorService);
  try {
    for (int i = 0; i < parallelism; i++) {
      completionService.submit(() -> operations.get().apply(uploadId));
    }

    List<PartETag> partETags = new ArrayList<>();
    for (int i = 0; i < parallelism; i++) {
      partETags.addAll(completionService.take().get());
    }

    s3.completeMultipartUpload(new CompleteMultipartUploadRequest()
        .withBucketName(bucket)
        .withKey(key)
        .withUploadId(uploadId)
        .withPartETags(partETags));
  }
  catch (InterruptedException interrupted) {
    s3.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
    Thread.currentThread().interrupt();
  }
  catch (CancellationException | ExecutionException ex) {
    s3.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
    throw new BlobStoreException(
        format("Error executing parallel requests for bucket:%s key:%s with uploadId:%s", bucket, key, uploadId), ex,
        null);
  }
}
 

@Override
public void findAllByName(Iterable<String> names, final Predicate<Profile> consumer) throws IOException, InterruptedException {
    CompletionService<Object> completion = new ExecutorCompletionService<>(executorService);
    int count = 0;
    for (final List<String> partition : Iterables.partition(names, getEffectiveProfilesPerJob())) {
        count++;
        completion.submit(() -> {
            resolver.findAllByName(partition, consumer);
            return null;
        });
    }

    Throwable throwable = null;
    for (int i = 0; i < count; i++) {
        try {
            completion.take().get();
        } catch (ExecutionException e) {
            throwable = e.getCause();
        }
    }

    if (throwable != null) {
        if (throwable instanceof IOException) {
            throw (IOException) throwable;
        } else {
            throw new RuntimeException("Error occurred during the operation", throwable);
        }
    }
}
 

/**
 * ecs.submit(null, val) throws NullPointerException
 */
public void testSubmitNullRunnable() {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    try {
        cs.submit((Runnable) null, Boolean.TRUE);
        shouldThrow();
    } catch (NullPointerException success) {}
}
 

public DoubleMatrix2D pairwiseCityBlockDoubleMatrix(double[][] in) {
    ExecutorService threadPool = Executors.newFixedThreadPool(nrThreads);
    CompletionService<Pair<Integer, double[]>> pool = new ExecutorCompletionService<Pair<Integer, double[]>>(threadPool);
    
    for (int i = 0; i < in.length; i++) {
        ConcurrentCityBlockTask task = new ConcurrentCityBlockTask(in, i);
        pool.submit(task);
    }

    int returned = 0;
    
    DoubleMatrix2D cityBlockDistanceMatrix;
    if((in.length * (long) in.length) > (Integer.MAX_VALUE - 2)){
        cityBlockDistanceMatrix = new DenseLargeDoubleMatrix2D(in.length, in.length);
    } else {
        cityBlockDistanceMatrix = new DenseDoubleMatrix2D(in.length, in.length);
    } 
    
    
    ProgressBar pb = new ProgressBar(in.length, "Calculation of City-Block Distance matrix: " + in.length + " x " + in.length);
    while (returned < in.length) {
        try {
            Pair<Integer, double[]> result = pool.take().get();
            if (result != null) {
                int rownr = result.getLeft(); //  < 0 when row is not to be included because of hashProbesToInclude.
                if (rownr >= 0) {
                    double[] doubles = result.getRight();
                    for(int i=0; i<doubles.length; ++i){
                        cityBlockDistanceMatrix.setQuick(rownr, i, doubles[i]);
                    }
                }
                result = null;
                returned++;
                pb.iterate();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    for(int r=1;r<cityBlockDistanceMatrix.rows(); r++){
        for(int c=0; c<r; c++){
            cityBlockDistanceMatrix.setQuick(r, c, cityBlockDistanceMatrix.getQuick(c, r));
        }
    }
    
    threadPool.shutdown();
    pb.close();
    return cityBlockDistanceMatrix;
}
 

/**
 * @param finalCheck
 *            If the internal nodePtrTbl should be restored for a subsequent
 *            liveness check. If this is the final/last check, it's pointless
 *            to re-create the nodePtrTable.
 */
protected int check0(final ITool tool, final boolean finalCheck) throws InterruptedException, IOException {
	final long startTime = System.currentTimeMillis();
	
	// Sum up the number of nodes in all disk graphs to indicate the amount
	// of work to be done by liveness checking.
	long sum = 0L;
	for (int i = 0; i < checker.length; i++) {
		sum += checker[i].getDiskGraph().size();
	}
	MP.printMessage(EC.TLC_CHECKING_TEMPORAL_PROPS, new String[] { finalCheck ? "complete" : "current",
			Long.toString(sum), checker.length == 1 ? "" : checker.length + " branches of " });

	// Copy the array of checkers into a concurrent-enabled queue
	// that allows LiveWorker threads to easily get the next 
	// LiveChecker to work on. We don't really need the FIFO
	// ordering of the BlockingQueue, just its support for removing
	// elements concurrently.
	//
	// Logically the queue is the unit of work the group of LiveWorkers
	// has to complete. Once the queue is empty, all work is done and
	// the LiveWorker threads will terminate.
	//
	// An alternative implementation could partition the array of
	// LiveChecker a-priori and assign one partition to each thread.
	// However, that assumes the work in all partitions is evenly
	// distributed, which is not necessarily true.
	final BlockingQueue<ILiveChecker> queue = new ArrayBlockingQueue<ILiveChecker>(checker.length);
	queue.addAll(Arrays.asList(checker));

	
	/*
	 * A LiveWorker below can either complete a unit of work a) without finding a
	 * liveness violation, b) finds a violation, or c) fails to check because of an
	 * exception/error (such as going out of memory). In case an LW fails to check,
	 * we still wait for all other LWs to complete. A subset of the LWs might have
	 * found a violation. In other words, the OOM of an LW has lower precedence than
	 * a violation found by another LW. However, if any LW fails to check, we terminate
	 * model checking after all LWs completed.
	 */
	final int wNum = TLCGlobals.doSequentialLiveness() ? 1 : Math.min(checker.length, TLCGlobals.getNumWorkers());
	final ExecutorService pool = Executors.newFixedThreadPool(wNum);
	// CS is really just a container around the set of Futures returned by the pool. It saves us from
	// creating a low-level array.
	final CompletionService<Boolean> completionService = new ExecutorCompletionService<Boolean>(pool);

	for (int i = 0; i < wNum; i++) {
		completionService.submit(new LiveWorker(tool, i, wNum, this, queue, finalCheck));
	}
	// Wait for all LWs to complete.
	pool.shutdown();
	pool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); // wait forever

	// Check if any one of the LWs found a violation (ignore failures for now).
	ExecutionException ee = null;
	for (int i = 0; i < wNum; i++) {
		try {
			final Future<Boolean> future = completionService.take();
			if (future.get()) {
				MP.printMessage(EC.TLC_CHECKING_TEMPORAL_PROPS_END,
						TLC.convertRuntimeToHumanReadable(System.currentTimeMillis() - startTime));
				return EC.TLC_TEMPORAL_PROPERTY_VIOLATED;
			}
		} catch (final ExecutionException e) {
			// handled below!
			ee = e;
		}
	}
	// Terminate if any one of the LWs failed c)
	if (ee != null) {
		final Throwable cause = ee.getCause();
		if (cause instanceof OutOfMemoryError) {
			MP.printError(EC.SYSTEM_OUT_OF_MEMORY_LIVENESS, cause);
		} else if (cause instanceof StackOverflowError) {
			MP.printError(EC.SYSTEM_STACK_OVERFLOW, cause);
		} else if (cause != null) {
			MP.printError(EC.GENERAL, cause);
		} else {
			MP.printError(EC.GENERAL, ee);
		}
		System.exit(1);
	}
	
	// Reset after checking unless it's the final check:
	if (finalCheck == false) {
		for (int i = 0; i < checker.length; i++) {
			checker[i].getDiskGraph().makeNodePtrTbl();
		}
	}
	MP.printMessage(EC.TLC_CHECKING_TEMPORAL_PROPS_END, TLC.convertRuntimeToHumanReadable(System.currentTimeMillis() - startTime));
	
	return EC.NO_ERROR;
}
 

/**
 * Generate a number of threads to fetch random numbers of certain bits
 * generated through a shared SecureRandom instance.
 * @param mech Mechanism name
 * @param byteLen Number of bytes of random number to produce
 * @param reSeed Call reseed() before generating random numbers
 * @throws NoSuchAlgorithmException
 * @throws InterruptedException
 * @throws ExecutionException
 */
private static void forEachMech(String mech, int byteLen, SEED reSeed)
        throws NoSuchAlgorithmException, InterruptedException,
        ExecutionException {

    if ("SHA1PRNG".equals(mech) && SEED.RESEED.equals(reSeed)) {
        System.out.printf(
                "%nreseed() api is not supported for '%s'", mech);
        return;
    }
    System.out.printf("%nTest SecureRandom mechanism: '%s' with support of"
            + " reseed: '%s'", mech, reSeed);
    int threadCount = (int) pow(2, 8 * byteLen);
    System.out.printf("%nCreating %s number of threads to generate secure "
            + "random numbers concurrently.", threadCount);

    ExecutorService executor
            = Executors.newCachedThreadPool(new ThreadFactory() {
                @Override
                public Thread newThread(Runnable r) {
                    Thread t = Executors.defaultThreadFactory()
                    .newThread(r);
                    t.setDaemon(true);
                    return t;
                }
            });
    CompletionService<Integer> completionService
            = new ExecutorCompletionService<Integer>(executor);

    CountDownLatch latch = new CountDownLatch(1);
    SecureRandom rnd = null;
    if (!mech.contains("_DRBG")) {
        rnd = SecureRandom.getInstance(mech);
    } else {
        Security.setProperty(DRBG_CONFIG, mech);
        rnd = SecureRandom.getInstance("DRBG");
    }
    try {
        for (int i = 0; i < threadCount; i++) {
            completionService.submit(new Task(rnd, latch, byteLen, reSeed));
        }
        latch.countDown();

        for (int i = 0; i < threadCount; i++) {
            completionService.take();
        }
    } finally {
        executor.shutdown();
    }
    System.out.printf("%nCompleted Test for algorithm '%s' with thread "
            + "counts to '%s' using reseeding '%s'",
            mech, threadCount, reSeed);

}
 

@Test
public void testCompletionServiceSubmit() throws InterruptedException {
    Executor x = Executors.newSingleThreadExecutor();
    CompletionService<String> s = new ExecutorCompletionService<String>(x);
    
    Callable<String> c1 = new WithInstrumentation();
    Callable<String> c2 = new WithoutInstrumentation();
    
    assertTrue(c1 instanceof InstrumentedExecution);
    assertFalse(c2 instanceof InstrumentedExecution);
    
    Future<String> f1 = s.submit(c1);
    
    assertTrue(f1 instanceof WrappedFuture);
    assertEquals(((InstrumentedExecution) c1).observeExecutionRunContext(), ((WrappedFuture) f1).instrumented);
    
    Future<String> f3 = s.submit(c2);
    
    assertFalse(f3 instanceof WrappedFuture);
    
}
 

public DoubleMatrix2D pairwiseCorrelationDoubleMatrix(double[][] in) {
	ExecutorService threadPool = Executors.newFixedThreadPool(nrThreads);
	CompletionService<Pair<Integer, double[]>> pool = new ExecutorCompletionService<Pair<Integer, double[]>>(threadPool);
	double meanOfSamples[] = new double[in.length];

	for (int i = 0; i < meanOfSamples.length; ++i) {
		meanOfSamples[i] = Descriptives.mean(in[i]);
	}

	for (int i = 0; i < in.length; i++) {
		ConcurrentCorrelationTask task = new ConcurrentCorrelationTask(in, meanOfSamples, i);
		pool.submit(task);
	}

	int returned = 0;

	DoubleMatrix2D correlationMatrix;
	if ((in.length * (long) in.length) > (Integer.MAX_VALUE - 2)) {
		correlationMatrix = new DenseLargeDoubleMatrix2D(in.length, in.length);
	} else {
		correlationMatrix = new DenseDoubleMatrix2D(in.length, in.length);
	}


	ProgressBar pb = new ProgressBar(in.length, "Calculation of correlation matrix: " + in.length + " x " + in.length);
	while (returned < in.length) {
		try {
			Pair<Integer, double[]> result = pool.take().get();
			if (result != null) {
				int rownr = result.getLeft(); //  < 0 when row is not to be included because of hashProbesToInclude.
				if (rownr >= 0) {
					double[] doubles = result.getRight();
					for (int i = 0; i < doubles.length; ++i) {
						correlationMatrix.setQuick(rownr, i, doubles[i]);
					}
				}
				result = null;
				returned++;
				pb.iterate();
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	for (int r = 1; r < correlationMatrix.rows(); r++) {
		for (int c = 0; c < r; c++) {
			correlationMatrix.setQuick(r, c, correlationMatrix.getQuick(c, r));
		}
	}

	threadPool.shutdown();
	pb.close();
	return correlationMatrix;
}
 

@Test
public void testCompletionServiceTake() throws InterruptedException {
    Executor x = Executors.newSingleThreadExecutor();
    CompletionService<String> s = new ExecutorCompletionService<String>(x);
    
    Callable<String> c1 = new WithInstrumentation();
    Callable<String> c2 = new WithoutInstrumentation();
    
    assertTrue(c1 instanceof InstrumentedExecution);
    assertFalse(c2 instanceof InstrumentedExecution);
    
    Future<String> f1 = s.submit(c1);
    
    assertTrue(f1 instanceof WrappedFuture);
    assertEquals(((InstrumentedExecution) c1).observeExecutionRunContext(), ((WrappedFuture) f1).instrumented);
    
    Future<String> f2 = s.take();
    
    assertTrue(f2 instanceof WrappedFuture);
    assertTrue(f1 == f2);
    assertEquals(f1, f2);
    assertEquals(f1.hashCode(), f2.hashCode());
    assertEquals(((InstrumentedExecution) c1).observeExecutionRunContext(), ((WrappedFuture) f2).instrumented);
    
    Future<String> f3 = s.submit(c2);
    
    assertFalse(f3 instanceof WrappedFuture);
    
    Future<String> f4 = s.take();
    
    assertFalse(f4 instanceof WrappedFuture);
    
}