org.apache.twill.discovery.DiscoveryServiceClient Java Examples

public BasicTwillContext(RunId runId, RunId appRunId, InetAddress host, String[] args, String[] appArgs,
                         TwillRunnableSpecification spec, int instanceId,
                         DiscoveryService discoveryService, DiscoveryServiceClient discoveryServiceClient,
                         ZKClient zkClient,
                         int instanceCount, int allowedMemoryMB, int virtualCores) {
  this.runId = runId;
  this.appRunId = appRunId;
  this.host = host;
  this.args = args;
  this.appArgs = appArgs;
  this.spec = spec;
  this.instanceId = instanceId;
  this.discoveryService = discoveryService;
  this.discoveryServiceClient = discoveryServiceClient;
  this.zkClient = zkClient;
  this.elections = new ElectionRegistry(zkClient);
  this.instanceCount = instanceCount;
  this.allowedMemoryMB = allowedMemoryMB;
  this.virtualCores = virtualCores;
}
 

@Provides
@Singleton
private DiscoveryServiceClient providesDiscoveryServiceClient(final ZKClientService zkClient,
                                                              final ZKDiscoveryService delegate) {
  return new DiscoveryServiceClient() {
    @Override
    public ServiceDiscovered discover(String s) {
      if (!zkClient.isRunning()) {
        zkClient.startAndWait();
      }
      return delegate.discover(s);
    }
  };
}
 

private void configureJob() throws Exception {
    job = Job.getInstance(getConf(),
            "UpdateStatistics-" + tableName + "-" + snapshotName);
    PhoenixMapReduceUtil.setInput(job, NullDBWritable.class,
            snapshotName, tableName, restoreDir);

    PhoenixConfigurationUtil.setMRJobType(job.getConfiguration(), MRJobType.UPDATE_STATS);

    // DO NOT allow mapper splits using statistics since it may result into many smaller chunks
    PhoenixConfigurationUtil.setSplitByStats(job.getConfiguration(), false);

    job.setJarByClass(UpdateStatisticsTool.class);
    job.setMapperClass(TableSnapshotMapper.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setOutputFormatClass(NullOutputFormat.class);
    job.setNumReduceTasks(0);
    job.setPriority(this.jobPriority);

    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), PhoenixConnection.class, Chronology.class,
            CharStream.class, TransactionSystemClient.class, TransactionNotInProgressException.class,
            ZKClient.class, DiscoveryServiceClient.class, ZKDiscoveryService.class,
            Cancellable.class, TTransportException.class, SpanReceiver.class, TransactionProcessor.class, Gauge.class, MetricRegistriesImpl.class);
    LOGGER.info("UpdateStatisticsTool running for: " + tableName
            + " on snapshot: " + snapshotName + " with restore dir: " + restoreDir);
}
 

@Inject(optional = true)
void setDiscoveryServiceClient(DiscoveryServiceClient discoveryServiceClient) {
  this.discoveryServiceClient = discoveryServiceClient;
}
 

@Override
protected void configure() {
  InMemoryDiscoveryService discovery = IN_MEMORY_DISCOVERY_SERVICE;
  bind(DiscoveryService.class).toInstance(discovery);
  bind(DiscoveryServiceClient.class).toInstance(discovery);
}
 

@Override
protected void configure() {
  expose(DiscoveryService.class);
  expose(DiscoveryServiceClient.class);
}
 

public SingleUseClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient, int timeout) {
  super(conf, discoveryServiceClient);
  this.timeout = timeout;
}
 

public PooledClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient) {
  super(conf, discoveryServiceClient);
}
 

public ThreadLocalClientProvider(Configuration conf, DiscoveryServiceClient discoveryServiceClient) {
  super(conf, discoveryServiceClient);
}
 

protected AbstractClientProvider(Configuration configuration, DiscoveryServiceClient discoveryServiceClient) {
  this.configuration = configuration;
  this.discoveryServiceClient = discoveryServiceClient;
}
 

private void startClientAndTestPool(Configuration conf) throws Exception {
  Injector injector = Guice.createInjector(
    new ConfigModule(conf),
    new ZKModule(),
    new DiscoveryModules().getDistributedModules(),
    new TransactionModules().getDistributedModules(),
    new TransactionClientModule()
  );

  ZKClientService zkClient = injector.getInstance(ZKClientService.class);
  zkClient.startAndWait();

  final PooledClientProvider clientProvider = new PooledClientProvider(conf,
    injector.getInstance(DiscoveryServiceClient.class));

  // Test simple case of get + return. Note: this also initializes the provider's pool, which
  // takes about one second (discovery). Doing it before we test the threads makes it so that one
  // thread doesn't take exceptionally longer than the others.

  // Need to retry, since TransactionServiceMain#start returning doesn't indicate that the TransactionService
  // has registered itself for discovery yet
  Tests.waitFor("Failed to get client.", new Callable<Boolean>() {
    @SuppressWarnings({"unused", "EmptyTryBlock"})
    @Override
    public Boolean call() throws Exception {
      try (CloseableThriftClient closeableThriftClient = clientProvider.getCloseableClient()) {
        // do nothing with the client
      } catch (TException e) {
        // simply retry
        return false;
      }
      return true;
    }
  });

  //Now race to get MAX_CLIENT_COUNT+1 clients, exhausting the pool and requesting 1 more.
  List<Future<Integer>> clientIds = new ArrayList<>();
  // We want to ensure that all clients have been exhausted before releasing any.
  // Only once all the clients are fetched from the pool, will any be released. The last thread will reuse one of
  // these clients from the pool.
  CountDownLatch clientDoneLatch = new CountDownLatch(MAX_CLIENT_COUNT);
  ExecutorService executor = Executors.newFixedThreadPool(MAX_CLIENT_COUNT + 1);
  for (int i = 0; i < MAX_CLIENT_COUNT + 1; i++) {
    clientIds.add(executor.submit(new RetrieveClient(clientProvider, clientDoneLatch)));
  }

  Set<Integer> ids = new HashSet<>();
  for (Future<Integer> id : clientIds) {
    ids.add(id.get());
  }
  Assert.assertEquals(MAX_CLIENT_COUNT, ids.size());

  // Now, try it again with, with a countdown latch equal to the number of threads. All of them will only progress
  // past it, once they all acquire a client or time out while attempting to obtain one.
  // One of the threads should throw a TimeOutException, because the other threads don't release their clients
  // until then and the client thread pool isn't enough for the number of threads.
  clientDoneLatch = new CountDownLatch(MAX_CLIENT_COUNT + 1);
  for (int i = 0; i < MAX_CLIENT_COUNT + 1; i++) {
    clientIds.add(executor.submit(new RetrieveClient(clientProvider, clientDoneLatch)));
  }
  int numTimeoutExceptions = 0;
  for (Future<Integer> clientId : clientIds) {
    try {
      clientId.get();
    } catch (ExecutionException expected) {
      Assert.assertEquals(TimeoutException.class, expected.getCause().getClass());
      numTimeoutExceptions++;
    }
  }
  // expect that exactly one of the threads hit the TimeoutException
  Assert.assertEquals(String.format("Expected one thread to not obtain a client within %s milliseconds.",
                                    CLIENT_OBTAIN_TIMEOUT),
                      1, numTimeoutExceptions);

  executor.shutdownNow();
}