org.apache.twill.discovery.ZKDiscoveryService Java Examples

@Override
public PhoenixTransactionService getTransactionService(Configuration config, ConnectionInfo connInfo, int port) {
    config.setInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, port);
    int retryTimeOut = config.getInt(TxConstants.Service.CFG_DATA_TX_CLIENT_DISCOVERY_TIMEOUT_SEC, 
            TxConstants.Service.DEFAULT_DATA_TX_CLIENT_DISCOVERY_TIMEOUT_SEC);
    ZKClientService zkClient = ZKClientServices.delegate(
      ZKClients.reWatchOnExpire(
        ZKClients.retryOnFailure(
          ZKClientService.Builder.of(connInfo.getZookeeperConnectionString())
            .setSessionTimeout(config.getInt(HConstants.ZK_SESSION_TIMEOUT,
                    HConstants.DEFAULT_ZK_SESSION_TIMEOUT))
            .build(),
          RetryStrategies.exponentialDelay(500, retryTimeOut, TimeUnit.MILLISECONDS)
        )
      )
    );

    DiscoveryService discovery = new ZKDiscoveryService(zkClient);
    TransactionManager txManager = new TransactionManager(config, new HDFSTransactionStateStorage(config, 
            new SnapshotCodecProvider(config), new TxMetricsCollector()), new TxMetricsCollector());
    TransactionService txService = new TransactionService(config, zkClient, discovery, Providers.of(txManager));
    TephraTransactionService service = new TephraTransactionService(zkClient, txService);
    service.start();
    return service;
}
 

@Provides
@Singleton
private DiscoveryService providesDiscoveryService(final ZKClientService zkClient,
                                                  final ZKDiscoveryService delegate) {
  return new DiscoveryService() {
    @Override
    public Cancellable register(Discoverable discoverable) {
      if (!zkClient.isRunning()) {
        zkClient.startAndWait();
      }
      return delegate.register(discoverable);
    }
  };
}
 

@Provides
@Singleton
private DiscoveryServiceClient providesDiscoveryServiceClient(final ZKClientService zkClient,
                                                              final ZKDiscoveryService delegate) {
  return new DiscoveryServiceClient() {
    @Override
    public ServiceDiscovered discover(String s) {
      if (!zkClient.isRunning()) {
        zkClient.startAndWait();
      }
      return delegate.discover(s);
    }
  };
}
 

private void configureJob() throws Exception {
    job = Job.getInstance(getConf(),
            "UpdateStatistics-" + tableName + "-" + snapshotName);
    PhoenixMapReduceUtil.setInput(job, NullDBWritable.class,
            snapshotName, tableName, restoreDir);

    PhoenixConfigurationUtil.setMRJobType(job.getConfiguration(), MRJobType.UPDATE_STATS);

    // DO NOT allow mapper splits using statistics since it may result into many smaller chunks
    PhoenixConfigurationUtil.setSplitByStats(job.getConfiguration(), false);

    job.setJarByClass(UpdateStatisticsTool.class);
    job.setMapperClass(TableSnapshotMapper.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setOutputFormatClass(NullOutputFormat.class);
    job.setNumReduceTasks(0);
    job.setPriority(this.jobPriority);

    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), PhoenixConnection.class, Chronology.class,
            CharStream.class, TransactionSystemClient.class, TransactionNotInProgressException.class,
            ZKClient.class, DiscoveryServiceClient.class, ZKDiscoveryService.class,
            Cancellable.class, TTransportException.class, SpanReceiver.class, TransactionProcessor.class, Gauge.class, MetricRegistriesImpl.class);
    LOGGER.info("UpdateStatisticsTool running for: " + tableName
            + " on snapshot: " + snapshotName + " with restore dir: " + restoreDir);
}
 

@Provides
@Singleton
private ZKDiscoveryService providesZKDiscoveryService(ZKClientService zkClient) {
  return new ZKDiscoveryService(zkClient);
}
 

private void doMain() throws Exception {
  // Try to load the secure store from localized file, which AM requested RM to localize it for this container.
  loadSecureStore();
  RunId appRunId = twillRuntimeSpec.getTwillAppRunId();
  RunId runId = RunIds.fromString(System.getenv(EnvKeys.TWILL_RUN_ID));
  String runnableName = System.getenv(EnvKeys.TWILL_RUNNABLE_NAME);
  int instanceId = Integer.valueOf(System.getenv(EnvKeys.TWILL_INSTANCE_ID));
  int instanceCount = Integer.valueOf(System.getenv(EnvKeys.TWILL_INSTANCE_COUNT));
  Map<String, String> defaultLogLevels = twillRuntimeSpec.getLogLevels().get(runnableName);
  Map<String, String> dynamicLogLevels = loadLogLevels().get(runnableName);

  Map<String, String> logLevels = new HashMap<>();
  if (defaultLogLevels != null) {
    logLevels.putAll(defaultLogLevels);
  } else {
    defaultLogLevels = ImmutableMap.of();
  }
  if (dynamicLogLevels != null) {
    logLevels.putAll(dynamicLogLevels);
  }

  ZKClientService zkClientService = createZKClient();
  ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClientService);

  ZKClient appRunZkClient = getAppRunZKClient(zkClientService, appRunId);

  TwillRunnableSpecification runnableSpec =
    twillRuntimeSpec.getTwillSpecification().getRunnables().get(runnableName).getRunnableSpecification();
  ContainerInfo containerInfo = new EnvContainerInfo();
  Arguments arguments = decodeArgs();
  BasicTwillContext context = new BasicTwillContext(
    runId, appRunId, containerInfo.getHost(),
    arguments.getRunnableArguments().get(runnableName).toArray(new String[0]),
    arguments.getArguments().toArray(new String[0]),
    runnableSpec, instanceId, discoveryService, discoveryService, appRunZkClient,
    instanceCount, containerInfo.getMemoryMB(), containerInfo.getVirtualCores()
  );

  ZKClient containerZKClient = getContainerZKClient(zkClientService, appRunId, runnableName);
  Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration()));
  TwillContainerService service = new TwillContainerService(context, containerInfo, containerZKClient,
                                                            runId, runnableSpec, getClassLoader(), conf,
                                                            createAppLocation(conf, twillRuntimeSpec.getFsUser(),
                                                                              twillRuntimeSpec.getTwillAppDir()),
                                                            defaultLogLevels, logLevels);
  doMain(
    service,
    zkClientService,
    new LogFlushService(),
    new TwillZKPathService(containerZKClient, runId),
    new CloseableServiceWrapper(discoveryService)
  );
}