Java Code Examples for org.apache.hadoop.yarn.server.MiniYARNCluster#init()

public void start() throws IOException {
  log.info("Checking if cluster=" + clusterName + " needs to be started");
  synchronized (this.startupShutdownMonitor) {
    if (started) {
      return;
    }
    log.info("Starting cluster=" + clusterName);
    configuration = new YarnConfiguration();

    //configuration.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
    configuration.setBoolean("yarn.is.minicluster", true);
    configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs");

    dfsCluster = new MiniDFSCluster.Builder(configuration).
            numDataNodes(nodes).
            build();

    yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1);
    yarnCluster.init(configuration);
    yarnCluster.start();

    log.info("Started cluster=" + clusterName);
    started = true;
  }
}
 

@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 

@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 

@Override
public void start() throws Exception {
    LOG.info("YARN: Starting MiniYarnCluster");
    configure();
    miniYARNCluster = new MiniYARNCluster(testName, numResourceManagers, numNodeManagers,
            numLocalDirs, numLogDirs, enableHa);

    miniYARNCluster.serviceInit(configuration);
    miniYARNCluster.init(configuration);
    miniYARNCluster.start();
}
 

protected MiniYARNCluster createMiniYARNCluster(Configuration yarnConf, int numOfNodeManagers) throws Exception {
  yarnConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
  yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
  MiniYARNCluster miniYarnCluster = new MiniYARNCluster("yarn", numOfNodeManagers, 1, 1);
  miniYarnCluster.init(yarnConf);
  yarnConf.set("yarn.resourcemanager.scheduler.address", "0.0.0.0:8030") ;
  miniYarnCluster.start();
  return miniYarnCluster ;
}
 

@Test
public void testRMWebAppRedirect() throws YarnException,
    InterruptedException, IOException {
  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);

  cluster.init(conf);
  cluster.start();
  getAdminService(0).transitionToActive(req);
  String rm1Url = "http://0.0.0.0:18088";
  String rm2Url = "http://0.0.0.0:28088";
  String redirectURL = getRedirectURL(rm2Url);
  // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
  assertEquals(redirectURL,rm1Url+"/");

  redirectURL = getRedirectURL(rm2Url + "/metrics");
  assertEquals(redirectURL,rm1Url + "/metrics");

  redirectURL = getRedirectURL(rm2Url + "/jmx");
  assertEquals(redirectURL,rm1Url + "/jmx");

  // standby RM links /conf, /stacks, /logLevel, /static, /logs,
  // /cluster/cluster as well as webService
  // /ws/v1/cluster/info should not be redirected to active RM
  redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/conf");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/stacks");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logLevel");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/static");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logs");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps");
  assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps");

  redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
  assertNull(redirectURL);
}
 

@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

@Override
protected void before() throws Throwable {
  tmpFolder.create();

  // Starts Zookeeper
  zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
  zkServer.startAndWait();

  // Start YARN mini cluster
  File miniDFSDir = tmpFolder.newFolder();
  LOG.info("Starting Mini DFS on path {}", miniDFSDir);
  Configuration fsConf = new HdfsConfiguration(new Configuration());
  fsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, miniDFSDir.getAbsolutePath());

  for (Map.Entry<String, String> entry : extraConfig.entrySet()) {
    fsConf.set(entry.getKey(), entry.getValue());
  }

  dfsCluster = new MiniDFSCluster.Builder(fsConf).numDataNodes(1).build();

  Configuration conf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());

  conf.set("yarn.resourcemanager.scheduler.class",
           "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
  conf.set("yarn.scheduler.capacity.resource-calculator",
           "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
  conf.setBoolean("yarn.scheduler.include-port-in-node-name", true);
  conf.set("yarn.nodemanager.vmem-pmem-ratio", "100.1");
  conf.set("yarn.nodemanager.vmem-check-enabled", "false");
  conf.set("yarn.scheduler.minimum-allocation-mb", "128");
  conf.set("yarn.nodemanager.delete.debug-delay-sec", "3600");

  conf.set(Configs.Keys.LOCAL_STAGING_DIRECTORY, tmpFolder.newFolder().getAbsolutePath());

  cluster = new MiniYARNCluster("test-cluster", 3, 1, 1);
  cluster.init(conf);
  cluster.start();

  config = new YarnConfiguration(cluster.getConfig());

  twillRunner = createTwillRunnerService();
  twillRunner.start();

  yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf);
}
 

@Test
public void testRMWebAppRedirect() throws YarnException,
    InterruptedException, IOException {
  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);

  cluster.init(conf);
  cluster.start();
  getAdminService(0).transitionToActive(req);
  String rm1Url = "http://0.0.0.0:18088";
  String rm2Url = "http://0.0.0.0:28088";
  String redirectURL = getRedirectURL(rm2Url);
  // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
  assertEquals(redirectURL,rm1Url+"/");

  redirectURL = getRedirectURL(rm2Url + "/metrics");
  assertEquals(redirectURL,rm1Url + "/metrics");

  redirectURL = getRedirectURL(rm2Url + "/jmx");
  assertEquals(redirectURL,rm1Url + "/jmx");

  // standby RM links /conf, /stacks, /logLevel, /static, /logs,
  // /cluster/cluster as well as webService
  // /ws/v1/cluster/info should not be redirected to active RM
  redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/conf");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/stacks");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logLevel");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/static");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logs");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps");
  assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps");

  redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
  assertNull(redirectURL);
}
 

@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

private static void setup(String[] args)
    throws Exception {
  // Parsing zk-string
  Preconditions.checkArgument(args.length == 1);
  zkString = args[0];

  // Initialize necessary external components: Yarn and Helix
  Closer closer = Closer.create();

  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  final YarnConfiguration clusterConf = new YarnConfiguration();
  clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");
  clusterConf.set("yarn.nodemanager.resource.memory-mb", "512");
  clusterConf.set("yarn.scheduler.maximum-allocation-mb", "1024");

  MiniYARNCluster miniYARNCluster = closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
  miniYARNCluster.init(clusterConf);
  miniYARNCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(log).timeoutMs(10000).assertTrue(new Predicate<Void>() {
    @Override
    public boolean apply(Void input) {
      return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
    }
  }, "Waiting for RM");

  try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH, "UTF-8")) {
    File dir = new File("target/dummydir");

    // dummy directory specified in configuration
    if (!dir.mkdir()) {
      log.error("The dummy folder's creation is not successful");
    }
    dir.deleteOnExit();

    pw.println("gobblin.cluster.zk.connection.string=\"" + zkString + "\"");
    pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
  }

  // YARN config is dynamic and needs to be passed to other processes
  try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
    clusterConf.writeXml(os);
  }

  /** Have to pass the same yarn-site.xml to the GobblinYarnAppLauncher to initialize Yarn Client. */
  fileAddress = new File(YARN_SITE_XML_PATH).getAbsolutePath();
}
 

@BeforeClass
public void setUp() throws Exception {
  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  final YarnConfiguration clusterConf = new YarnConfiguration();
  clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

  MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
  miniYARNCluster.init(clusterConf);
  miniYARNCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(LOG).timeoutMs(10000)
      .assertTrue(new Predicate<Void>() {
        @Override public boolean apply(Void input) {
          return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
      }, "Waiting for RM");

  this.yarnClient = this.closer.register(YarnClient.createYarnClient());
  this.yarnClient.init(clusterConf);
  this.yarnClient.start();

  // Use a random ZK port
  TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
  LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

  // the zk port is dynamically configured
  try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH)) {
    File dir = new File("target/dummydir");

    // dummy directory specified in configuration
    dir.mkdir();

    pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
    pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
  }

  // YARN config is dynamic and needs to be passed to other processes
  try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
    clusterConf.writeXml(os);
  }

  this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

  URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
      .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
  Assert.assertNotNull(url, "Could not find resource " + url);

  this.config = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
                 ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .resolve();

  String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
  this.helixManager = HelixManagerFactory.getZKHelixManager(
      this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME,
      InstanceType.CONTROLLER, zkConnectionString);

  this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);

  this.configManagedHelix = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
          ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .withValue(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY, ConfigValueFactory.fromAnyRef(MANAGED_HELIX_CLUSTER_NAME))
      .withValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, ConfigValueFactory.fromAnyRef(TEST_HELIX_INSTANCE_NAME_MANAGED))
      .withValue(GobblinClusterConfigurationKeys.IS_HELIX_CLUSTER_MANAGED, ConfigValueFactory.fromAnyRef("true"))
      .resolve();

  this.helixManagerManagedHelix = HelixManagerFactory.getZKHelixManager(
      this.configManagedHelix.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TEST_HELIX_INSTANCE_NAME_MANAGED,
      InstanceType.PARTICIPANT, zkConnectionString);

  this.gobblinYarnAppLauncherManagedHelix = new GobblinYarnAppLauncher(this.configManagedHelix, clusterConf);
}