org.apache.hadoop.yarn.api.records.LocalResource Java Examples

static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 

private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
    Path destDir, FileSystem localFs) throws IOException {

  for (String localFilePath : SPLITTER.split(localFilePathList)) {
    Path srcFilePath = new Path(localFilePath);
    Path destFilePath = new Path(destDir, srcFilePath.getName());
    if (localFs.exists(srcFilePath)) {
      this.fs.copyFromLocalFile(srcFilePath, destFilePath);
      if (resourceMap.isPresent()) {
        YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
      }
    } else {
      LOGGER.warn(String.format("The request file %s doesn't exist", srcFilePath));
    }
  }
}
 

static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(ConverterUtils
    .getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}
 

private void addResource(
        FileSystem fs,
        Path destPath,
        Map<String, LocalResource> localResources,
        LocalResourceType resourceType,
        String link) throws IOException {

  FileStatus destStatus = fs.getFileStatus(destPath);
  LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
  amJarRsrc.setType(resourceType);
  amJarRsrc.setVisibility(LocalResourceVisibility.PUBLIC);
  amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
  amJarRsrc.setTimestamp(destStatus.getModificationTime());
  amJarRsrc.setSize(destStatus.getLen());
  localResources.put(link, amJarRsrc);
}
 

private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
    Map<String, LocalResource> reqLRs) {
  Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
  for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
    LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
    if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cannot match container: Attempting to use same target resource name: "
            + srcLREntry.getKey()
            + ", but with different source resources. Already localized: "
            + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
      }
      return false;
    }
  }
  for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
    LocalResource lr = additionalLREntry.getValue();
    if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) {
      return false;
    }
  }
  return true;
}
 

/**
 * Creates the JAR file with the global files on the driver and then uploads it to the job folder on
 * (H)DFS.
 *
 * @return the map to be used as the "global" resources when submitting Evaluators.
 * @throws IOException if the creation of the JAR or the upload fails
 */
@Override
public synchronized Map<String, LocalResource> call() throws IOException {
  final Map<String, LocalResource> globalResources = new HashMap<>(1);
  if (!this.isUploaded){
    this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar());
    this.isUploaded = true;
  }

  final LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar);

  if (this.globalJarResource != null
      && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) {
    LOG.log(Level.WARNING,
            "The global JAR LocalResource timestamp has been changed from "
            + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp());
  }

  this.globalJarResource = updatedGlobalJarResource;

  // For now, always rewrite the information due to REEF-348
  globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource);

  return globalResources;
}
 

static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 

private Map<String, LocalResource> buildResourceMapping() {
  ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

  List<String> resourceNames = resourceConfig.getResourceNames();
  for (String resourceName : resourceNames) {
    String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
    LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
    LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
    Path resourcePath = resourceConfig.getResourcePath(resourceName);

    LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

    localResourceMapBuilder.put(resourceLocalName, localResource);
    log.info("preparing local resource: {}", resourceLocalName);
  }

  return localResourceMapBuilder.build();
}
 

@Test
public void testResourceMapWithDefaultValues() {

  Map<String, String> configMap = new HashMap<>();

  configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

  Config conf = new MapConfig(configMap);

  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
  Map<String, LocalResource> resourceMap = mapper.getResourceMap();

  assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
  assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
  assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
  assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
  assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}
 

/**
 * Uploads and registers a single resource and adds it to <tt>localResources</tt>.
 *
 * @param key
 * 		the key to add the resource under
 * @param fs
 * 		the remote file system to upload to
 * @param appId
 * 		application ID
 * @param localSrcPath
 * 		local path to the file
 * @param localResources
 * 		map of resources
 *
 * @return the remote path to the uploaded resource
 */
private static Path setupSingleLocalResource(
		String key,
		FileSystem fs,
		ApplicationId appId,
		Path localSrcPath,
		Map<String, LocalResource> localResources,
		Path targetHomeDir,
		String relativeTargetPath) throws IOException, URISyntaxException {

	Tuple2<Path, LocalResource> resource = Utils.setupLocalResource(
		fs,
		appId.toString(),
		localSrcPath,
		targetHomeDir,
		relativeTargetPath);

	localResources.put(key, resource.f1);

	return resource.f0;
}
 

/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 

/**
 * Sets up the LocalResources for a new Evaluator.
 *
 * @param resourceLaunchEvent
 * @return
 * @throws IOException
 */
Map<String, LocalResource> getResources(
    final ResourceLaunchEvent resourceLaunchEvent)
    throws IOException {

  final Map<String, LocalResource> result = new HashMap<>();
  result.putAll(getGlobalResources());

  final File localStagingFolder = this.tempFileCreator.createTempDirectory(this.fileNames.getEvaluatorFolderPrefix());

  // Write the configuration
  final File configurationFile = new File(localStagingFolder, this.fileNames.getEvaluatorConfigurationName());
  this.configurationSerializer.toFile(makeEvaluatorConfiguration(resourceLaunchEvent), configurationFile);

  // Copy files to the staging folder
  JobJarMaker.copy(resourceLaunchEvent.getFileSet(), localStagingFolder);

  // Make a JAR file out of it
  final File localFile = tempFileCreator.createTempFile(
      this.fileNames.getEvaluatorFolderPrefix(), this.fileNames.getJarFileSuffix());
  new JARFileMaker(localFile).addChildren(localStagingFolder).close();

  // Upload the JAR to the job folder
  final Path pathToEvaluatorJar = this.uploader.uploadToJobFolder(localFile);
  result.put(this.fileNames.getLocalFolderPath(), this.uploader.makeLocalResourceForJarFile(pathToEvaluatorJar));

  if (this.deleteTempFiles) {
    LOG.log(Level.FINE, "Marking [{0}] for deletion at the exit of this JVM and deleting [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
    localFile.deleteOnExit();
    if (!localStagingFolder.delete()) {
      LOG.log(Level.WARNING, "Failed to delete [{0}]", localStagingFolder.getAbsolutePath());
    }
  } else {
    LOG.log(Level.FINE, "The evaluator staging folder will be kept at [{0}], the JAR at [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
  }
  return result;
}
 

@Override
public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
	Map<String, LocalResource> resources = new HashMap<String, LocalResource>();
	resources.putAll(context.getLocalResources());
	resources.putAll(artifactResourceLocalizer.getResources());
	context.setLocalResources(resources);
	return context;
}
 

public AMContainerTask(boolean shouldDie, TaskSpec tezTask,
    Map<String, LocalResource> additionalResources, Credentials credentials, boolean credentialsChanged) {
  this.shouldDie = shouldDie;
  this.tezTask = tezTask;
  this.additionalResources = additionalResources;
  this.credentials = credentials;
  this.credentialsChanged = credentialsChanged;
}
 

public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 

private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap) throws IOException {
  if (!this.fs.exists(destDir)) {
    LOGGER.warn(String.format("Path %s does not exist so no container LocalResource to add", destDir));
    return;
  }

  FileStatus[] statuses = this.fs.listStatus(destDir);
  if (statuses != null) {
    for (FileStatus status : statuses) {
      YarnHelixUtils.addFileAsLocalResource(this.fs, status.getPath(), LocalResourceType.FILE, resourceMap);
    }
  }
}
 

/**
 *
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesMultipleTarballs() throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  StringBuilder tezLibUris = new StringBuilder();

  // Create 2 files
  Path topDir = new Path(TEST_ROOT_DIR, "validatemultipletarballs");
  if (localFs.exists(topDir)) {
    localFs.delete(topDir, true);
  }
  localFs.mkdirs(topDir);

  Path tarFile1 = new Path(topDir, "f1.tar.gz");
  Path tarFile2 = new Path(topDir, "f2.tar.gz");

  Assert.assertTrue(localFs.createNewFile(tarFile1));
  Assert.assertTrue(localFs.createNewFile(tarFile2));
  tezLibUris.append(localFs.makeQualified(tarFile1).toString()).append("#tar1").append(",");
  tezLibUris.append(localFs.makeQualified(tarFile2).toString()).append("#tar2").append(",");

  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString());
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap);
  Set<String> resourceNames = localizedMap.keySet();
  Assert.assertEquals(2, resourceNames.size());
  Assert.assertTrue(resourceNames.contains("tar1"));
  Assert.assertTrue(resourceNames.contains("tar2"));
  Assert.assertFalse(resourceNames.contains("f1.tar.gz"));
  Assert.assertFalse(resourceNames.contains("f2.tar.gz"));


  Assert.assertTrue(localFs.delete(tarFile1, true));
  Assert.assertTrue(localFs.delete(tarFile2, true));
  Assert.assertTrue(localFs.delete(topDir, true));
}
 

private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}
 

private LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath)
        throws IOException
{
    LocalResource localResource = Records.newRecord(LocalResource.class);
    FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    return localResource;
}
 

private LocalResource createApplicationResource(FileContext fs, Path p,
    LocalResourceType type) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 

@Test(timeout = 5000)
public void testEdgeManagerSerde() {
  DAG dag = DAG.create("testDag");
  ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
  ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
  Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .addTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .addTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = InputDescriptor.create("input")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
  OutputDescriptor outputDescriptor = OutputDescriptor.create("output")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
  Edge edge = Edge.create(v1, v2, EdgeProperty.create(
      EdgeManagerPluginDescriptor.create("emClass").setUserPayload(
          UserPayload.create(ByteBuffer.wrap("emPayload".getBytes()))),
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);

  EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto
      .getEdgeList().get(0));

  EdgeManagerPluginDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
  Assert.assertNotNull(emDesc);
  Assert.assertEquals("emClass", emDesc.getClassName());
  Assert.assertTrue(
      Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload().deepCopyAsArray()));
}
 

public Map<String, LocalResource> addTezResources(Set<URI> resources) throws Exception {
    Set<String> resourceNames = new HashSet<String>();
    for (URI uri : resources) {
        addTezResource(uri);
        resourceNames.add(new Path(uri.getPath()).getName());
    }
    return getTezResources(resourceNames);
}
 

private SharedCacheUploader createSpiedUploader(LocalResource resource, Path localPath,
    String user, Configuration conf, SCMUploaderProtocol scmClient,
    FileSystem fs, FileSystem localFs)
        throws IOException {
  SharedCacheUploader uploader = new SharedCacheUploader(resource, localPath, user, conf, scmClient,
      fs, localFs);
  return spy(uploader);
}
 

public void assignTaskAttempt(TezTaskAttemptID taID,
    Map<String, LocalResource> additionalResources, Credentials credentials) {
  reset(eventHandler);
  doReturn(taID).when(taskSpec).getTaskAttemptID();
  amContainer.handle(new AMContainerEventAssignTA(containerID, taID, taskSpec,
      additionalResources, credentials, taskPriority));
}
 

public static Map<String, LocalResource> convertFromPlanLocalResources(
  PlanLocalResourcesProto proto) {
  Map<String, LocalResource> localResources =
    new HashMap<String, LocalResource>(proto.getLocalResourcesCount());
  for (PlanLocalResource plr : proto.getLocalResourcesList()) {
    String name = plr.getName();
    LocalResource lr = convertPlanLocalResourceToLocalResource(plr);
    localResources.put(name, lr);
  }
  return localResources;
}
 

/**
 * 
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesNotDefined() throws Exception {

  TezConfiguration conf = new TezConfiguration(false);
  Credentials credentials = new Credentials();
  try {
    Map<String,LocalResource> resources = new HashMap<String, LocalResource>();
    TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources);
    Assert.fail("Expected TezUncheckedException");
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains("Invalid configuration of tez jars"));
  }
}
 

AMConfiguration(TezConfiguration tezConf, Map<String, LocalResource> localResources,
    Credentials credentials) {
  this.localResources = Maps.newHashMap();
  this.tezConf = tezConf;
  if (localResources != null) {
    addLocalResources(localResources);
  }
  if (credentials != null) {
    setCredentials(credentials);
  }

}
 

static LocalResource createResource() {
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  assertTrue(ret instanceof LocalResourcePBImpl);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
    "hdfs://y.ak:8020/foo/bar")));
  ret.setSize(4344L);
  ret.setTimestamp(3141592653589793L);
  ret.setVisibility(LocalResourceVisibility.PUBLIC);
  return ret;
}
 

public ContainerContext(Map<String, LocalResource> localResources,
    Credentials credentials, Map<String, String> environment, String javaOpts) {
  Preconditions.checkNotNull(localResources,
      "localResources should not be null");
  Preconditions.checkNotNull(credentials, "credentials should not be null");
  Preconditions.checkNotNull(environment, "environment should not be null");
  Preconditions.checkNotNull(javaOpts, "javaOpts should not be null");
  this.localResources = localResources;
  this.credentials = credentials;
  this.environment = environment;
  this.javaOpts = javaOpts;
  this.vertex = null;
}
 

public ContainerContext(Map<String, LocalResource> localResources,
    Credentials credentials, Map<String, String> environment, String javaOpts) {
  Objects.requireNonNull(localResources,
      "localResources should not be null");
  Objects.requireNonNull(credentials, "credentials should not be null");
  Objects.requireNonNull(environment, "environment should not be null");
  Objects.requireNonNull(javaOpts, "javaOpts should not be null");
  this.localResources = localResources;
  this.credentials = credentials;
  this.environment = environment;
  this.javaOpts = javaOpts;
  this.vertex = null;
}