org.apache.hadoop.yarn.api.records.URL Java Examples

/**
 * return a {@link URI} from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
@Private
public static URI getURIFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();

  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }

  return new URI(scheme, authority, url.getFile(), null, null).normalize();
}
 

@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 

/**
 * return a hadoop path from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();
  
  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }
  
  return new Path(
      (new URI(scheme, authority, url.getFile(), null, null)).normalize());
}
 

/**
 * return a {@link URI} from a given url
 *
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
@Private
public static URI getURIFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();

  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }

  return new URI(scheme, authority, url.getFile(), null, null).normalize();
}
 

private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
  try {
    FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

    if (null == resourceFileStatus) {
      throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
    }

    localResource.setResource(resourceUrl);
    log.info("setLocalizerResource for {}", resourceUrl);
    localResource.setSize(resourceFileStatus.getLen());
    localResource.setTimestamp(resourceFileStatus.getModificationTime());
    localResource.setType(resourceType);
    localResource.setVisibility(resourceVisibility);
    return localResource;
  } catch (IOException ioe) {
    log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
    throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
  }

}
 

/**
 * return a hadoop path from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();
  
  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }
  
  return new Path(
      (new URI(scheme, authority, url.getFile(), null, null)).normalize());
}
 

static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 

static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 

@Test
public void testRecreateDAG() {
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("dummyProcessor1"), 1,
      Resource.newInstance(1, 1));
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("dummyProcessor2"), 1,
      Resource.newInstance(1, 1));
  DAG dag = DAG.create("dag1").addVertex(v1).addVertex(v2).addTaskLocalFiles(lrDAG);

  TezConfiguration tezConf = new TezConfiguration();
  DAGPlan firstPlan = dag.createDag(tezConf, null, null, null, false);
  for (int i = 0; i < 3; ++i) {
      DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false);
      Assert.assertEquals(dagPlan, firstPlan);
  }
}
 

private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
 

@Test(timeout = 5000)
public void testYarnPathTranslation() {
  // Without port
  String p1String = "hdfs://mycluster/file";
  Path p1Path = new Path(p1String);
  // Users would translate this via this mechanic.
  URL lr1Url = ConverterUtils.getYarnUrlFromPath(p1Path);
  // Serialize to dag plan.
  String p1StringSerialized = DagTypeConverters.convertToDAGPlan(lr1Url);
  // Deserialize
  URL lr1UrlDeserialized = DagTypeConverters.convertToYarnURL(p1StringSerialized);
  Assert.assertEquals("mycluster", lr1UrlDeserialized.getHost());
  Assert.assertEquals("/file", lr1UrlDeserialized.getFile());
  Assert.assertEquals("hdfs", lr1UrlDeserialized.getScheme());


  // With port
  String p2String = "hdfs://mycluster:2311/file";
  Path p2Path = new Path(p2String);
  // Users would translate this via this mechanic.
  URL lr2Url = ConverterUtils.getYarnUrlFromPath(p2Path);
  // Serialize to dag plan.
  String p2StringSerialized = DagTypeConverters.convertToDAGPlan(lr2Url);
  // Deserialize
  URL lr2UrlDeserialized = DagTypeConverters.convertToYarnURL(p2StringSerialized);
  Assert.assertEquals("mycluster", lr2UrlDeserialized.getHost());
  Assert.assertEquals("/file", lr2UrlDeserialized.getFile());
  Assert.assertEquals("hdfs", lr2UrlDeserialized.getScheme());
  Assert.assertEquals(2311, lr2UrlDeserialized.getPort());
}
 

/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
public static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 

private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 

/**
 * This class is invoked from Org.Apache.REEF.Client.Yarn.LegacyJobResourceUploader in .NET code.
 * Arguments:
 * [0] : Local path for file.
 * [1] : Type for file.
 * [2] : Path of job submission directory
 * [3] : File path for output with details of uploaded resource
 */
public static void main(final String[] args) throws InjectionException, IOException {
  Validate.isTrue(args.length == 4, "Job resource uploader requires 4 args");
  final File localFile = new File(args[0]);
  Validate.isTrue(localFile.exists(), "Local file does not exist " + localFile.getAbsolutePath());
  final String fileType = args[1];
  final String jobSubmissionDirectory = args[2];
  final String localOutputPath = args[3];

  LOG.log(Level.INFO, "Received args: LocalPath " + localFile.getAbsolutePath() + " Submission directory " +
      jobSubmissionDirectory + " LocalOutputPath " + localOutputPath);
  final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
      .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
      .build();

  final JobUploader jobUploader = Tang.Factory.getTang()
      .newInjector(configuration)
      .getInstance(JobUploader.class);
  final LocalResource localResource = jobUploader.createJobFolder(jobSubmissionDirectory)
      .uploadAsLocalResource(localFile, LocalResourceType.valueOf(fileType));

  // Output: <UploadedPath>;<LastModificationUnixTimestamp>;<ResourceSize>
  final URL resource = localResource.getResource();
  final String outputString = String.format("%s://%s:%d%s;%d;%d", resource.getScheme(), resource.getHost(),
      resource.getPort(), resource.getFile(), localResource.getTimestamp(), localResource.getSize());
  LOG.log(Level.INFO, "Writing output: " + outputString);
  try (Writer writer = new BufferedWriter(new OutputStreamWriter(
      new FileOutputStream(localOutputPath), "utf-8"))) {
    writer.write(outputString);
  }

  LOG.log(Level.FINER, "Done writing output file");
}
 

@Override
public URL getLocalPath() {
  LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
  if (this.localPath != null) {
    return this.localPath;
  }
  if (!p.hasLocalPath()) {
    return null;
  }
  this.localPath = convertFromProtoFormat(p.getLocalPath());
  return this.localPath;
}
 

private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}
 

@Override
public synchronized URL getResource() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (this.url != null) {
    return this.url;
  }
  if (!p.hasResource()) {
    return null;
  }
  this.url = convertFromProtoFormat(p.getResource());
  return this.url;
}
 

@Override
public void setLocalPath(URL localPath) {
  maybeInitBuilder();
  if (localPath == null)
    builder.clearLocalPath();
  this.localPath = localPath;
}
 

public static String convertToDAGPlan(URL resource) {
  Path p;
  try {
    p = ConverterUtils.getPathFromYarnURL(resource);
  } catch (URISyntaxException e) {
    throw new TezUncheckedException("Unable to translate resource: " + resource + " to Path");
  }
  String urlString = p.toString();
  return urlString;
}
 

@Override
public URL getDestinationDirectory() {
  ResourceLocalizationSpecProtoOrBuilder p = viaProto ? proto : builder;
  if (destinationDirectory != null) {
    return destinationDirectory;
  }
  if (!p.hasDestinationDirectory()) {
    return null;
  }
  destinationDirectory = new URLPBImpl(p.getDestinationDirectory());
  return destinationDirectory;
}
 

private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 

private static LocalResource getMockedResource(Random r, 
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = getPath("/local/PRIVATE/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return rsrc;
}
 

public static URL newURL(String scheme, String host, int port, String file) {
  URL url = recordFactory.newRecordInstance(URL.class);
  url.setScheme(scheme);
  url.setHost(host);
  url.setPort(port);
  url.setFile(file);
  return url;
}
 

public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 

@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
  Path expectedPath = new Path("hdfs://foo.com");
  URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
  Path actualPath = ConverterUtils.getPathFromYarnURL(url);
  assertEquals(expectedPath, actualPath);
}
 

public static URL getYarnUrlFromURI(URI uri) {
  URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
  if (uri.getHost() != null) {
    url.setHost(uri.getHost());
  }
  if (uri.getUserInfo() != null) {
    url.setUserInfo(uri.getUserInfo());
  }
  url.setPort(uri.getPort());
  url.setScheme(uri.getScheme());
  url.setFile(uri.getPath());
  return url;
}
 

@Override
public synchronized void setResource(URL resource) {
  maybeInitBuilder();
  if (resource == null) 
    builder.clearResource();
  this.url = resource;
}
 

@Override
public synchronized URL getResource() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (this.url != null) {
    return this.url;
  }
  if (!p.hasResource()) {
    return null;
  }
  this.url = convertFromProtoFormat(p.getResource());
  return this.url;
}
 

@Test (timeout = 10000)
public void testMultipleSubmissions() throws Exception {
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  DAGClient dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
  
  // submit the same DAG again to verify it can be done.
  tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 

/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}