Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResource#setType()
The following examples show how to use
org.apache.hadoop.yarn.api.records.LocalResource#setType() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFSDownload.java From hadoop with Apache License 2.0 | 6 votes |
static LocalResource createJar(FileContext files, Path p, LocalResourceVisibility vis) throws IOException { LOG.info("Create jar file " + p); File jarFile = new File((files.makeQualified(p)).toUri()); FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); FileStatus status = files.getFileStatus(p); ret.setSize(status.getLen()); ret.setTimestamp(status.getModificationTime()); ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret; }
Example 2
Source File: IgniteYarnUtils.java From ignite with Apache License 2.0 | 6 votes |
/** * @param file Path. * @param fs File system. * @param type Local resource type. * @throws Exception If failed. */ public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type) throws Exception { LocalResource resource = Records.newRecord(LocalResource.class); file = fs.makeQualified(file); FileStatus stat = fs.getFileStatus(file); resource.setResource(ConverterUtils.getYarnUrlFromPath(file)); resource.setSize(stat.getLen()); resource.setTimestamp(stat.getModificationTime()); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); return resource; }
Example 3
Source File: YARNRunner.java From incubator-tez with Apache License 2.0 | 5 votes |
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type) throws IOException { LocalResource rsrc = Records.newRecord(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs .getDefaultFileSystem().resolvePath(rsrcStat.getPath()))); rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); rsrc.setType(type); rsrc.setVisibility(LocalResourceVisibility.APPLICATION); return rsrc; }
Example 4
Source File: TestFSDownload.java From big-c with Apache License 2.0 | 5 votes |
static LocalResource createTarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { byte[] bytes = new byte[len]; r.nextBytes(bytes); File archiveFile = new File(p.toUri().getPath() + ".tar"); archiveFile.createNewFile(); TarArchiveOutputStream out = new TarArchiveOutputStream( new FileOutputStream(archiveFile)); TarArchiveEntry entry = new TarArchiveEntry(p.getName()); entry.setSize(bytes.length); out.putArchiveEntry(entry); out.write(bytes); out.closeArchiveEntry(); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + ".tar"))); ret.setSize(len); ret.setType(LocalResourceType.ARCHIVE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar")) .getModificationTime()); return ret; }
Example 5
Source File: Util.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
public static LocalResource newYarnAppResource( FileSystem fs, Path path, LocalResourceType type, LocalResourceVisibility vis) throws IOException { Path qualified = fs.makeQualified(path); FileStatus status = fs.getFileStatus(qualified); LocalResource resource = Records.newRecord(LocalResource.class); resource.setType(type); resource.setVisibility(vis); resource.setResource(ConverterUtils.getYarnUrlFromPath(qualified)); resource.setTimestamp(status.getModificationTime()); resource.setSize(status.getLen()); return resource; }
Example 6
Source File: TestFSDownload.java From big-c with Apache License 2.0 | 5 votes |
static LocalResource createFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException { createFile(files, p, len, r); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); ret.setSize(len); ret.setType(LocalResourceType.FILE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(p).getModificationTime()); return ret; }
Example 7
Source File: YARNRunner.java From hadoop with Apache License 2.0 | 5 votes |
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type) throws IOException { LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs .getDefaultFileSystem().resolvePath(rsrcStat.getPath()))); rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); rsrc.setType(type); rsrc.setVisibility(LocalResourceVisibility.APPLICATION); return rsrc; }
Example 8
Source File: TestFSDownload.java From big-c with Apache License 2.0 | 5 votes |
static LocalResource createTgzFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { byte[] bytes = new byte[len]; r.nextBytes(bytes); File gzipFile = new File(p.toUri().getPath() + ".tar.gz"); gzipFile.createNewFile(); TarArchiveOutputStream out = new TarArchiveOutputStream( new GZIPOutputStream(new FileOutputStream(gzipFile))); TarArchiveEntry entry = new TarArchiveEntry(p.getName()); entry.setSize(bytes.length); out.putArchiveEntry(entry); out.write(bytes); out.closeArchiveEntry(); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + ".tar.gz"))); ret.setSize(len); ret.setType(LocalResourceType.ARCHIVE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz")) .getModificationTime()); return ret; }
Example 9
Source File: TezClientUtils.java From incubator-tez with Apache License 2.0 | 5 votes |
/** * Helper function to create a YARN LocalResource * @param fs FileSystem object * @param p Path of resource to localize * @param type LocalResource Type * @return a YARN LocalResource for the given Path * @throws IOException */ static LocalResource createLocalResource(FileSystem fs, Path p, LocalResourceType type, LocalResourceVisibility visibility) throws IOException { LocalResource rsrc = Records.newRecord(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs.resolvePath(rsrcStat .getPath()))); rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); rsrc.setType(type); rsrc.setVisibility(visibility); return rsrc; }
Example 10
Source File: YARNRunner.java From tez with Apache License 2.0 | 5 votes |
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type) throws IOException { LocalResource rsrc = Records.newRecord(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs .getDefaultFileSystem().resolvePath(rsrcStat.getPath()))); rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); rsrc.setType(type); rsrc.setVisibility(LocalResourceVisibility.APPLICATION); return rsrc; }
Example 11
Source File: TestFSDownload.java From hadoop with Apache License 2.0 | 5 votes |
static LocalResource createTgzFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { byte[] bytes = new byte[len]; r.nextBytes(bytes); File gzipFile = new File(p.toUri().getPath() + ".tar.gz"); gzipFile.createNewFile(); TarArchiveOutputStream out = new TarArchiveOutputStream( new GZIPOutputStream(new FileOutputStream(gzipFile))); TarArchiveEntry entry = new TarArchiveEntry(p.getName()); entry.setSize(bytes.length); out.putArchiveEntry(entry); out.write(bytes); out.closeArchiveEntry(); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + ".tar.gz"))); ret.setSize(len); ret.setType(LocalResourceType.ARCHIVE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz")) .getModificationTime()); return ret; }
Example 12
Source File: VMResources.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
public void addFile(FileStatus fstatus) { Path fpath = fstatus.getPath() ; LocalResource libJar = Records.newRecord(LocalResource.class); libJar.setResource(ConverterUtils.getYarnUrlFromPath(fpath)); libJar.setSize(fstatus.getLen()); libJar.setTimestamp(fstatus.getModificationTime()); libJar.setType(LocalResourceType.FILE); libJar.setVisibility(LocalResourceVisibility.PUBLIC); put(fpath.getName(), libJar) ; }
Example 13
Source File: TestFSDownload.java From hadoop with Apache License 2.0 | 5 votes |
static LocalResource createFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException { createFile(files, p, len, r); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); ret.setSize(len); ret.setType(LocalResourceType.FILE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(p).getModificationTime()); return ret; }
Example 14
Source File: Utilities.java From XLearning with Apache License 2.0 | 5 votes |
public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(path); localResource.setResource(ConverterUtils.getYarnUrlFromPath(path)); localResource.setSize(fileStatus.getLen()); localResource.setTimestamp(fileStatus.getModificationTime()); localResource.setType(type); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
Example 15
Source File: Utils.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a YARN resource for the remote object at the given location. * * @param remoteRsrcPath remote location of the resource * @param resourceSize size of the resource * @param resourceModificationTime last modification time of the resource * * @return YARN resource */ private static LocalResource registerLocalResource( Path remoteRsrcPath, long resourceSize, long resourceModificationTime) { LocalResource localResource = Records.newRecord(LocalResource.class); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(resourceSize); localResource.setTimestamp(resourceModificationTime); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
Example 16
Source File: Utils.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); localResource.setSize(jarStat.getLen()); localResource.setTimestamp(jarStat.getModificationTime()); localResource.setType(LocalResourceType.FILE); localResource.setVisibility(LocalResourceVisibility.APPLICATION); return localResource; }
Example 17
Source File: TestContainerManager.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testContainerSetup() throws Exception { containerManager.start(); // ////// Create the resources for the container File dir = new File(tmpDir, "dir"); dir.mkdirs(); File file = new File(dir, "file"); PrintWriter fileWriter = new PrintWriter(file); fileWriter.write("Hello World!"); fileWriter.close(); // ////// Construct the Container-id ContainerId cId = createContainerId(0); // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(file.getAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class); rsrc_alpha.setResource(resource_alpha); rsrc_alpha.setSize(-1); rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION); rsrc_alpha.setType(LocalResourceType.FILE); rsrc_alpha.setTimestamp(file.lastModified()); String destinationFile = "dest_file"; Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); StartContainerRequest scRequest = StartContainerRequest.newInstance( containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager())); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); // Now ascertain that the resources are localised correctly. ApplicationId appId = cId.getApplicationAttemptId().getApplicationId(); String appIDStr = ConverterUtils.toString(appId); String containerIDStr = ConverterUtils.toString(cId); File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE); File userDir = new File(userCacheDir, user); File appCache = new File(userDir, ContainerLocalizer.APPCACHE); File appDir = new File(appCache, appIDStr); File containerDir = new File(appDir, containerIDStr); File targetFile = new File(containerDir, destinationFile); File sysDir = new File(localDir, ResourceLocalizationService.NM_PRIVATE_DIR); File appSysDir = new File(sysDir, appIDStr); File containerSysDir = new File(appSysDir, containerIDStr); for (File f : new File[] { localDir, sysDir, userCacheDir, appDir, appSysDir, containerDir, containerSysDir }) { Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists()); Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!", f.isDirectory()); } Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!", targetFile.exists()); // Now verify the contents of the file BufferedReader reader = new BufferedReader(new FileReader(targetFile)); Assert.assertEquals("Hello World!", reader.readLine()); Assert.assertEquals(null, reader.readLine()); }
Example 18
Source File: TestContainerManager.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testContainerSetup() throws Exception { containerManager.start(); // ////// Create the resources for the container File dir = new File(tmpDir, "dir"); dir.mkdirs(); File file = new File(dir, "file"); PrintWriter fileWriter = new PrintWriter(file); fileWriter.write("Hello World!"); fileWriter.close(); // ////// Construct the Container-id ContainerId cId = createContainerId(0); // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(file.getAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class); rsrc_alpha.setResource(resource_alpha); rsrc_alpha.setSize(-1); rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION); rsrc_alpha.setType(LocalResourceType.FILE); rsrc_alpha.setTimestamp(file.lastModified()); String destinationFile = "dest_file"; Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); StartContainerRequest scRequest = StartContainerRequest.newInstance( containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager())); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); // Now ascertain that the resources are localised correctly. ApplicationId appId = cId.getApplicationAttemptId().getApplicationId(); String appIDStr = ConverterUtils.toString(appId); String containerIDStr = ConverterUtils.toString(cId); File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE); File userDir = new File(userCacheDir, user); File appCache = new File(userDir, ContainerLocalizer.APPCACHE); File appDir = new File(appCache, appIDStr); File containerDir = new File(appDir, containerIDStr); File targetFile = new File(containerDir, destinationFile); File sysDir = new File(localDir, ResourceLocalizationService.NM_PRIVATE_DIR); File appSysDir = new File(sysDir, appIDStr); File containerSysDir = new File(appSysDir, containerIDStr); for (File f : new File[] { localDir, sysDir, userCacheDir, appDir, appSysDir, containerDir, containerSysDir }) { Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists()); Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!", f.isDirectory()); } Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!", targetFile.exists()); // Now verify the contents of the file BufferedReader reader = new BufferedReader(new FileReader(targetFile)); Assert.assertEquals("Hello World!", reader.readLine()); Assert.assertEquals(null, reader.readLine()); }
Example 19
Source File: TestLogAggregationService.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testLogAggregationForRealContainerLaunch() throws IOException, InterruptedException, YarnException { this.containerManager.start(); File scriptFile = new File(tmpDir, "scriptFile.sh"); PrintWriter fileWriter = new PrintWriter(scriptFile); fileWriter.write("\necho Hello World! Stdout! > " + new File(localLogDir, "stdout")); fileWriter.write("\necho Hello World! Stderr! > " + new File(localLogDir, "stderr")); fileWriter.write("\necho Hello World! Syslog! > " + new File(localLogDir, "syslog")); fileWriter.close(); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); // ////// Construct the Container-id ApplicationId appId = ApplicationId.newInstance(0, 0); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1); ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class); rsrc_alpha.setResource(resource_alpha); rsrc_alpha.setSize(-1); rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION); rsrc_alpha.setType(LocalResourceType.FILE); rsrc_alpha.setTimestamp(scriptFile.lastModified()); String destinationFile = "dest_file"; Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); List<String> commands = new ArrayList<String>(); commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken( cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager())); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); this.containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(this.containerManager, cId, ContainerState.COMPLETE); this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays .asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN)); this.containerManager.stop(); }
Example 20
Source File: TestContainerManager.java From big-c with Apache License 2.0 | 4 votes |
private void testContainerLaunchAndExit(int exitCode) throws IOException, InterruptedException, YarnException { File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); File processStartFile = new File(tmpDir, "start_file.txt").getAbsoluteFile(); // ////// Construct the Container-id ContainerId cId = createContainerId(0); if (Shell.WINDOWS) { fileWriter.println("@echo Hello World!> " + processStartFile); fileWriter.println("@echo " + cId + ">> " + processStartFile); if (exitCode != 0) { fileWriter.println("@exit " + exitCode); } } else { fileWriter.write("\numask 0"); // So that start file is readable by the test fileWriter.write("\necho Hello World! > " + processStartFile); fileWriter.write("\necho $$ >> " + processStartFile); // Have script throw an exit code at the end if (exitCode != 0) { fileWriter.write("\nexit "+exitCode); } } fileWriter.close(); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); URL resource_alpha = ConverterUtils.getYarnUrlFromPath(localFS .makeQualified(new Path(scriptFile.getAbsolutePath()))); LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class); rsrc_alpha.setResource(resource_alpha); rsrc_alpha.setSize(-1); rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION); rsrc_alpha.setType(LocalResourceType.FILE); rsrc_alpha.setTimestamp(scriptFile.lastModified()); String destinationFile = "dest_file"; Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); StartContainerRequest scRequest = StartContainerRequest.newInstance( containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager())); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); List<ContainerId> containerIds = new ArrayList<ContainerId>(); containerIds.add(cId); GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); // Verify exit status matches exit state of script Assert.assertEquals(exitCode, containerStatus.getExitStatus()); }