org.apache.hadoop.yarn.api.records.LocalResourceVisibility Java Examples
The following examples show how to use
org.apache.hadoop.yarn.api.records.LocalResourceVisibility.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerLocalizer.java From big-c with Apache License 2.0 | 6 votes |
static ResourceLocalizationSpec getMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = mock(ResourceLocalizationSpec.class); LocalResource rsrc = mock(LocalResource.class); String name = Long.toHexString(r.nextLong()); URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class); when(uri.getScheme()).thenReturn("file"); when(uri.getHost()).thenReturn(null); when(uri.getFile()).thenReturn("/local/" + vis + "/" + name); when(rsrc.getResource()).thenReturn(uri); when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L); when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L); when(rsrc.getType()).thenReturn(LocalResourceType.FILE); when(rsrc.getVisibility()).thenReturn(vis); when(resourceLocalizationSpec.getResource()).thenReturn(rsrc); when(resourceLocalizationSpec.getDestinationDirectory()). thenReturn(ConverterUtils.getYarnUrlFromPath(p)); return resourceLocalizationSpec; }
Example #2
Source File: TestResourceRetention.java From big-c with Apache License 2.0 | 6 votes |
LocalResourcesTracker createMockTracker(String user, final long rsrcSize, long nRsrcs, long timestamp, long tsstep) { Configuration conf = new Configuration(); ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources = new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>(); LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null, null, trackerResources, false, conf, new NMNullStateStoreService())); for (int i = 0; i < nRsrcs; ++i) { final LocalResourceRequest req = new LocalResourceRequest( new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null); final long ts = timestamp + i * tsstep; final Path p = new Path("file:///local/" + user + "/rsrc" + i); LocalizedResource rsrc = new LocalizedResource(req, null) { @Override public int getRefCount() { return 0; } @Override public long getSize() { return rsrcSize; } @Override public Path getLocalPath() { return p; } @Override public long getTimestamp() { return ts; } @Override public ResourceState getState() { return ResourceState.LOCALIZED; } }; trackerResources.put(req, rsrc); } return ret; }
Example #3
Source File: TestResourceRetention.java From hadoop with Apache License 2.0 | 6 votes |
LocalResourcesTracker createMockTracker(String user, final long rsrcSize, long nRsrcs, long timestamp, long tsstep) { Configuration conf = new Configuration(); ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources = new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>(); LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null, null, trackerResources, false, conf, new NMNullStateStoreService(),null)); for (int i = 0; i < nRsrcs; ++i) { final LocalResourceRequest req = new LocalResourceRequest( new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null); final long ts = timestamp + i * tsstep; final Path p = new Path("file:///local/" + user + "/rsrc" + i); LocalizedResource rsrc = new LocalizedResource(req, null) { @Override public int getRefCount() { return 0; } @Override public long getSize() { return rsrcSize; } @Override public Path getLocalPath() { return p; } @Override public long getTimestamp() { return ts; } @Override public ResourceState getState() { return ResourceState.LOCALIZED; } }; trackerResources.put(req, rsrc); } return ret; }
Example #4
Source File: TestContainer.java From hadoop with Apache License 2.0 | 6 votes |
/** * Verify correct container request events sent to localizer. */ @Test public void testLocalizationRequest() throws Exception { WrappedContainer wc = null; try { wc = new WrappedContainer(7, 314159265358979L, 4344, "yak"); assertEquals(ContainerState.NEW, wc.c.getContainerState()); wc.initContainer(); // Verify request for public/private resources to localizer ResourcesRequestedMatcher matchesReq = new ResourcesRequestedMatcher(wc.localResources, EnumSet.of( LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE, LocalResourceVisibility.APPLICATION)); verify(wc.localizerBus).handle(argThat(matchesReq)); assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState()); } finally { if (wc != null) { wc.finished(); } } }
Example #5
Source File: TestLocalizerResourceMapper.java From samza with Apache License 2.0 | 6 votes |
@Test public void testResourceMapWithDefaultValues() { Map<String, String> configMap = new HashMap<>(); configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme"); Config conf = new MapConfig(configMap); YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName()); LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration); Map<String, LocalResource> resourceMap = mapper.getResourceMap(); assertNull("Resource does not exist with a name readme", resourceMap.get("readme")); assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1")); assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost()); assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType()); assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility()); }
Example #6
Source File: TestFSDownload.java From big-c with Apache License 2.0 | 6 votes |
static LocalResource createJar(FileContext files, Path p, LocalResourceVisibility vis) throws IOException { LOG.info("Create jar file " + p); File jarFile = new File((files.makeQualified(p)).toUri()); FileOutputStream stream = new FileOutputStream(jarFile); LOG.info("Create jar out stream "); JarOutputStream out = new JarOutputStream(stream, new Manifest()); LOG.info("Done writing jar stream "); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); FileStatus status = files.getFileStatus(p); ret.setSize(status.getLen()); ret.setTimestamp(status.getModificationTime()); ret.setType(LocalResourceType.PATTERN); ret.setVisibility(vis); ret.setPattern("classes/.*"); return ret; }
Example #7
Source File: ResourceLocalizationService.java From big-c with Apache License 2.0 | 6 votes |
private Path getPathForLocalization(LocalResource rsrc) throws IOException, URISyntaxException { String user = context.getUser(); ApplicationId appId = context.getContainerId().getApplicationAttemptId().getApplicationId(); LocalResourceVisibility vis = rsrc.getVisibility(); LocalResourcesTracker tracker = getLocalResourcesTracker(vis, user, appId); String cacheDirectory = null; if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only cacheDirectory = getUserFileCachePath(user); } else {// APPLICATION ONLY cacheDirectory = getAppFileCachePath(user, appId.toString()); } Path dirPath = dirsHandler.getLocalPathForWrite(cacheDirectory, ContainerLocalizer.getEstimatedSize(rsrc), false); return tracker.getPathForLocalization(new LocalResourceRequest(rsrc), dirPath); }
Example #8
Source File: TestSharedCacheUploader.java From hadoop with Apache License 2.0 | 6 votes |
/** * If the localPath does not exists, getActualPath should get to one level * down */ @Test public void testGetActualPath() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); LocalResource resource = mock(LocalResource.class); // give public visibility when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC); Path localPath = new Path("foo.jar"); String user = "joe"; SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class); FileSystem fs = mock(FileSystem.class); FileSystem localFs = mock(FileSystem.class); // stub it to return a status that indicates a directory FileStatus status = mock(FileStatus.class); when(status.isDirectory()).thenReturn(true); when(localFs.getFileStatus(localPath)).thenReturn(status); SharedCacheUploader spied = createSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); Path actualPath = spied.getActualPath(); assertEquals(actualPath.getName(), localPath.getName()); assertEquals(actualPath.getParent().getName(), localPath.getName()); }
Example #9
Source File: TestFSDownload.java From hadoop with Apache License 2.0 | 6 votes |
static LocalResource createJarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { byte[] bytes = new byte[len]; r.nextBytes(bytes); File archiveFile = new File(p.toUri().getPath() + ".jar"); archiveFile.createNewFile(); JarOutputStream out = new JarOutputStream( new FileOutputStream(archiveFile)); out.putNextEntry(new JarEntry(p.getName())); out.write(bytes); out.closeEntry(); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + ".jar"))); ret.setSize(len); ret.setType(LocalResourceType.ARCHIVE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar")) .getModificationTime()); return ret; }
Example #10
Source File: YarnApplicationFileUploader.java From flink with Apache License 2.0 | 6 votes |
/** * Register all the files in the provided lib directories as Yarn local resources with PUBLIC visibility, which * means that they will be cached in the nodes and reused by different applications. * * @return list of class paths with the file name */ List<String> registerProvidedLocalResources() { checkNotNull(localResources); final ArrayList<String> classPaths = new ArrayList<>(); providedSharedLibs.forEach( (fileName, fileStatus) -> { final Path filePath = fileStatus.getPath(); LOG.debug("Using remote file {} to register local resource", filePath); final YarnLocalResourceDescriptor descriptor = YarnLocalResourceDescriptor .fromFileStatus(fileName, fileStatus, LocalResourceVisibility.PUBLIC); localResources.put(fileName, descriptor.toLocalResource()); remotePaths.add(filePath); envShipResourceList.add(descriptor); if (!isFlinkDistJar(filePath.getName()) && !isPlugin(filePath)) { classPaths.add(fileName); } else if (isFlinkDistJar(filePath.getName())) { flinkDist = descriptor; } }); return classPaths; }
Example #11
Source File: TestContainer.java From big-c with Apache License 2.0 | 6 votes |
/** * Verify correct container request events sent to localizer. */ @Test public void testLocalizationRequest() throws Exception { WrappedContainer wc = null; try { wc = new WrappedContainer(7, 314159265358979L, 4344, "yak"); assertEquals(ContainerState.NEW, wc.c.getContainerState()); wc.initContainer(); // Verify request for public/private resources to localizer ResourcesRequestedMatcher matchesReq = new ResourcesRequestedMatcher(wc.localResources, EnumSet.of( LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE, LocalResourceVisibility.APPLICATION)); verify(wc.localizerBus).handle(argThat(matchesReq)); assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState()); } finally { if (wc != null) { wc.finished(); } } }
Example #12
Source File: LocalizerResourceMapper.java From samza with Apache License 2.0 | 6 votes |
private Map<String, LocalResource> buildResourceMapping() { ImmutableMap.Builder<String, LocalResource> localResourceMapBuilder = ImmutableMap.builder(); List<String> resourceNames = resourceConfig.getResourceNames(); for (String resourceName : resourceNames) { String resourceLocalName = resourceConfig.getResourceLocalName(resourceName); LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName); LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName); Path resourcePath = resourceConfig.getResourcePath(resourceName); LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility); localResourceMapBuilder.put(resourceLocalName, localResource); log.info("preparing local resource: {}", resourceLocalName); } return localResourceMapBuilder.build(); }
Example #13
Source File: ContainerImpl.java From hadoop with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") // dispatcher not typed public void cleanup() { Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); if (!publicRsrcs.isEmpty()) { rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs); } if (!privateRsrcs.isEmpty()) { rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs); } if (!appRsrcs.isEmpty()) { rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs); } dispatcher.getEventHandler().handle( new ContainerLocalizationCleanupEvent(this, rsrc)); }
Example #14
Source File: JstormOnYarn.java From jstorm with Apache License 2.0 | 6 votes |
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance( ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); }
Example #15
Source File: IgniteYarnUtils.java From ignite with Apache License 2.0 | 6 votes |
/** * @param file Path. * @param fs File system. * @param type Local resource type. * @throws Exception If failed. */ public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type) throws Exception { LocalResource resource = Records.newRecord(LocalResource.class); file = fs.makeQualified(file); FileStatus stat = fs.getFileStatus(file); resource.setResource(ConverterUtils.getYarnUrlFromPath(file)); resource.setSize(stat.getLen()); resource.setTimestamp(stat.getModificationTime()); resource.setType(type); resource.setVisibility(LocalResourceVisibility.APPLICATION); return resource; }
Example #16
Source File: TestFSDownload.java From big-c with Apache License 2.0 | 6 votes |
static LocalResource createJarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { byte[] bytes = new byte[len]; r.nextBytes(bytes); File archiveFile = new File(p.toUri().getPath() + ".jar"); archiveFile.createNewFile(); JarOutputStream out = new JarOutputStream( new FileOutputStream(archiveFile)); out.putNextEntry(new JarEntry(p.getName())); out.write(bytes); out.closeEntry(); out.close(); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + ".jar"))); ret.setSize(len); ret.setType(LocalResourceType.ARCHIVE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar")) .getModificationTime()); return ret; }
Example #17
Source File: TestTezClient.java From tez with Apache License 2.0 | 6 votes |
@Test(timeout = 5000) public void testClientResubmit() throws Exception { TezClientForTest client = configureAndCreateTezClient(null, true, null); client.start(); Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1", LocalResource.newInstance( URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1, Resource.newInstance(1, 1)); vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC"); Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1, Resource.newInstance(1, 1)); vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC"); DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG); for (int i = 0; i < 3; ++i) { try { client.submitDAG(dag); Assert.fail("Expected TezUncheckedException here."); } catch(TezUncheckedException ex) { Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found")); } } client.stop(); }
Example #18
Source File: TestContainer.java From hadoop with Apache License 2.0 | 5 votes |
private void verifyCleanupCall(WrappedContainer wc) throws Exception { ResourcesReleasedMatcher matchesReq = new ResourcesReleasedMatcher(wc.localResources, EnumSet.of( LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE, LocalResourceVisibility.APPLICATION)); verify(wc.localizerBus).handle(argThat(matchesReq)); }
Example #19
Source File: LocalResourceRequest.java From big-c with Apache License 2.0 | 5 votes |
LocalResourceRequest(Path loc, long timestamp, LocalResourceType type, LocalResourceVisibility visibility, String pattern) { this.loc = loc; this.timestamp = timestamp; this.type = type; this.visibility = visibility; this.pattern = pattern; }
Example #20
Source File: TestLocalResource.java From hadoop with Apache License 2.0 | 5 votes |
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size, long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern) throws URISyntaxException { org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri())); ret.setSize(size); ret.setTimestamp(timestamp); ret.setType(type); ret.setVisibility(state); ret.setPattern(pattern); return ret; }
Example #21
Source File: TestMRRJobsDAGApi.java From tez with Apache License 2.0 | 5 votes |
private static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat .getPath())); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); }
Example #22
Source File: TestFSDownload.java From big-c with Apache License 2.0 | 5 votes |
static LocalResource createFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException { createFile(files, p, len, r); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(p)); ret.setSize(len); ret.setType(LocalResourceType.FILE); ret.setVisibility(vis); ret.setTimestamp(files.getFileStatus(p).getModificationTime()); return ret; }
Example #23
Source File: TestDAGVerify.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testDAGCreateDataInference() { Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName)); Map<String, LocalResource> lrs1 = Maps.newHashMap(); String lrName1 = "LR1"; lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> lrs2 = Maps.newHashMap(); String lrName2 = "LR2"; lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Set<String> hosts = Sets.newHashSet(); hosts.add("h1"); hosts.add("h2"); List<TaskLocationHint> taskLocationHints = Lists.newLinkedList(); taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null)); taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null)); VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints); DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"), InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2); v1.addDataSource("i1", ds); DAG dag = DAG.create("testDag"); dag.addVertex(v1); dag.addTaskLocalFiles(lrs1); DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true); Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName()); VertexPlan vPlan = dagPlan.getVertex(0); PlanTaskConfiguration taskPlan = vPlan.getTaskConfig(); Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks()); Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb()); Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName()); Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount()); }
Example #24
Source File: YarnHelixUtils.java From incubator-gobblin with Apache License 2.0 | 5 votes |
/** * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}. * * @param fs a {@link FileSystem} instance * @param destFilePath the destination file path * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file * @param resourceMap a {@link Map} of file names to their corresponding * {@link org.apache.hadoop.yarn.api.records.LocalResource}s * @throws IOException if there's something wrong adding the file as a * {@link org.apache.hadoop.yarn.api.records.LocalResource} */ public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType, Map<String, LocalResource> resourceMap) throws IOException { LocalResource fileResource = Records.newRecord(LocalResource.class); FileStatus fileStatus = fs.getFileStatus(destFilePath); fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath)); fileResource.setSize(fileStatus.getLen()); fileResource.setTimestamp(fileStatus.getModificationTime()); fileResource.setType(resourceType); fileResource.setVisibility(LocalResourceVisibility.APPLICATION); resourceMap.put(destFilePath.getName(), fileResource); }
Example #25
Source File: TestLocalizedResource.java From big-c with Apache License 2.0 | 5 votes |
public LocalizerEventMatcher(ContainerId idRef, Credentials creds, LocalResourceVisibility vis, LocalizerEventType type) { this.vis = vis; this.type = type; this.creds = creds; this.idRef = idRef; }
Example #26
Source File: TaskAttemptImpl.java From hadoop with Apache License 2.0 | 5 votes |
/** * Create a {@link LocalResource} record with all the given parameters. */ private static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat .getPath())); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); }
Example #27
Source File: LocalResourcePBImpl.java From big-c with Apache License 2.0 | 5 votes |
@Override public synchronized void setVisibility(LocalResourceVisibility visibility) { maybeInitBuilder(); if (visibility == null) { builder.clearVisibility(); return; } builder.setVisibility(convertToProtoFormat(visibility)); }
Example #28
Source File: TestContainer.java From big-c with Apache License 2.0 | 5 votes |
ResourcesReleasedMatcher(Map<String, LocalResource> allResources, EnumSet<LocalResourceVisibility> vis) throws URISyntaxException { for (Entry<String, LocalResource> e : allResources.entrySet()) { if (vis.contains(e.getValue().getVisibility())) { resources.add(new LocalResourceRequest(e.getValue())); } } }
Example #29
Source File: LocalizerResourceRequestEvent.java From big-c with Apache License 2.0 | 5 votes |
public LocalizerResourceRequestEvent(LocalizedResource resource, LocalResourceVisibility vis, LocalizerContext context, String pattern) { super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION, ConverterUtils.toString(context.getContainerId())); this.vis = vis; this.context = context; this.resource = resource; this.pattern = pattern; }
Example #30
Source File: TestContainer.java From hadoop with Apache License 2.0 | 5 votes |
private static Entry<String, LocalResource> getMockRsrc(Random r, LocalResourceVisibility vis) { String name = Long.toHexString(r.nextLong()); URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name); LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false); return new SimpleEntry<String, LocalResource>(name, rsrc); }