org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl Java Examples
The following examples show how to use
org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: LocalResourceStatusPBImpl.java From hadoop with Apache License 2.0 | 6 votes |
|
private void mergeLocalToBuilder() {
if (this.resource != null &&
!((LocalResourcePBImpl)this.resource).getProto()
.equals(builder.getResource())) {
builder.setResource(convertToProtoFormat(this.resource));
}
if (this.localPath != null &&
!((URLPBImpl)this.localPath).getProto()
.equals(builder.getLocalPath())) {
builder.setLocalPath(convertToProtoFormat(this.localPath));
}
if (this.exception != null &&
!((SerializedExceptionPBImpl)this.exception).getProto()
.equals(builder.getException())) {
builder.setException(convertToProtoFormat(this.exception));
}
}
Example #2
| Source File: ResourceLocalizationSpecPBImpl.java From hadoop with Apache License 2.0 | 6 votes |
|
private void mergeLocalToBuilder() {
ResourceLocalizationSpecProtoOrBuilder l = viaProto ? proto : builder;
if (this.resource != null
&& !(l.getResource()
.equals(((LocalResourcePBImpl) resource).getProto()))) {
maybeInitBuilder();
builder.setResource(((LocalResourcePBImpl) resource).getProto());
}
if (this.destinationDirectory != null
&& !(l.getDestinationDirectory()
.equals(((URLPBImpl) destinationDirectory).getProto()))) {
maybeInitBuilder();
builder.setDestinationDirectory(((URLPBImpl) destinationDirectory)
.getProto());
}
}
Example #3
| Source File: LocalResourceStatusPBImpl.java From big-c with Apache License 2.0 | 6 votes |
|
private void mergeLocalToBuilder() {
if (this.resource != null &&
!((LocalResourcePBImpl)this.resource).getProto()
.equals(builder.getResource())) {
builder.setResource(convertToProtoFormat(this.resource));
}
if (this.localPath != null &&
!((URLPBImpl)this.localPath).getProto()
.equals(builder.getLocalPath())) {
builder.setLocalPath(convertToProtoFormat(this.localPath));
}
if (this.exception != null &&
!((SerializedExceptionPBImpl)this.exception).getProto()
.equals(builder.getException())) {
builder.setException(convertToProtoFormat(this.exception));
}
}
Example #4
| Source File: ResourceLocalizationSpecPBImpl.java From big-c with Apache License 2.0 | 6 votes |
|
private void mergeLocalToBuilder() {
ResourceLocalizationSpecProtoOrBuilder l = viaProto ? proto : builder;
if (this.resource != null
&& !(l.getResource()
.equals(((LocalResourcePBImpl) resource).getProto()))) {
maybeInitBuilder();
builder.setResource(((LocalResourcePBImpl) resource).getProto());
}
if (this.destinationDirectory != null
&& !(l.getDestinationDirectory()
.equals(((URLPBImpl) destinationDirectory).getProto()))) {
maybeInitBuilder();
builder.setDestinationDirectory(((URLPBImpl) destinationDirectory)
.getProto());
}
}
Example #5
| Source File: DagTypeConverters.java From incubator-tez with Apache License 2.0 | 6 votes |
|
public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
List<PlanLocalResource> localResourcesList) {
Map<String, LocalResource> map = new HashMap<String, LocalResource>();
for(PlanLocalResource res : localResourcesList){
LocalResource r = new LocalResourcePBImpl();
//NOTE: have to check every optional field in protobuf generated classes for existence before accessing
//else we will receive a default value back, eg ""
if(res.hasPattern()){
r.setPattern(res.getPattern());
}
r.setResource(ConverterUtils.getYarnUrlFromPath(new Path(res.getUri()))); // see above notes on HDFS URL handling
r.setSize(res.getSize());
r.setTimestamp(res.getTimeStamp());
r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
map.put(res.getName(), r);
}
return map;
}
Example #6
| Source File: DagTypeConverters.java From tez with Apache License 2.0 | 6 votes |
|
public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
List<PlanLocalResource> localResourcesList) {
Map<String, LocalResource> map = new HashMap<String, LocalResource>();
for(PlanLocalResource res : localResourcesList){
LocalResource r = new LocalResourcePBImpl();
//NOTE: have to check every optional field in protobuf generated classes for existence before accessing
//else we will receive a default value back, eg ""
if(res.hasPattern()){
r.setPattern(res.getPattern());
}
r.setResource(convertToYarnURL(res.getUri()));
r.setSize(res.getSize());
r.setTimestamp(res.getTimeStamp());
r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
map.put(res.getName(), r);
}
return map;
}
Example #7
| Source File: TestPBRecordImpl.java From hadoop with Apache License 2.0 | 5 votes |
|
static LocalResource createResource() {
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
assertTrue(ret instanceof LocalResourcePBImpl);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
"hdfs://y.ak:8020/foo/bar")));
ret.setSize(4344L);
ret.setTimestamp(3141592653589793L);
ret.setVisibility(LocalResourceVisibility.PUBLIC);
return ret;
}
Example #8
| Source File: TestPBRecordImpl.java From big-c with Apache License 2.0 | 5 votes |
|
static LocalResource createResource() {
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
assertTrue(ret instanceof LocalResourcePBImpl);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
"hdfs://y.ak:8020/foo/bar")));
ret.setSize(4344L);
ret.setTimestamp(3141592653589793L);
ret.setVisibility(LocalResourceVisibility.PUBLIC);
return ret;
}
Example #9
| Source File: LocalResourceStatusPBImpl.java From hadoop with Apache License 2.0 | 4 votes |
|
private LocalResourceProto convertToProtoFormat(LocalResource rsrc) {
return ((LocalResourcePBImpl)rsrc).getProto();
}
Example #10
| Source File: LocalResourceStatusPBImpl.java From hadoop with Apache License 2.0 | 4 votes |
|
private LocalResourcePBImpl convertFromProtoFormat(LocalResourceProto rsrc) {
return new LocalResourcePBImpl(rsrc);
}
Example #11
| Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
final String user = "someuser";
final ApplicationId appId = ApplicationId.newInstance(1, 1);
// This is a random path. NO File creation will take place at this place.
final Path localDir = new Path("/tmp");
Configuration conf = new YarnConfiguration();
DrainDispatcher dispatcher = null;
dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler =
mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler =
mock(EventHandler.class);
dispatcher.register(LocalizerEventType.class, localizerEventHandler);
dispatcher.register(ContainerEventType.class, containerEventHandler);
DeletionService mockDelService = mock(DeletionService.class);
NMStateStoreService stateStore = mock(NMStateStoreService.class);
try {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
appId, dispatcher, false, conf, stateStore);
// Container 1 needs lr1 resource
ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
LocalResourceVisibility.APPLICATION);
LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
// Container 1 requests lr1 to be localized
ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
LocalResourceVisibility.APPLICATION, lc1);
tracker.handle(reqEvent1);
dispatcher.await();
// Simulate the process of localization of lr1
Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
null);
ArgumentCaptor<LocalResourceProto> localResourceCaptor =
ArgumentCaptor.forClass(LocalResourceProto.class);
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
verify(stateStore).startResourceLocalization(eq(user), eq(appId),
localResourceCaptor.capture(), pathCaptor.capture());
LocalResourceProto lrProto = localResourceCaptor.getValue();
Path localizedPath1 = pathCaptor.getValue();
Assert.assertEquals(lr1,
new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
// Simulate lr1 getting localized
ResourceLocalizedEvent rle1 =
new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
tracker.handle(rle1);
dispatcher.await();
ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor =
ArgumentCaptor.forClass(LocalizedResourceProto.class);
verify(stateStore).finishResourceLocalization(eq(user), eq(appId),
localizedProtoCaptor.capture());
LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
Assert.assertEquals(lr1, new LocalResourceRequest(
new LocalResourcePBImpl(localizedProto.getResource())));
Assert.assertEquals(localizedPath1.toString(),
localizedProto.getLocalPath());
LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
Assert.assertNotNull(localizedRsrc1);
// simulate release and retention processing
tracker.handle(new ResourceReleaseEvent(lr1, cId1));
dispatcher.await();
boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);
Assert.assertTrue(removeResult);
verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
eq(localizedPath1));
} finally {
if (dispatcher != null) {
dispatcher.stop();
}
}
}
Example #12
| Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
final String user = "someuser";
final ApplicationId appId = ApplicationId.newInstance(1, 1);
// This is a random path. NO File creation will take place at this place.
final Path localDir = new Path("/tmp");
Configuration conf = new YarnConfiguration();
DrainDispatcher dispatcher = null;
dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler =
mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler =
mock(EventHandler.class);
dispatcher.register(LocalizerEventType.class, localizerEventHandler);
dispatcher.register(ContainerEventType.class, containerEventHandler);
NMStateStoreService stateStore = mock(NMStateStoreService.class);
try {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
appId, dispatcher, false, conf, stateStore);
// Container 1 needs lr1 resource
ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
LocalResourceVisibility.APPLICATION);
LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
// Container 1 requests lr1 to be localized
ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
LocalResourceVisibility.APPLICATION, lc1);
tracker.handle(reqEvent1);
dispatcher.await();
// Simulate the process of localization of lr1
Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
null);
ArgumentCaptor<LocalResourceProto> localResourceCaptor =
ArgumentCaptor.forClass(LocalResourceProto.class);
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
verify(stateStore).startResourceLocalization(eq(user), eq(appId),
localResourceCaptor.capture(), pathCaptor.capture());
LocalResourceProto lrProto = localResourceCaptor.getValue();
Path localizedPath1 = pathCaptor.getValue();
Assert.assertEquals(lr1,
new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
ResourceFailedLocalizationEvent rfe1 =
new ResourceFailedLocalizationEvent(
lr1, new Exception("Test").toString());
tracker.handle(rfe1);
dispatcher.await();
verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
eq(localizedPath1));
} finally {
if (dispatcher != null) {
dispatcher.stop();
}
}
}
Example #13
| Source File: LocalResourceStatusPBImpl.java From big-c with Apache License 2.0 | 4 votes |
|
private LocalResourceProto convertToProtoFormat(LocalResource rsrc) {
return ((LocalResourcePBImpl)rsrc).getProto();
}
Example #14
| Source File: LocalResourceStatusPBImpl.java From big-c with Apache License 2.0 | 4 votes |
|
private LocalResourcePBImpl convertFromProtoFormat(LocalResourceProto rsrc) {
return new LocalResourcePBImpl(rsrc);
}
Example #15
| Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
final String user = "someuser";
final ApplicationId appId = ApplicationId.newInstance(1, 1);
// This is a random path. NO File creation will take place at this place.
final Path localDir = new Path("/tmp");
Configuration conf = new YarnConfiguration();
DrainDispatcher dispatcher = null;
dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler =
mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler =
mock(EventHandler.class);
dispatcher.register(LocalizerEventType.class, localizerEventHandler);
dispatcher.register(ContainerEventType.class, containerEventHandler);
DeletionService mockDelService = mock(DeletionService.class);
NMStateStoreService stateStore = mock(NMStateStoreService.class);
try {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
appId, dispatcher, false, conf, stateStore);
// Container 1 needs lr1 resource
ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
LocalResourceVisibility.APPLICATION);
LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
// Container 1 requests lr1 to be localized
ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
LocalResourceVisibility.APPLICATION, lc1);
tracker.handle(reqEvent1);
dispatcher.await();
// Simulate the process of localization of lr1
Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);
ArgumentCaptor<LocalResourceProto> localResourceCaptor =
ArgumentCaptor.forClass(LocalResourceProto.class);
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
verify(stateStore).startResourceLocalization(eq(user), eq(appId),
localResourceCaptor.capture(), pathCaptor.capture());
LocalResourceProto lrProto = localResourceCaptor.getValue();
Path localizedPath1 = pathCaptor.getValue();
Assert.assertEquals(lr1,
new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
// Simulate lr1 getting localized
ResourceLocalizedEvent rle1 =
new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
tracker.handle(rle1);
dispatcher.await();
ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor =
ArgumentCaptor.forClass(LocalizedResourceProto.class);
verify(stateStore).finishResourceLocalization(eq(user), eq(appId),
localizedProtoCaptor.capture());
LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
Assert.assertEquals(lr1, new LocalResourceRequest(
new LocalResourcePBImpl(localizedProto.getResource())));
Assert.assertEquals(localizedPath1.toString(),
localizedProto.getLocalPath());
LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
Assert.assertNotNull(localizedRsrc1);
// simulate release and retention processing
tracker.handle(new ResourceReleaseEvent(lr1, cId1));
dispatcher.await();
boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);
Assert.assertTrue(removeResult);
verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
eq(localizedPath1));
} finally {
if (dispatcher != null) {
dispatcher.stop();
}
}
}
Example #16
| Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
final String user = "someuser";
final ApplicationId appId = ApplicationId.newInstance(1, 1);
// This is a random path. NO File creation will take place at this place.
final Path localDir = new Path("/tmp");
Configuration conf = new YarnConfiguration();
DrainDispatcher dispatcher = null;
dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler =
mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler =
mock(EventHandler.class);
dispatcher.register(LocalizerEventType.class, localizerEventHandler);
dispatcher.register(ContainerEventType.class, containerEventHandler);
NMStateStoreService stateStore = mock(NMStateStoreService.class);
try {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
appId, dispatcher, false, conf, stateStore);
// Container 1 needs lr1 resource
ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
LocalResourceVisibility.APPLICATION);
LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
// Container 1 requests lr1 to be localized
ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
LocalResourceVisibility.APPLICATION, lc1);
tracker.handle(reqEvent1);
dispatcher.await();
// Simulate the process of localization of lr1
Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);
ArgumentCaptor<LocalResourceProto> localResourceCaptor =
ArgumentCaptor.forClass(LocalResourceProto.class);
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
verify(stateStore).startResourceLocalization(eq(user), eq(appId),
localResourceCaptor.capture(), pathCaptor.capture());
LocalResourceProto lrProto = localResourceCaptor.getValue();
Path localizedPath1 = pathCaptor.getValue();
Assert.assertEquals(lr1,
new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
ResourceFailedLocalizationEvent rfe1 =
new ResourceFailedLocalizationEvent(
lr1, new Exception("Test").toString());
tracker.handle(rfe1);
dispatcher.await();
verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
eq(localizedPath1));
} finally {
if (dispatcher != null) {
dispatcher.stop();
}
}
}
