Java Code Examples for org.apache.hadoop.net.NetworkTopology#DEFAULT_RACK

private ContainerRequestEvent
    createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
        boolean earlierFailedAttempt, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  Resource containerNeed = Resource.newInstance(memory, 1);
  if (earlierFailedAttempt) {
    return ContainerRequestEvent
        .createContainerRequestEventForFailedContainer(attemptId,
            containerNeed);
  }
  return new ContainerRequestEvent(attemptId, containerNeed, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK });
}
 

private void resolveNetworkLocation (DatanodeDescriptor node) {
  List<String> names = new ArrayList<String>(1);
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    // get the node's IP address
    names.add(node.getHost());
  } else {
    // get the node's host name
    String hostName = node.getHostName();
    int colon = hostName.indexOf(":");
    hostName = (colon==-1)?hostName:hostName.substring(0,colon);
    names.add(hostName);
  }
  
  // resolve its network location
  List<String> rName = dnsToSwitchMapping.resolve(names);
  String networkLocation;
  if (rName == null) {
    LOG.error("The resolve call returned null! Using " + 
        NetworkTopology.DEFAULT_RACK + " for host " + names);
    networkLocation = NetworkTopology.DEFAULT_RACK;
  } else {
    networkLocation = rName.get(0);
  }
  node.setNetworkLocation(networkLocation);
}
 

private ContainerRequestEvent
    createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
        boolean earlierFailedAttempt, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  Resource containerNeed = Resource.newInstance(memory, 1);
  if (earlierFailedAttempt) {
    return ContainerRequestEvent
        .createContainerRequestEventForFailedContainer(attemptId,
            containerNeed);
  }
  return new ContainerRequestEvent(attemptId, containerNeed, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK });
}
 

private void validateLabelsRequests(ResourceRequest resourceRequest,
    boolean isReduce) {
  switch (resourceRequest.getResourceName()) {
  case "map":
  case "reduce":
  case NetworkTopology.DEFAULT_RACK:
    Assert.assertNull(resourceRequest.getNodeLabelExpression());
    break;
  case "*":
    Assert.assertEquals(isReduce ? "ReduceNodes" : "MapNodes",
        resourceRequest.getNodeLabelExpression());
    break;
  default:
    Assert.fail("Invalid resource location "
        + resourceRequest.getResourceName());
  }
}
 

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 

private static Node coreResolve(String hostName) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(hostName);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = null;
  if (rNameList == null || rNameList.get(0) == null) {
    rName = NetworkTopology.DEFAULT_RACK;
    LOG.info("Couldn't resolve " + hostName + ". Falling back to "
        + NetworkTopology.DEFAULT_RACK);
  } else {
    rName = rNameList.get(0);
    LOG.info("Resolved " + hostName + " to " + rName);
  }
  return new NodeBase(hostName, rName);
}
 

private static Node coreResolve(String hostName) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(hostName);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = null;
  if (rNameList == null || rNameList.get(0) == null) {
    rName = NetworkTopology.DEFAULT_RACK;
    LOG.info("Couldn't resolve " + hostName + ". Falling back to "
        + NetworkTopology.DEFAULT_RACK);
  } else {
    rName = rNameList.get(0);
    LOG.info("Resolved " + hostName + " to " + rName);
  }
  return new NodeBase(hostName, rName);
}
 

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 

@Test
public void testStartup() throws Exception {
  conf = new Configuration();
  conf.setClass("dfs.block.replicator.classname",
      BlockPlacementPolicyConfigurable.class, BlockPlacementPolicy.class);
  File baseDir = MiniDFSCluster.getBaseDirectory(conf);
  baseDir.mkdirs();
  File hostsFile = new File(baseDir, "hosts");
  FileOutputStream out = new FileOutputStream(hostsFile);
  out.write("h1\n".getBytes());
  out.write("h2\n".getBytes());
  out.write("h3\n".getBytes());
  out.close();
  conf.set("dfs.hosts", hostsFile.getAbsolutePath());
  StaticMapping.addNodeToRack("h1", "/r1");
  StaticMapping.addNodeToRack("h2", "/r2");
  StaticMapping.addNodeToRack("h3", NetworkTopology.DEFAULT_RACK);
  cluster = new MiniDFSCluster(conf, 3, new String[] { "/r1", "/r2",
      NetworkTopology.DEFAULT_RACK }, new String[] { "h1", "h2", "h3" },
      true, false);
  DFSTestUtil util = new DFSTestUtil("/testStartup", 10, 10, 1024);
  util.createFiles(cluster.getFileSystem(), "/");
  util.checkFiles(cluster.getFileSystem(), "/");
  assertEquals(2,
      cluster.getNameNode().getDatanodeReport(DatanodeReportType.LIVE).length);
  cluster.shutdown();
}
 

private MiniCoronaCluster(Builder builder) throws IOException {
  ContextFactory.resetFactory();
  setNoEmitMetricsContext();
  if (builder.racks != null && builder.hosts != null) {
    if (builder.racks.length != builder.hosts.length) {
      throw new IllegalArgumentException(
          "The number of hosts and racks must be the same");
    }
  }
  this.conf = builder.conf != null ? builder.conf : new JobConf();
  this.namenode = builder.namenode;
  this.ugi = builder.ugi;
  this.conf.set(CoronaConf.CM_ADDRESS, "localhost:0");
  this.conf.set(CoronaConf.CPU_TO_RESOURCE_PARTITIONING, TstUtils.std_cpu_to_resource_partitioning);
  this.clusterManagerPort = startClusterManager(this.conf);
  this.conf.set(CoronaConf.PROXY_JOB_TRACKER_ADDRESS, "localhost:0");
  pjt = ProxyJobTracker.startProxyTracker(new CoronaConf(conf));
  this.proxyJobTrackerPort = pjt.getRpcPort();
  configureJobConf(conf, builder.namenode, clusterManagerPort,
    proxyJobTrackerPort, builder.ugi);
  for (int i = 0; i < builder.numTaskTrackers; ++i) {
    String host = builder.hosts == null ?
        "host" + i + ".foo.com" : builder.hosts[i];
    String rack = builder.racks == null ?
        NetworkTopology.DEFAULT_RACK : builder.racks[i];
    startTaskTracker(host, rack, i, builder.numDir);
  }
  waitTaskTrackers();
}
 

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 

private ContainerRequest createReq(int priority, int memory, String[] hosts) {
  Resource capability = Resource.newInstance(memory, 1);
  Priority priorityOfContainer = Priority.newInstance(priority);
  return new ContainerRequest(capability, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK }, priorityOfContainer);
}
 

public static String resolve(String hostName) {
  return NetworkTopology.DEFAULT_RACK;
}
 

private ContainerRequest createReq(int priority, int memory, String[] hosts) {
  Resource capability = Resource.newInstance(memory, 1);
  Priority priorityOfContainer = Priority.newInstance(priority);
  return new ContainerRequest(capability, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK }, priorityOfContainer);
}
 

public MiniMRCluster(int jobTrackerPort, int taskTrackerPort,
    int numTaskTrackers, String namenode, 
    int numDir, String[] racks, String[] hosts, UnixUserGroupInformation ugi,
    JobConf conf, int numTrackerToExclude) throws IOException {
  if (racks != null && racks.length < numTaskTrackers) {
    LOG.error("Invalid number of racks specified. It should be at least " +
        "equal to the number of tasktrackers");
    shutdown();
  }
  if (hosts != null && numTaskTrackers > hosts.length ) {
    throw new IllegalArgumentException( "The length of hosts [" + hosts.length
        + "] is less than the number of tasktrackers [" + numTaskTrackers + "].");
  }
   
   //Generate rack names if required
   if (racks == null) {
     System.out.println("Generating rack names for tasktrackers");
     racks = new String[numTaskTrackers];
     for (int i=0; i < racks.length; ++i) {
       racks[i] = NetworkTopology.DEFAULT_RACK;
     }
   }
   
  //Generate some hostnames if required
  if (hosts == null) {
    System.out.println("Generating host names for tasktrackers");
    hosts = new String[numTaskTrackers];
    for (int i = 0; i < numTaskTrackers; i++) {
      hosts[i] = "host" + i + ".foo.com";
    }
  }
  this.jobTrackerPort = jobTrackerPort;
  this.taskTrackerPort = taskTrackerPort;
  this.jobTrackerInfoPort = 0;
  this.numTaskTrackers = 0;
  this.namenode = namenode;
  this.ugi = ugi;
  this.conf = conf; // this is the conf the mr starts with
  this.numTrackerToExclude = numTrackerToExclude;

  // start the jobtracker
  startJobTracker();

  // Create the TaskTrackers
  for (int idx = 0; idx < numTaskTrackers; idx++) {
    String rack = null;
    String host = null;
    if (racks != null) {
      rack = racks[idx];
    }
    if (hosts != null) {
      host = hosts[idx];
    }
    
    startTaskTracker(host, rack, idx, numDir);
  }

  this.job = createJobConf(conf);
  waitUntilIdle();
}
 

public MiniMRCluster(int jobTrackerPort, int taskTrackerPort,
    int numTaskTrackers, String namenode, 
    int numDir, String[] racks, String[] hosts, UnixUserGroupInformation ugi,
    JobConf conf) throws IOException {
  if (racks != null && racks.length < numTaskTrackers) {
    LOG.error("Invalid number of racks specified. It should be at least " +
        "equal to the number of tasktrackers");
    shutdown();
  }
  if (hosts != null && numTaskTrackers > hosts.length ) {
    throw new IllegalArgumentException( "The length of hosts [" + hosts.length
        + "] is less than the number of tasktrackers [" + numTaskTrackers + "].");
  }
   
   //Generate rack names if required
   if (racks == null) {
     System.out.println("Generating rack names for tasktrackers");
     racks = new String[numTaskTrackers];
     for (int i=0; i < racks.length; ++i) {
       racks[i] = NetworkTopology.DEFAULT_RACK;
     }
   }
   
  //Generate some hostnames if required
  if (hosts == null) {
    System.out.println("Generating host names for tasktrackers");
    hosts = new String[numTaskTrackers];
    for (int i = 0; i < numTaskTrackers; i++) {
      hosts[i] = "host" + i + ".foo.com";
    }
  }
  this.jobTrackerPort = jobTrackerPort;
  this.taskTrackerPort = taskTrackerPort;
  this.jobTrackerInfoPort = 0;
  this.numTaskTrackers = 0;
  this.namenode = namenode;
  this.ugi = ugi;
  this.conf = conf; // this is the conf the mr starts with

  // start the jobtracker
  startJobTracker();

  // Create the TaskTrackers
  for (int idx = 0; idx < numTaskTrackers; idx++) {
    String rack = null;
    String host = null;
    if (racks != null) {
      rack = racks[idx];
    }
    if (hosts != null) {
      host = hosts[idx];
    }
    
    startTaskTracker(host, rack, idx, numDir);
  }

  this.job = createJobConf(conf);
  waitUntilIdle();
}
 

public static String resolve(String hostName) {
  return NetworkTopology.DEFAULT_RACK;
}