org.apache.hadoop.net.NetworkTopology Java Examples

/**
 * {@inheritDoc}
 */
@Override
protected void chooseRemoteRack(int numOfReplicas,
    DatanodeDescriptor localMachine, Set<Node> excludedNodes,
    long blocksize, int maxReplicasPerRack, List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes)
    throws NotEnoughReplicasException {
  int oldNumOfReplicas = results.size();

  final String rackLocation = NetworkTopology.getFirstHalf(
      localMachine.getNetworkLocation());
  try {
    // randomly choose from remote racks
    chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e) {
    // fall back to the local rack
    chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas),
        rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 

/** A formatted string for printing the status of the DataNode. */
public String dumpDatanode() {
  StringBuffer buffer = new StringBuffer();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  buffer.append(name);
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append(" "+location);
  }
  if (isDecommissioned()) {
    buffer.append(" DD");
  } else if (isDecommissionInProgress()) {
    buffer.append(" DP");
  } else {
    buffer.append(" IN");
  }
  buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
  buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
  buffer.append(" " + StringUtils.limitDecimalTo2(((1.0*u)/c)*100)+"%");
  buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
  buffer.append(" " + new Date(lastUpdate));
  return buffer.toString();
}
 

@Override
public void initialize(Configuration conf,  FSClusterStats stats,
                       NetworkTopology clusterMap, 
                       Host2NodesMap host2datanodeMap) {
  this.considerLoad = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, true);
  this.stats = stats;
  this.clusterMap = clusterMap;
  this.host2datanodeMap = host2datanodeMap;
  this.heartbeatInterval = conf.getLong(
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000;
  this.tolerateHeartbeatMultiplier = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY,
      DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT);
  this.staleInterval = conf.getLong(
      DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 
      DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT);
}
 

/**
 * {@inheritDoc}
 */
@Override
protected void chooseRemoteRack(int numOfReplicas,
    DatanodeDescriptor localMachine, Set<Node> excludedNodes,
    long blocksize, int maxReplicasPerRack, List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes)
    throws NotEnoughReplicasException {
  int oldNumOfReplicas = results.size();

  final String rackLocation = NetworkTopology.getFirstHalf(
      localMachine.getNetworkLocation());
  try {
    // randomly choose from remote racks
    chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e) {
    // fall back to the local rack
    chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas),
        rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 

private void validateLabelsRequests(ResourceRequest resourceRequest,
    boolean isReduce) {
  switch (resourceRequest.getResourceName()) {
  case "map":
  case "reduce":
  case NetworkTopology.DEFAULT_RACK:
    Assert.assertNull(resourceRequest.getNodeLabelExpression());
    break;
  case "*":
    Assert.assertEquals(isReduce ? "ReduceNodes" : "MapNodes",
        resourceRequest.getNodeLabelExpression());
    break;
  default:
    Assert.fail("Invalid resource location "
        + resourceRequest.getResourceName());
  }
}
 

private ContainerRequestEvent
    createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
        boolean earlierFailedAttempt, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  Resource containerNeed = Resource.newInstance(memory, 1);
  if (earlierFailedAttempt) {
    return ContainerRequestEvent
        .createContainerRequestEventForFailedContainer(attemptId,
            containerNeed);
  }
  return new ContainerRequestEvent(attemptId, containerNeed, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK });
}
 

OneBlockInfo(Path path, long offset, long len,
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i],
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 

/**
 * Return the Node in the network topology that corresponds to the hostname
 */
public Node getNode(String name) {
  Node n = hostnameToNodeMap.get(name);

  // it's ok if multiple threads try to resolve the same host at the same time
  // the assumption is that resolve() will return a canonical node object and
  // the put operation is therefore idempotent
  if (n == null) {
    n = resolveAndGetNode(name);
    hostnameToNodeMap.put(name, n);
    // Make an entry for the node at the max level in the cache
    nodesAtMaxLevel.add(
      getParentNode(n, NetworkTopology.DEFAULT_HOST_LEVEL - 1));
  }

  return n;
}
 

/**
 * Return the distances among an array of nodes
 * @param nodes
 *        the nodes array
 * @return
 *        an distance matrix, distance of nodes[i][j] is put in
 *        element indexed by [i][j]
 */
static public int[][] getRealDistances(DatanodeInfo[] nodes) {
	// TODO Auto-generated method stub
	int[][] distances = new int[nodes.length][nodes.length];
	NetworkTopology netTopology = new NetworkTopology();
	for(int i = 0; i < distances.length; i++)
		netTopology.add(nodes[i]);
	for (int i = 0; i < distances.length; i++) {
		for (int j = 0; j < i; j++) {
			distances[i][j] = netTopology.getDistance(nodes[i], nodes[j]);
			distances[j][i] = distances[i][j];
		}
		distances[i][i] = 0;
	}

	return distances;
}
 

boolean chooseTarget(DBlock db, Source source,
    List<StorageType> targetTypes, Matcher matcher) {
  final NetworkTopology cluster = dispatcher.getCluster(); 
  for (StorageType t : targetTypes) {
    for(StorageGroup target : storages.getTargetStorages(t)) {
      if (matcher.match(cluster, source.getDatanodeInfo(),
          target.getDatanodeInfo())) {
        final PendingMove pm = source.addPendingMove(db, target);
        if (pm != null) {
          dispatcher.executePendingMove(pm);
          return true;
        }
      }
    }
  }
  return false;
}
 

OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 

/**
 * Scan the targets list: all targets should be on different NodeGroups.
 * Return false if two targets are found on the same NodeGroup.
 */
private static boolean checkTargetsOnDifferentNodeGroup(
    DatanodeStorageInfo[] targets) {
  if(targets.length == 0)
    return true;
  Set<String> targetSet = new HashSet<String>();
  for(DatanodeStorageInfo storage:targets) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    String nodeGroup = NetworkTopology.getLastHalf(node.getNetworkLocation());
    if(targetSet.contains(nodeGroup)) {
      return false;
    } else {
      targetSet.add(nodeGroup);
    }
  }
  return true;
}
 

OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 

OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 

private VerifiablePolicy initTest() throws Exception {
  VerifiablePolicy policy = new VerifiablePolicy();
  Configuration conf = new Configuration();
  TestClusterStats stats = new TestClusterStats();
  NetworkTopology clusterMap = new NetworkTopology();
  TestHostsReader hostsReader = new TestHostsReader();
  TestMapping dnsToSwitchMapping = new TestMapping();

  for (DatanodeDescriptor d: dataNodes) {
    clusterMap.add(d);
  }

  conf.setInt("dfs.replication.rackwindow", 2);
  conf.setInt("dfs.replication.machineWindow", 2);

  policy.initialize(conf, stats, clusterMap, hostsReader, dnsToSwitchMapping, null);
  return policy;
}
 

/** A formatted string for printing the status of the DataNode. */
public String dumpDatanode() {
  StringBuffer buffer = new StringBuffer();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  buffer.append(name);
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append(" "+location);
  }
  if (isDecommissioned()) {
    buffer.append(" DD");
  } else if (isDecommissionInProgress()) {
    buffer.append(" DP");
  } else {
    buffer.append(" IN");
  }
  buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
  buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
  buffer.append(" " + StringUtils.limitDecimalTo2(((1.0*u)/c)*100)+"%");
  buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
  buffer.append(" " + new Date(lastUpdate));
  return buffer.toString();
}
 

public void testChooseTargetWithDefaultRack() throws Exception {
  VerifiablePolicy policy = initTest();
  TestMapping mapping = (TestMapping) policy.dnsToSwitchMapping;
  mapping.assignDefaultRack = true;
  try {
    policy.hostsUpdated();
    fail("Did not throw : " + DefaultRackException.class);
  } catch (DefaultRackException e) {}

  // Verify there is no default rack.
  RackRingInfo info = policy.racksMap.get(NetworkTopology.DEFAULT_RACK);
  assertNull(info);

  HashMap<Node, Node> emptyMap = new HashMap<Node, Node>();
  List<DatanodeDescriptor> results = new ArrayList<DatanodeDescriptor>();
  for (int i = 0; i < dataNodes.length; i++) {
    policy.chooseTarget(3, dataNodes[i], emptyMap, 512, 4, results, true);
  }
}
 

OneBlockInfo(Path path, long offset, long len,
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i],
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 

/**
 * Scan the targets list: all targets should be on different NodeGroups.
 * Return false if two targets are found on the same NodeGroup.
 */
private static boolean checkTargetsOnDifferentNodeGroup(
    DatanodeStorageInfo[] targets) {
  if(targets.length == 0)
    return true;
  Set<String> targetSet = new HashSet<String>();
  for(DatanodeStorageInfo storage:targets) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    String nodeGroup = NetworkTopology.getLastHalf(node.getNetworkLocation());
    if(targetSet.contains(nodeGroup)) {
      return false;
    } else {
      targetSet.add(nodeGroup);
    }
  }
  return true;
}
 

@Test
public void testCaching() {
  Configuration conf = new Configuration();
  conf.setClass(
    CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
    MyResolver.class, DNSToSwitchMapping.class);
  RackResolver.init(conf);
  try {
    InetAddress iaddr = InetAddress.getByName("host1");
    MyResolver.resolvedHost1 = iaddr.getHostAddress();
  } catch (UnknownHostException e) {
    // Ignore if not found
  }
  Node node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve(invalidHost);
  Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation());
}
 

boolean chooseTarget(DBlock db, Source source,
    List<StorageType> targetTypes, Matcher matcher) {
  final NetworkTopology cluster = dispatcher.getCluster(); 
  for (StorageType t : targetTypes) {
    for(StorageGroup target : storages.getTargetStorages(t)) {
      if (matcher.match(cluster, source.getDatanodeInfo(),
          target.getDatanodeInfo())) {
        final PendingMove pm = source.addPendingMove(db, target);
        if (pm != null) {
          dispatcher.executePendingMove(pm);
          return true;
        }
      }
    }
  }
  return false;
}
 

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 

/**
 * Get an instance of the configured Block Placement Policy based on the
 * the configuration property
 * {@link  DFSConfigKeys#DFS_BLOCK_REPLICATOR_CLASSNAME_KEY}.
 * 
 * @param conf the configuration to be used
 * @param stats an object that is used to retrieve the load on the cluster
 * @param clusterMap the network topology of the cluster
 * @return an instance of BlockPlacementPolicy
 */
public static BlockPlacementPolicy getInstance(Configuration conf, 
                                               FSClusterStats stats,
                                               NetworkTopology clusterMap,
                                               Host2NodesMap host2datanodeMap) {
  final Class<? extends BlockPlacementPolicy> replicatorClass = conf.getClass(
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
      BlockPlacementPolicy.class);
  final BlockPlacementPolicy replicator = ReflectionUtils.newInstance(
      replicatorClass, conf);
  replicator.initialize(conf, stats, clusterMap, host2datanodeMap);
  return replicator;
}
 

static void setNameNodeLoggingLevel(Level logLevel) {
  LOG.fatal("Log level = " + logLevel.toString());
  // change log level to NameNode logs
  DFSTestUtil.setNameNodeLogLevel(logLevel);
  GenericTestUtils.setLogLevel(LogManager.getLogger(
          NetworkTopology.class.getName()), logLevel);
  GenericTestUtils.setLogLevel(LogManager.getLogger(
          Groups.class.getName()), logLevel);
}
 

private Set<ExtendedBlock> getBlocksOnRack(List<LocatedBlock> blks, String rack) {
  Set<ExtendedBlock> ret = new HashSet<ExtendedBlock>();
  for (LocatedBlock blk : blks) {
    for (DatanodeInfo di : blk.getLocations()) {
      if (rack.equals(NetworkTopology.getFirstHalf(di.getNetworkLocation()))) {
        ret.add(blk.getBlock());
        break;
      }
    }
  }
  return ret;
}
 

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 

public BlockPlacementPolicyFakeData(Configuration conf,  FSClusterStats stats,
                         NetworkTopology clusterMap) {
  initialize(conf, stats, clusterMap, null, null, null);
  
  FSNamesystem.LOG.info("new BlockPlacementPolicyFakeData: " + this);
  lastInstance = this;
}
 

/** A formatted string for printing the status of the DataNode. */
public String dumpDatanode() {
  StringBuilder buffer = new StringBuilder();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  long cc = getCacheCapacity();
  long cr = getCacheRemaining();
  long cu = getCacheUsed();
  buffer.append(getName());
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append(" "+location);
  }
  if (isDecommissioned()) {
    buffer.append(" DD");
  } else if (isDecommissionInProgress()) {
    buffer.append(" DP");
  } else {
    buffer.append(" IN");
  }
  buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
  buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
  buffer.append(" " + percent2String(u/(double)c));
  buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
  buffer.append(" " + cc + "(" + StringUtils.byteDesc(cc)+")");
  buffer.append(" " + cu + "(" + StringUtils.byteDesc(cu)+")");
  buffer.append(" " + percent2String(cu/(double)cc));
  buffer.append(" " + cr + "(" + StringUtils.byteDesc(cr)+")");
  buffer.append(" " + new Date(lastUpdate));
  return buffer.toString();
}
 

private Set<ExtendedBlock> getBlocksOnRack(List<LocatedBlock> blks, String rack) {
  Set<ExtendedBlock> ret = new HashSet<ExtendedBlock>();
  for (LocatedBlock blk : blks) {
    for (DatanodeInfo di : blk.getLocations()) {
      if (rack.equals(NetworkTopology.getFirstHalf(di.getNetworkLocation()))) {
        ret.add(blk.getBlock());
        break;
      }
    }
  }
  return ret;
}
 

/** A formatted string for reporting the status of the DataNode. */
public String getDatanodeReport() {
  StringBuffer buffer = new StringBuffer();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  long nonDFSUsed = getNonDfsUsed();
  float usedPercent = getDfsUsedPercent();
  float remainingPercent = getRemainingPercent();

  buffer.append("Name: "+name+"\n");
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append("Rack: "+location+"\n");
  }
  buffer.append("Decommission Status : ");
  if (isDecommissioned()) {
    buffer.append("Decommissioned\n");
  } else if (isDecommissionInProgress()) {
    buffer.append("Decommission in progress\n");
  } else {
    buffer.append("Normal\n");
  }
  buffer.append("Configured Capacity: "+c+" ("+StringUtils.byteDesc(c)+")"+"\n");
  buffer.append("DFS Used: "+u+" ("+StringUtils.byteDesc(u)+")"+"\n");
  buffer.append("Non DFS Used: "+nonDFSUsed+" ("+StringUtils.byteDesc(nonDFSUsed)+")"+"\n");
  buffer.append("DFS Remaining: " +r+ "("+StringUtils.byteDesc(r)+")"+"\n");
  buffer.append("DFS Used%: "+StringUtils.limitDecimalTo2(usedPercent)+"%\n");
  buffer.append("DFS Remaining%: "+StringUtils.limitDecimalTo2(remainingPercent)+"%\n");
  buffer.append("Last contact: "+new Date(lastUpdate)+"\n");
  return buffer.toString();
}