Java Code Examples for org.apache.hadoop.hdfs.DFSUtil#setGenericConf()

The following examples show how to use org.apache.hadoop.hdfs.DFSUtil#setGenericConf() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NameNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * In federation configuration is set for a set of
 * namenode and secondary namenode/backup/checkpointer, which are
 * grouped under a logical nameservice ID. The configuration keys specific 
 * to them have suffix set to configured nameserviceId.
 * 
 * This method copies the value from specific key of format key.nameserviceId
 * to key, to set up the generic configuration. Once this is done, only
 * generic version of the configuration is read in rest of the code, for
 * backward compatibility and simpler code changes.
 * 
 * @param conf
 *          Configuration object to lookup specific key and to set the value
 *          to the key passed. Note the conf object is modified
 * @param nameserviceId name service Id (to distinguish federated NNs)
 * @param namenodeId the namenode ID (to distinguish HA NNs)
 * @see DFSUtil#setGenericConf(Configuration, String, String, String...)
 */
public static void initializeGenericKeys(Configuration conf,
    String nameserviceId, String namenodeId) {
  if ((nameserviceId != null && !nameserviceId.isEmpty()) || 
      (namenodeId != null && !namenodeId.isEmpty())) {
    if (nameserviceId != null) {
      conf.set(DFS_NAMESERVICE_ID, nameserviceId);
    }
    if (namenodeId != null) {
      conf.set(DFS_HA_NAMENODE_ID_KEY, namenodeId);
    }
    
    DFSUtil.setGenericConf(conf, nameserviceId, namenodeId,
        NAMENODE_SPECIFIC_KEYS);
    DFSUtil.setGenericConf(conf, nameserviceId, null,
        NAMESERVICE_SPECIFIC_KEYS);
  }
  
  // If the RPC address is set use it to (re-)configure the default FS
  if (conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY) != null) {
    URI defaultUri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://"
        + conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY));
    conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
    LOG.debug("Setting " + FS_DEFAULT_NAME_KEY + " to " + defaultUri.toString());
  }
}
Example 2
Source File: DFSZKFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static DFSZKFailoverController create(Configuration conf) {
  Configuration localNNConf = DFSHAAdmin.addSecurityConfiguration(conf);
  String nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(localNNConf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  String nnId = HAUtil.getNameNodeId(localNNConf, nsId);
  if (nnId == null) {
    String msg = "Could not get the namenode ID of this node. " +
        "You may run zkfc on the node other than namenode.";
    throw new HadoopIllegalArgumentException(msg);
  }
  NameNode.initializeGenericKeys(localNNConf, nsId, nnId);
  DFSUtil.setGenericConf(localNNConf, nsId, nnId, ZKFC_CONF_KEYS);
  
  NNHAServiceTarget localTarget = new NNHAServiceTarget(
      localNNConf, nsId, nnId);
  return new DFSZKFailoverController(localNNConf, localTarget);
}
Example 3
Source File: NameNode.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * In federation configuration is set for a set of
 * namenode and secondary namenode/backup/checkpointer, which are
 * grouped under a logical nameservice ID. The configuration keys specific 
 * to them have suffix set to configured nameserviceId.
 * 
 * This method copies the value from specific key of format key.nameserviceId
 * to key, to set up the generic configuration. Once this is done, only
 * generic version of the configuration is read in rest of the code, for
 * backward compatibility and simpler code changes.
 * 
 * @param conf
 *          Configuration object to lookup specific key and to set the value
 *          to the key passed. Note the conf object is modified
 * @param nameserviceId name service Id (to distinguish federated NNs)
 * @param namenodeId the namenode ID (to distinguish HA NNs)
 * @see DFSUtil#setGenericConf(Configuration, String, String, String...)
 */
public static void initializeGenericKeys(Configuration conf,
    String nameserviceId, String namenodeId) {
  if ((nameserviceId != null && !nameserviceId.isEmpty()) || 
      (namenodeId != null && !namenodeId.isEmpty())) {
    if (nameserviceId != null) {
      conf.set(DFS_NAMESERVICE_ID, nameserviceId);
    }
    if (namenodeId != null) {
      conf.set(DFS_HA_NAMENODE_ID_KEY, namenodeId);
    }
    
    DFSUtil.setGenericConf(conf, nameserviceId, namenodeId,
        NAMENODE_SPECIFIC_KEYS);
    DFSUtil.setGenericConf(conf, nameserviceId, null,
        NAMESERVICE_SPECIFIC_KEYS);
  }
  
  // If the RPC address is set use it to (re-)configure the default FS
  if (conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY) != null) {
    URI defaultUri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://"
        + conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY));
    conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
    LOG.debug("Setting " + FS_DEFAULT_NAME_KEY + " to " + defaultUri.toString());
  }
}
Example 4
Source File: DFSZKFailoverController.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static DFSZKFailoverController create(Configuration conf) {
  Configuration localNNConf = DFSHAAdmin.addSecurityConfiguration(conf);
  String nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(localNNConf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  String nnId = HAUtil.getNameNodeId(localNNConf, nsId);
  if (nnId == null) {
    String msg = "Could not get the namenode ID of this node. " +
        "You may run zkfc on the node other than namenode.";
    throw new HadoopIllegalArgumentException(msg);
  }
  NameNode.initializeGenericKeys(localNNConf, nsId, nnId);
  DFSUtil.setGenericConf(localNNConf, nsId, nnId, ZKFC_CONF_KEYS);
  
  NNHAServiceTarget localTarget = new NNHAServiceTarget(
      localNNConf, nsId, nnId);
  return new DFSZKFailoverController(localNNConf, localTarget);
}
Example 5
Source File: SecondaryNameNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * main() has some simple utility methods.
 * @param argv Command line parameters.
 * @exception Exception if the filesystem does not exist.
 */
public static void main(String[] argv) throws Exception {
  StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG);
  Configuration tconf = new Configuration();
  try {
    argv = DFSUtil.setGenericConf(argv, tconf);
  } catch (IllegalArgumentException e) {
    System.err.println(e.getMessage());
    printUsage("");
    return;
  }
  if (argv.length >= 1) {
    SecondaryNameNode secondary = new SecondaryNameNode(tconf);
    int ret = secondary.processArgs(argv);
    System.exit(ret);
  }

  // Create a never ending deamon
  Daemon checkpointThread = new Daemon(new SecondaryNameNode(tconf)); 
  checkpointThread.start();
}
Example 6
Source File: DFSck.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
/**
 * @param args
 */
public int run(String[] args) throws Exception {
  try { 
    args = DFSUtil.setGenericConf(args, getConf()); 
  } catch (IllegalArgumentException e) {  
    System.err.println(e.getMessage()); 
    printUsage(); 
    return -1; 
  }
  String fsName = getInfoServer();
  if (args.length == 0) {
    printUsage();
    return -1;
  }
  StringBuffer url = new StringBuffer("http://"+fsName+"/fsck?path=");
  String dir = "/";
  int limit = 500; // limit output.
  // find top-level dir first
  for (int idx = 0; idx < args.length; ) {
    if (args[idx].equals("-limit")) {
      idx++; // Skip over limit value
    } else if (!args[idx].startsWith("-")) {
      dir = args[idx];
      break;
    }
    idx++;
  }
  url.append(URLEncoder.encode(dir, "UTF-8"));
  boolean doListCorruptFileBlocks = false;
  for (int idx = 0; idx < args.length; ) {
    if (args[idx].equals("-move")) { url.append("&move=1"); }
    else if (args[idx].equals("-delete")) { url.append("&delete=1"); }
    else if (args[idx].equals("-files")) { url.append("&files=1"); }
    else if (args[idx].equals("-openforwrite")) { url.append("&openforwrite=1"); }
    else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); }
    else if (args[idx].equals("-locations")) { url.append("&locations=1"); }
    else if (args[idx].equals("-racks")) { url.append("&racks=1"); }
    else if (args[idx].equals("-list-corruptfileblocks")) {
      url.append("&listcorruptfileblocks=1");
      doListCorruptFileBlocks = true;
    } else if (args[idx].equals("-limit")) {
      idx++;
      limit = Integer.parseInt(args[idx]);
    } else if (args[idx].equals("-list-decommissioningblocks")) {
      url.append("&decommissioning=1");
    }
    idx++;
  }
  if (doListCorruptFileBlocks) {
    return listCorruptFileBlocks(dir, limit, url.toString());
  }
  URL path = new URL(url.toString());
  System.err.println("Connecting to : " + path);
  URLConnection connection = path.openConnection();
  InputStream stream = connection.getInputStream();
  BufferedReader input = new BufferedReader(new InputStreamReader(
                                            stream, "UTF-8"));
  String line = null;
  String lastLine = null;
  int errCode = -1;
  try {
    while ((line = input.readLine()) != null) {
      out.println(line);
      lastLine = line;
    }
  } finally {
    input.close();
  }
  if (lastLine.endsWith(NamenodeFsck.HEALTHY_STATUS)) {
    errCode = 0;
  } else if (lastLine.endsWith(NamenodeFsck.CORRUPT_STATUS)) {
    errCode = 1;
  } else if (lastLine.endsWith(NamenodeFsck.NONEXISTENT_STATUS)) {
    errCode = 0;
  }
  return errCode;
}
Example 7
Source File: AvatarNode.java    From RDFS with Apache License 2.0 3 votes vote down vote up 
/**  
 * In federation configuration is set for a set of
 * avartanodes, namenodes etc, which are
 * grouped under a logical nameservice ID. The configuration keys specific 
 * to them have suffix set to configured nameserviceId.
 * 
 * This method copies the value from specific key of format key.nameserviceId
 * to key, to set up the generic configuration. Once this is done, only
 * generic version of the configuration is read in rest of the code, for
 * backward compatibility and simpler code changes.
 * 
 * @param conf
 *          Configuration object to lookup specific key and to set the value
 *          to the key passed. Note the conf object is modified
 * @see DFSUtil#setGenericConf(Configuration, String, String...)
 */
public static void initializeGenericKeys(Configuration conf, String serviceKey) {                                
  if ((serviceKey == null) || serviceKey.isEmpty()) {
    return;
  }    
  NameNode.initializeGenericKeys(conf, serviceKey);
  
  // adjust meta directory names for this service
  adjustMetaDirectoryNames(conf, serviceKey);
  
  DFSUtil.setGenericConf(conf, serviceKey, AVATARSERVICE_SPECIFIC_KEYS);    
}
Example 8
Source File: NameNode.java    From RDFS with Apache License 2.0 3 votes vote down vote up 
/**  
 * In federation configuration is set for a set of
 * namenode and secondary namenode/backup/checkpointer, which are
 * grouped under a logical nameservice ID. The configuration keys specific 
 * to them have suffix set to configured nameserviceId.
 * 
 * This method copies the value from specific key of format key.nameserviceId
 * to key, to set up the generic configuration. Once this is done, only
 * generic version of the configuration is read in rest of the code, for
 * backward compatibility and simpler code changes.
 * 
 * @param conf
 *          Configuration object to lookup specific key and to set the value
 *          to the key passed. Note the conf object is modified
 * @see DFSUtil#setGenericConf(Configuration, String, String...)
 */
public static void initializeGenericKeys(Configuration conf, String serviceKey) {                                
  if ((serviceKey == null) || serviceKey.isEmpty()) {
    return;
  }    
  
  // adjust meta directory names by service key
  adjustMetaDirectoryNames(conf, serviceKey);
  
  DFSUtil.setGenericConf(conf, serviceKey, NAMESERVICE_SPECIFIC_KEYS);
}