Java Code Examples for org.apache.hadoop.util.StringUtils#split()

@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    //value:   马老师 一名老师 刚老师 周老师

    String[] strs = StringUtils.split(value.toString(), ' ');


    for (int i = 1; i < strs.length; i++) {
              mkey.set(getFof(strs[0],strs[i]));
              mval.set(0);
              context.write(mkey,mval);
        for (int j = i+1; j < strs.length; j++) {
            mkey.set(getFof(strs[i],strs[j]));
            mval.set(1);
            context.write(mkey,mval);

        }

    }
}
 

public static void setQueriesFromDirRecursive(Configuration conf, String dirs, boolean sparqlUpdate, int stage) throws IOException {
    for (String dir : StringUtils.split(dirs)) {
        Path p = new Path(StringUtils.unEscapeString(dir));
        FileStatus[] matches = p.getFileSystem(conf).globStatus(p);
        if (matches == null) {
            throw new IOException("Input path does not exist: " + p);
        } else if (matches.length == 0) {
            throw new IOException("Input Pattern " + p + " matches 0 files");
        } else {
            for (FileStatus globStat : matches) {
                if (globStat.isDirectory()) {
                    addQueryRecursively(conf, p, sparqlUpdate, stage);
                } else {
                    addQuery(conf, globStat, sparqlUpdate, stage);
                }
            }
        }
    }
}
 

private void rename(BackupFile backup) {
    if (this.rename == null) {
        return;
    }
    String[] temp = StringUtils.split(this.rename, '=');
    if (temp.length != 2) {
        println("invalid rename pattern: " + this.rename);
        System.exit(-1);
    }
    String newname = temp[0];
    String oldname = temp[1];
    for (TableBackupInfo i:backup.tables) {
        if (i.catalog.equals(oldname)) {
            i.catalog = newname;
        }
    }
}
 

@Override
protected void map(LongWritable key, Text value,
		Mapper<LongWritable, Text, FlowBean, NullWritable>.Context context)
		throws IOException, InterruptedException {

	//super.map(key, value, context);
	line = value.toString();
	String[] fields = StringUtils.split(line, SEPARATOR);
	if (fields.length != 4) {
		LOGGER.error("invalid line: {}", line);
		System.err.println("invalid line: " + line);
	} else {
		phoneNum = fields[0];
		upFlow = Long.parseLong(fields[1]);
		downFlow = Long.parseLong(fields[2]);
		sumFlow = Long.parseLong(fields[3]);
		flowBean.setPhoneNum(phoneNum);
		flowBean.setUpFlow(upFlow);
		flowBean.setDownFlow(downFlow);
		flowBean.setSumFlow(sumFlow);
		context.write(flowBean, NullWritable.get());
	}
}
 

/**
 * Parse a single line of history. 
 * @param line
 * @param l
 * @throws IOException
 */
private static void parseLine(String line, Listener l, boolean isEscaped) 
throws IOException{
  // extract the record type 
  int idx = line.indexOf(' '); 
  String recType = line.substring(0, idx);
  String data = line.substring(idx+1, line.length());
  
  Matcher matcher = pattern.matcher(data); 
  Map<Keys,String> parseBuffer = new HashMap<Keys, String>();

  while(matcher.find()){
    String tuple = matcher.group(0);
    String []parts = StringUtils.split(tuple, StringUtils.ESCAPE_CHAR, '=');
    String value = parts[1].substring(1, parts[1].length() -1);
    if (isEscaped) {
      value = StringUtils.unEscapeString(value, StringUtils.ESCAPE_CHAR,
                                         charsToEscape);
    }
    parseBuffer.put(Keys.valueOf(parts[0]), value);
  }

  l.handle(RecordTypes.valueOf(recType), parseBuffer); 
  
  parseBuffer.clear(); 
}
 

private void anonymize(StatePool statePool, Configuration conf) {
  FileNameState fState = (FileNameState) statePool.getState(getClass());
  if (fState == null) {
    fState = new FileNameState();
    statePool.addState(getClass(), fState);
  }
  
  String[] files = StringUtils.split(fileName);
  String[] anonymizedFileNames = new String[files.length];
  int i = 0;
  for (String f : files) {
    anonymizedFileNames[i++] = 
      anonymize(statePool, conf, fState, f);
  }

  anonymizedFileName = StringUtils.arrayToString(anonymizedFileNames);
}
 

/**
 * Parse a single line of history. 
 * @param line
 * @param l
 * @throws IOException
 */
private static void parseLine(String line, Listener l, boolean isEscaped) 
throws IOException{
  // extract the record type 
  int idx = line.indexOf(' '); 
  String recType = line.substring(0, idx);
  String data = line.substring(idx+1, line.length());
  
  Matcher matcher = pattern.matcher(data); 

  while(matcher.find()){
    String tuple = matcher.group(0);
    String []parts = StringUtils.split(tuple, StringUtils.ESCAPE_CHAR, '=');
    String value = parts[1].substring(1, parts[1].length() -1);
    if (isEscaped) {
      value = StringUtils.unEscapeString(value, StringUtils.ESCAPE_CHAR,
                                         charsToEscape);
    }
    parseBuffer.put(Keys.valueOf(parts[0]), value);
  }

  l.handle(RecordTypes.valueOf(recType), parseBuffer); 
  
  parseBuffer.clear(); 
}
 

public static List<Integer> getReadColumnIDs(String confString, Configuration conf) {
    String skips = conf.get(confString, "");
    String[] list = StringUtils.split(skips);
    ArrayList result = new ArrayList(list.length);
    String[] arr$ = list;
    int len$ = list.length;

    for(int i$ = 0; i$ < len$; ++i$) {
        String element = arr$[i$];
        Integer toAdd = Integer.valueOf(Integer.parseInt(element));
        result.add(toAdd);
    }

    return result;
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  String auth =  conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
  if (auth == null || "simple".equals(auth)) {
    isSecurityEnabled = false;
  } else if ("kerberos".equals(auth)) {
    isSecurityEnabled = true;
  } else {
    LOG.warn("Unrecongized attribute value for " +
        CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION +
        " of " + auth);
  }
  String proxy = WebAppUtils.getProxyHostAndPort(conf);
  String[] proxyParts = proxy.split(":");
  proxyHost = proxyParts[0];

  fetcher = new AppReportFetcher(conf);
  bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
  if(bindAddress == null || bindAddress.isEmpty()) {
    throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + 
        " is not set so the proxy will not run.");
  }
  LOG.info("Instantiating Proxy at " + bindAddress);
  String[] parts = StringUtils.split(bindAddress, ':');
  port = 0;
  if (parts.length == 2) {
    bindAddress = parts[0];
    port = Integer.parseInt(parts[1]);
  }
  acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL, 
      YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
  super.serviceInit(conf);
}
 

/**
 * Get the list of input {@link Path}s for the map-reduce job.
 * 
 * @param conf The configuration of the job 
 * @return the list of input {@link Path}s for the map-reduce job.
 */
public static Path[] getInputPaths(JobConf conf) {
  String dirs = conf.get("mapred.input.dir", "");
  String [] list = StringUtils.split(dirs);
  Path[] result = new Path[list.length];
  for (int i = 0; i < list.length; i++) {
    result[i] = new Path(StringUtils.unEscapeString(list[i]));
  }
  return result;
}
 

/**
 * Whether the pathname is valid.  Currently prohibits relative paths, 
 * names which contain a ":" or "//", or other non-canonical paths.
 */
public static boolean isValidName(String src) {
  // Path must be absolute.
  if (!src.startsWith(Path.SEPARATOR)) {
    return false;
  }
    
  // Check for ".." "." ":" "/"
  String[] components = StringUtils.split(src, '/');
  for (int i = 0; i < components.length; i++) {
    String element = components[i];
    if (element.equals(".")  ||
        (element.indexOf(":") >= 0)  ||
        (element.indexOf("/") >= 0)) {
      return false;
    }
    // ".." is allowed in path starting with /.reserved/.inodes
    if (element.equals("..")) {
      if (components.length > 4
          && components[1].equals(FSDirectory.DOT_RESERVED_STRING)
          && components[2].equals(FSDirectory.DOT_INODES_STRING)) {
        continue;
      }
      return false;
    }
    // The string may start or end with a /, but not have
    // "//" in the middle.
    if (element.isEmpty() && i != components.length - 1 &&
        i != 0) {
      return false;
    }
  }
  return true;
}
 

/**
 * find the table either by name or id
 * @param name
 * @return
 * @throws Exception 
 */
protected GTable findTable(String name) throws Exception {
    String ns = null;
    String[] words = StringUtils.split(name, '.');
    Humpback humpback = getHumpbackReadOnly();
    if (words.length == 2) {
        ns = words[0].toLowerCase();
        name = words[1].toLowerCase();
        for (SysMetaRow i:humpback.getTablesMeta()) {
            if (i.isDeleted()) {
                continue;
            }
            if (ns != null) {
                if (!ns.equals(i.getNamespace().toLowerCase())) {
                    continue;
                }
                if (!name.equals(i.getTableName().toLowerCase())) {
                    continue;
                }
            }
            return humpback.getTable(i.getTableId());
        }
    }
    else {
        int tableId = parseInteger(name);
        return humpback.getTable(tableId);
    }
    return null;
}
 

/**
 * Get the list of input {@link Path}s for the map-reduce job.
 * 
 * @param conf The configuration of the job 
 * @return the list of input {@link Path}s for the map-reduce job.
 */
public static Path[] getInputPaths(JobConf conf) {
  String dirs = conf.get(org.apache.hadoop.mapreduce.lib.input.
    FileInputFormat.INPUT_DIR, "");
  String [] list = StringUtils.split(dirs);
  Path[] result = new Path[list.length];
  for (int i = 0; i < list.length; i++) {
    result[i] = new Path(StringUtils.unEscapeString(list[i]));
  }
  return result;
}
 

/**
 * Get the list of input {@link Path}s for the map-reduce job.
 * 
 * @param conf The configuration of the job 
 * @return the list of input {@link Path}s for the map-reduce job.
 */
public static Path[] getInputPaths(JobConf conf) {
  String dirs = conf.get(org.apache.hadoop.mapreduce.lib.input.
    FileInputFormat.INPUT_DIR, "");
  String [] list = StringUtils.split(dirs);
  Path[] result = new Path[list.length];
  for (int i = 0; i < list.length; i++) {
    result[i] = new Path(StringUtils.unEscapeString(list[i]));
  }
  return result;
}
 

/**
 * Get the list of input {@link Path}s for the map-reduce job.
 * 
 * @param context The job
 * @return the list of input {@link Path}s for the map-reduce job.
 */
public static Path[] getInputPaths(JobContext context) {
  String dirs = context.getConfiguration().get(INPUT_DIR, "");
  String [] list = StringUtils.split(dirs);
  Path[] result = new Path[list.length];
  for (int i = 0; i < list.length; i++) {
    result[i] = new Path(StringUtils.unEscapeString(list[i]));
  }
  return result;
}
 

/**
 * Get the list of input {@link Path}s for the map-reduce job.
 * 
 * @param context The job
 * @return the list of input {@link Path}s for the map-reduce job.
 */
public static Path[] getInputPaths(JobContext context) {
  String dirs = context.getConfiguration().get("mapred.input.dir", "");
  String [] list = StringUtils.split(dirs);
  Path[] result = new Path[list.length];
  for (int i = 0; i < list.length; i++) {
    result[i] = new Path(StringUtils.unEscapeString(list[i]));
  }
  return result;
}
 

@Override
public boolean isValidPath(String path) {
  String[] components = StringUtils.split(path, Path.SEPARATOR_CHAR);
  return isValidPath(path, components);
}
 

public void setTickers(String tickers)
{
  this.tickers = tickers;
  symbols = StringUtils.split(tickers, ',');
}
 

private String[] getPathNames(String path) {
  if (path == null || !path.startsWith(Path.SEPARATOR)) {
    return null;
  }
  return StringUtils.split(path, Path.SEPARATOR_CHAR);
}
 

Map<String,char[]> readPasswordDB(Configuration conf) throws IOException {
  String passwordFileName = conf.get(PASSWORD_FILE_KEY);
  if (passwordFileName == null) {
    throw new RuntimeException(PASSWORD_FILE_KEY
        + " is not defined in configuration, cannot use this implementation");
  }

  Path passwordFile = new Path(passwordFileName);
  FileSystem fs = passwordFile.getFileSystem(conf);
  if (!fs.exists(passwordFile)) {
    throw new RuntimeException("Configured password file does not exist: " + passwordFile);
  }

  Map<String,char[]> passwordDb = new HashMap<>();
  try (FSDataInputStream fdis = fs.open(passwordFile);
      BufferedReader reader = new BufferedReader(new InputStreamReader(fdis))) {
    String line = null;
    int offset = 0;
    while ((line = reader.readLine()) != null) {
      line = line.trim();
      String[] parts = StringUtils.split(line, SEPARATOR);
      if (parts.length < 2) {
        LOG.warn("Password file contains invalid record on line {}, skipping", offset + 1);
        continue;
      }

      final String username = parts[0];
      StringBuilder builder = new StringBuilder();
      for (int i = 1; i < parts.length; i++) {
        if (builder.length() > 0) {
          builder.append(SEPARATOR);
        }
        builder.append(parts[i]);
      }

      passwordDb.put(username, builder.toString().toCharArray());
      offset++;
    }
  }

  return passwordDb;
}