Java Code Examples for org.apache.hadoop.fs.shell.CommandFormat#getOpt()

protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK, OPTION_FOLLOW_ARG_LINK);
  cf.parse(args);
  
  if(cf.getOpt(OPTION_FOLLOW_LINK)) {
    options.setFollowLink(true);
  }
  else if(cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    options.setFollowArgLink(true);
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  final int minArgs = 2;  // We need the replication and at least one path.
  CommandFormat c =
    new CommandFormat("setrep", minArgs, SETREP_MAX_PATHS, "R", "w");
  short rep = 0;
  List<String> dsts = null;
  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dsts = parameters.subList(1, parameters.size());
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  for (String dst: dsts) {
    setReplication(rep, dst, c.getOpt("R"), waitList);
  }

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("setrep", 2, 2, "R", "w");
  String dst = null;
  short rep = 0;

  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dst = parameters.get(1);
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  setReplication(rep, dst, c.getOpt("R"), waitList);

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 

@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf =
      new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK,
          OPTION_FOLLOW_ARG_LINK, null);
  cf.parse(args);

  if (cf.getOpt(OPTION_FOLLOW_LINK)) {
    getOptions().setFollowLink(true);
  } else if (cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    getOptions().setFollowArgLink(true);
  }

  // search for first non-path argument (ie starts with a "-") and capture and
  // remove the remaining arguments as expressions
  LinkedList<String> expressionArgs = new LinkedList<String>();
  Iterator<String> it = args.iterator();
  boolean isPath = true;
  while (it.hasNext()) {
    String arg = it.next();
    if (isPath) {
      if (arg.startsWith("-")) {
        isPath = false;
      }
    }
    if (!isPath) {
      expressionArgs.add(arg);
      it.remove();
    }
  }

  if (args.isEmpty()) {
    args.add(Path.CUR_DIR);
  }

  Expression expression = parseExpression(expressionArgs);
  if (!expression.isAction()) {
    Expression and = getExpression(And.class);
    Deque<Expression> children = new LinkedList<Expression>();
    children.add(getExpression(Print.class));
    children.add(expression);
    and.addChildren(children);
    expression = and;
  }

  setRootExpression(expression);
}
 

@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf =
      new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK,
          OPTION_FOLLOW_ARG_LINK, null);
  cf.parse(args);

  if (cf.getOpt(OPTION_FOLLOW_LINK)) {
    getOptions().setFollowLink(true);
  } else if (cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    getOptions().setFollowArgLink(true);
  }

  // search for first non-path argument (ie starts with a "-") and capture and
  // remove the remaining arguments as expressions
  LinkedList<String> expressionArgs = new LinkedList<String>();
  Iterator<String> it = args.iterator();
  boolean isPath = true;
  while (it.hasNext()) {
    String arg = it.next();
    if (isPath) {
      if (arg.startsWith("-")) {
        isPath = false;
      }
    }
    if (!isPath) {
      expressionArgs.add(arg);
      it.remove();
    }
  }

  if (args.isEmpty()) {
    args.add(Path.CUR_DIR);
  }

  Expression expression = parseExpression(expressionArgs);
  if (!expression.isAction()) {
    Expression and = getExpression(And.class);
    Deque<Expression> children = new LinkedList<Expression>();
    children.add(getExpression(Print.class));
    children.add(expression);
    and.addChildren(children);
    expression = and;
  }

  setRootExpression(expression);
}
 

/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory.
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]
 * @exception: IOException
 * @see org.apache.hadoop.fs.FileSystem.globStatus
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc", "gencrc");

  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean genCrc = cf.getOpt("gencrc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum, genCrc);
  } else {
    File dst = new File(dststr);
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory();
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc, genCrc);
    }
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}
 

/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory. 
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]  
 * @exception: IOException  
 * @see org.apache.hadoop.fs.FileSystem.globStatus 
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
  
  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FreightStreamer " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum);
  } else {
    File dst = new File(dststr);      
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory(); 
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc);
    }
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FreightStreamer " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}
 

/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory. 
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]  
 * @exception: IOException  
 * @see org.apache.hadoop.fs.FileSystem.globStatus 
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
  
  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum);
  } else {
    File dst = new File(dststr);      
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory(); 
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc);
    }
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}