org.apache.hadoop.fs.shell.CommandFormat Java Examples

/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  try {
    quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  } catch (NumberFormatException nfe) {
    throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota.");
  }
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

private static <T> CommandFormat checkArgLimits(
    Class<? extends IllegalArgumentException> expectedErr,
    int min, int max, String ... opts)
{
  CommandFormat cf = new CommandFormat(min, max, opts);
  List<String> parsedArgs = new ArrayList<String>(args);
  
  Class<?> cfError = null;
  try {
    cf.parse(parsedArgs);
  } catch (IllegalArgumentException e) {
    System.out.println(e.getMessage());
    cfError = e.getClass();
  }

  assertEquals(expectedErr, cfError);
  if (expectedErr == null) {
    assertEquals(expectedArgs, parsedArgs);
    assertEquals(expectedOpts, cf.getOpts());
  }
  return cf;
}
 

@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R", null);
  cf.parse(args);
  setRecursive(cf.getOpt("R"));

  String modeStr = args.removeFirst();
  try {
    pp = new ChmodParser(modeStr);
  } catch (IllegalArgumentException iea) {
    // TODO: remove "chmod : " so it's not doubled up in output, but it's
    // here for backwards compatibility...
    throw new IllegalArgumentException(
        "chmod : mode '" + modeStr + "' does not match the expected pattern.");      
  }
}
 

private static <T> CommandFormat checkArgLimits(
    Class<? extends IllegalArgumentException> expectedErr,
    int min, int max, String ... opts)
{
  CommandFormat cf = new CommandFormat(min, max, opts);
  List<String> parsedArgs = new ArrayList<String>(args);
  
  Class<?> cfError = null;
  try {
    cf.parse(parsedArgs);
  } catch (IllegalArgumentException e) {
    System.out.println(e.getMessage());
    cfError = e.getClass();
  }

  assertEquals(expectedErr, cfError);
  if (expectedErr == null) {
    assertEquals(expectedArgs, parsedArgs);
    assertEquals(expectedOpts, cf.getOpts());
  }
  return cf;
}
 

@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R", null);
  cf.parse(args);
  setRecursive(cf.getOpt("R"));

  String modeStr = args.removeFirst();
  try {
    pp = new ChmodParser(modeStr);
  } catch (IllegalArgumentException iea) {
    // TODO: remove "chmod : " so it's not doubled up in output, but it's
    // here for backwards compatibility...
    throw new IllegalArgumentException(
        "chmod : mode '" + modeStr + "' does not match the expected pattern.");      
  }
}
 

/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  try {
    quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  } catch (NumberFormatException nfe) {
    throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota.");
  }
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  final int minArgs = 2;  // We need the replication and at least one path.
  CommandFormat c =
    new CommandFormat("setrep", minArgs, SETREP_MAX_PATHS, "R", "w");
  short rep = 0;
  List<String> dsts = null;
  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dsts = parameters.subList(1, parameters.size());
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  for (String dst: dsts) {
    setReplication(rep, dst, c.getOpt("R"), waitList);
  }

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void head(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("head", 1, 1);
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + HEAD_USAGE);
    throw iae;
  }

  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  int len = (fileSize > 1024) ? 1024 : (int) fileSize;

  FSDataInputStream in = srcFs.open(path);
  byte buf[] = new byte[len];
  IOUtils.readFully(in, buf, 0, len);
  System.out.write(buf);
  System.out.write('\n');
  in.close();
}
 

/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("setrep", 2, 2, "R", "w");
  String dst = null;
  short rep = 0;

  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dst = parameters.get(1);
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  setReplication(rep, dst, c.getOpt("R"), waitList);

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 

/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK, OPTION_FOLLOW_ARG_LINK);
  cf.parse(args);
  
  if(cf.getOpt(OPTION_FOLLOW_LINK)) {
    options.setFollowLink(true);
  }
  else if(cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    options.setFollowArgLink(true);
  }
}
 

@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R");
  cf.parse(args);
  setRecursive(cf.getOpt("R"));
  parseOwnerGroup(args.removeFirst());
}
 

/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R");
  cf.parse(args);
  setRecursive(cf.getOpt("R"));
  parseOwnerGroup(args.removeFirst());
}
 

/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 

/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory.
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]
 * @exception: IOException
 * @see org.apache.hadoop.fs.FileSystem.globStatus
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc", "gencrc");

  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean genCrc = cf.getOpt("gencrc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum, genCrc);
  } else {
    File dst = new File(dststr);
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory();
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc, genCrc);
    }
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}
 

/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory. 
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]  
 * @exception: IOException  
 * @see org.apache.hadoop.fs.FileSystem.globStatus 
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
  
  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FreightStreamer " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum);
  } else {
    File dst = new File(dststr);      
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory(); 
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc);
    }
  }
}
 

/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FreightStreamer " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}