org.apache.hadoop.fs.FileAlreadyExistsException Java Examples

/**
 * Create an FSDataOutputStream at the indicated Path with write-progress
 * reporting.
 * @param f the file name to open
 * @param permission
 * @param overwrite if a file with this name already exists, then if true,
 *   the file will be overwritten, and if false an error will be thrown.
 * @param bufferSize the size of the buffer to be used.
 * @param replication required block replication for the file.
 * @param blockSize
 * @param progress
 * @throws IOException
 * @see #setPermission(Path, FsPermission)
 */
public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, 
  int bufferSize, short replication, long blockSize, Progressable progress) throws IOException {
  String key = pathToKey(f);

  if (!overwrite && exists(f)) {
    throw new FileAlreadyExistsException(f + " already exists");
  }
  if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) {
    return new FSDataOutputStream(new S3AFastOutputStream(s3, this, bucket,
        key, progress, statistics, cannedACL,
        serverSideEncryptionAlgorithm, partSize, (long)multiPartThreshold,
        threadPoolExecutor), statistics);
  }
  // We pass null to FSDataOutputStream so it won't count writes that are being buffered to a file
  return new FSDataOutputStream(new S3AOutputStream(getConf(), transfers, this,
    bucket, key, progress, cannedACL, statistics, 
    serverSideEncryptionAlgorithm), null);
}
 

@Test
public void testOverwrite() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
  ServiceRecord resolved1 = operations.resolve(ENTRY_PATH);
  resolved1.description = "resolved1";
  try {
    operations.bind(ENTRY_PATH, resolved1, 0);
    fail("overwrite succeeded when it should have failed");
  } catch (FileAlreadyExistsException expected) {
    // expected
  }

  // verify there's no changed
  ServiceRecord resolved2 = operations.resolve(ENTRY_PATH);
  assertMatches(written, resolved2);
  operations.bind(ENTRY_PATH, resolved1, BindFlags.OVERWRITE);
  ServiceRecord resolved3 = operations.resolve(ENTRY_PATH);
  assertMatches(resolved1, resolved3);
}
 

private void mkdir(FileSystem fs, Path path, FsPermission fsp)
    throws IOException {
  if (!fs.exists(path)) {
    try {
      fs.mkdirs(path, fsp);
      FileStatus fsStatus = fs.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        fs.setPermission(path, fsp);
      }
    } catch (FileAlreadyExistsException e) {
      LOG.info("Directory: [" + path + "] already exists.");
    }
  }
}
 

@Test
public void testOverwrite() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
  ServiceRecord resolved1 = operations.resolve(ENTRY_PATH);
  resolved1.description = "resolved1";
  try {
    operations.bind(ENTRY_PATH, resolved1, 0);
    fail("overwrite succeeded when it should have failed");
  } catch (FileAlreadyExistsException expected) {
    // expected
  }

  // verify there's no changed
  ServiceRecord resolved2 = operations.resolve(ENTRY_PATH);
  assertMatches(written, resolved2);
  operations.bind(ENTRY_PATH, resolved1, BindFlags.OVERWRITE);
  ServiceRecord resolved3 = operations.resolve(ENTRY_PATH);
  assertMatches(resolved1, resolved3);
}
 

private boolean mkdir(Path f) throws IOException {
  try {
    FileStatus fileStatus = getFileStatus(f);
    if (fileStatus.isFile()) {
      throw new FileAlreadyExistsException(String.format(
          "Can't make directory for path '%s' since it is a file.", f));

    }
  } catch (FileNotFoundException e) {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Making dir '" + f + "' in S3");
    }
    String key = pathToKey(f) + FOLDER_SUFFIX;
    store.storeEmptyFile(key);    
  }
  return true;
}
 

private void mkdir(FileContext fc, Path path, FsPermission fsp)
    throws IOException {
  if (!fc.util().exists(path)) {
    try {
      fc.mkdir(path, fsp, true);

      FileStatus fsStatus = fc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        fc.setPermission(path, fsp);
      }
    } catch (FileAlreadyExistsException e) {
      LOG.info("Directory: [" + path + "] already exists.");
    }
  }
}
 

private void mkdir(FileSystem fs, Path path, FsPermission fsp)
    throws IOException {
  if (!fs.exists(path)) {
    try {
      fs.mkdirs(path, fsp);
      FileStatus fsStatus = fs.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        fs.setPermission(path, fsp);
      }
    } catch (FileAlreadyExistsException e) {
      LOG.info("Directory: [" + path + "] already exists.");
    }
  }
}
 

@Override
public FSDataOutputStream create(Path f, FsPermission permission,
    boolean overwrite, int bufferSize, short replication, long blockSize,
    Progressable progress) throws IOException {

  if (exists(f) && !overwrite) {
    throw new FileAlreadyExistsException("File already exists: " + f);
  }
  
  if(LOG.isDebugEnabled()) {
    LOG.debug("Creating new file '" + f + "' in S3");
  }
  Path absolutePath = makeAbsolute(f);
  String key = pathToKey(absolutePath);
  return new FSDataOutputStream(new NativeS3FsOutputStream(getConf(), store,
      key, progress, bufferSize), statistics);
}
 

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 

@Override
public void createSymlink(String target, String link, FsPermission dirPerm,
    boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
    IOException {
  CreateSymlinkRequestProto req = CreateSymlinkRequestProto.newBuilder()
      .setTarget(target)
      .setLink(link)
      .setDirPerm(PBHelper.convert(dirPerm))
      .setCreateParent(createParent)
      .build();
  try {
    rpcProxy.createSymlink(null, req);
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

/**
* Opens an FSDataOutputStream at the indicated Path with write-progress
* reporting. Same as create(), except fails if parent directory doesn't
* already exist.
* @param path the file name to open
* @param permission
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
* @param blockSize
* @param progress
* @throws IOException
* @see #setPermission(Path, FsPermission)
* @deprecated API only for 0.20-append
*/
@Deprecated
public FSDataOutputStream createNonRecursive(Path path, FsPermission permission,
    boolean overwrite,
    int bufferSize, short replication, long blockSize,
    Progressable progress) throws IOException {

  path = makeAbsolute(path);

  Path parent = path.getParent();

  if (parent != null) {
    CephStat stat = new CephStat();
    ceph.lstat(parent, stat); // handles FileNotFoundException case
    if (stat.isFile())
      throw new FileAlreadyExistsException(parent.toString());
  }

  return this.create(path, permission, overwrite,
      bufferSize, replication, blockSize, progress);
}
 

@Test
public void testCreateFileOverExistingFileNoOverwrite() throws Throwable {
  describe("Verify overwriting an existing file fails");
  Path path = path("testCreateFileOverExistingFileNoOverwrite");
  byte[] data = dataset(256, 'a', 'z');
  writeDataset(getFileSystem(), path, data, data.length, 1024, false);
  byte[] data2 = dataset(10 * 1024, 'A', 'Z');
  try {
    writeDataset(getFileSystem(), path, data2, data2.length, 1024, false);
    fail("writing without overwrite unexpectedly succeeded");
  } catch (FileAlreadyExistsException expected) {
    //expected
    handleExpectedException(expected);
  } catch (IOException relaxed) {
    handleRelaxedException("Creating a file over a file with overwrite==false",
                           "FileAlreadyExistsException",
                           relaxed);
  }
}
 

/**
 * Move a {@link Path}.
 *
 * <p>
 *   This method submits a task to move a {@link Path} and returns immediately
 *   after the task is submitted.
 * </p>
 *
 * @param src path to be moved
 * @param dstFs the destination {@link FileSystem}
 * @param dst the destination path
 * @param overwrite true to overwrite the destination
 * @param group an optional group name for the destination path
 */
public void movePath(final Path src, final FileSystem dstFs, final Path dst, final boolean overwrite,
    final Optional<String> group) {
  this.futures.add(new NamedFuture(this.executor.submit(new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      Lock lock = ParallelRunner.this.locks.get(src.toString());
      lock.lock();
      try {
        if (ParallelRunner.this.fs.exists(src)) {
          HadoopUtils.movePath(ParallelRunner.this.fs, src, dstFs, dst, overwrite, dstFs.getConf());
          if (group.isPresent()) {
            HadoopUtils.setGroup(dstFs, dst, group.get());
          }
        }
        return null;
      } catch (FileAlreadyExistsException e) {
        LOGGER.warn(String.format("Failed to move %s to %s: dst already exists", src, dst), e);
        return null;
      } finally {
        lock.unlock();
      }
    }
  }), "Move " + src + " to " + dst));
}
 

@Test
public void testCreateFileOverExistingFileNoOverwrite() throws Throwable {
  describe("Verify overwriting an existing file fails");
  Path path = path("testCreateFileOverExistingFileNoOverwrite");
  byte[] data = dataset(256, 'a', 'z');
  writeDataset(getFileSystem(), path, data, data.length, 1024, false);
  byte[] data2 = dataset(10 * 1024, 'A', 'Z');
  try {
    writeDataset(getFileSystem(), path, data2, data2.length, 1024, false);
    fail("writing without overwrite unexpectedly succeeded");
  } catch (FileAlreadyExistsException expected) {
    //expected
    handleExpectedException(expected);
  } catch (IOException relaxed) {
    handleRelaxedException("Creating a file over a file with overwrite==false",
                           "FileAlreadyExistsException",
                           relaxed);
  }
}
 

private boolean mkdir(Path f) throws IOException {
  try {
    FileStatus fileStatus = getFileStatus(f);
    if (fileStatus.isFile()) {
      throw new FileAlreadyExistsException(String.format(
          "Can't make directory for path '%s' since it is a file.", f));

    }
  } catch (FileNotFoundException e) {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Making dir '" + f + "' in S3");
    }
    String key = pathToKey(f) + FOLDER_SUFFIX;
    store.storeEmptyFile(key);    
  }
  return true;
}
 

/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
  try {
    namenode.rename2(src, dst, options);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   DSQuotaExceededException.class,
                                   FileAlreadyExistsException.class,
                                   FileNotFoundException.class,
                                   ParentNotDirectoryException.class,
                                   SafeModeException.class,
                                   NSQuotaExceededException.class,
                                   UnresolvedPathException.class,
                                   SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}
 

/**
 * Rename a {@link Path}.
 *
 * <p>
 *   This method submits a task to rename a {@link Path} and returns immediately
 *   after the task is submitted.
 * </p>
 *
 * @param src path to be renamed
 * @param dst new path after rename
 * @param group an optional group name for the destination path
 */
public void renamePath(final Path src, final Path dst, final Optional<String> group) {
  this.futures.add(new NamedFuture(this.executor.submit(new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      Lock lock = ParallelRunner.this.locks.get(src.toString());
      lock.lock();
      try {
        if (ParallelRunner.this.fs.exists(src)) {
          HadoopUtils.renamePath(ParallelRunner.this.fs, src, dst);
          if (group.isPresent()) {
            HadoopUtils.setGroup(ParallelRunner.this.fs, dst, group.get());
          }
        }
        return null;
      } catch (FileAlreadyExistsException e) {
        LOGGER.warn(String.format("Failed to rename %s to %s: dst already exists", src, dst), e);
        return null;
      } finally {
        lock.unlock();
      }
    }
  }), "Rename " + src + " to " + dst));
}
 

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 

/**
 * Translates HDFS specific Exceptions to Pravega-equivalent Exceptions.
 *
 * @param segmentName Name of the stream segment on which the exception occurs.
 * @param e           The exception to be translated.
 * @return  The exception to be thrown.
 */
static <T> StreamSegmentException convertException(String segmentName, Throwable e) {
    if (e instanceof RemoteException) {
        e = ((RemoteException) e).unwrapRemoteException();
    }

    if (e instanceof PathNotFoundException || e instanceof FileNotFoundException) {
        return new StreamSegmentNotExistsException(segmentName, e);
    } else if (e instanceof FileAlreadyExistsException || e instanceof AlreadyBeingCreatedException) {
        return new StreamSegmentExistsException(segmentName, e);
    } else if (e instanceof AclException) {
        return new StreamSegmentSealedException(segmentName, e);
    } else {
        throw Exceptions.sneakyThrow(e);
    }
}
 

@Override
public boolean mkdirs(String src, FsPermission masked, boolean createParent)
    throws AccessControlException, FileAlreadyExistsException,
    FileNotFoundException, NSQuotaExceededException,
    ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
    IOException {
  MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
      .setSrc(src)
      .setMasked(PBHelper.convert(masked))
      .setCreateParent(createParent).build();

  try {
    return rpcProxy.mkdirs(null, req).getResult();
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

/**
 * create a directory at path specified by parent
 */
private static INodesInPath unprotectedMkdir(FSDirectory fsd, long inodeId,
    INodesInPath parent, byte[] name, PermissionStatus permission,
    List<AclEntry> aclEntries, long timestamp)
    throws QuotaExceededException, AclException, FileAlreadyExistsException {
  assert fsd.hasWriteLock();
  assert parent.getLastINode() != null;
  if (!parent.getLastINode().isDirectory()) {
    throw new FileAlreadyExistsException("Parent path is not a directory: " +
        parent.getPath() + " " + DFSUtil.bytes2String(name));
  }
  final INodeDirectory dir = new INodeDirectory(inodeId, name, permission,
      timestamp);

  INodesInPath iip = fsd.addLastINode(parent, dir, true);
  if (iip != null && aclEntries != null) {
    AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID);
  }
  return iip;
}
 

@Override
public boolean mkdirs(String src, FsPermission masked, boolean createParent)
    throws AccessControlException, FileAlreadyExistsException,
    FileNotFoundException, NSQuotaExceededException,
    ParentNotDirectoryException, SafeModeException, UnresolvedLinkException,
    IOException {
  MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
      .setSrc(src)
      .setMasked(PBHelper.convert(masked))
      .setCreateParent(createParent).build();

  try {
    return rpcProxy.mkdirs(null, req).getResult();
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

private static void validateDestination(
    String src, String dst, INode srcInode)
    throws IOException {
  String error;
  if (srcInode.isSymlink() &&
      dst.equals(srcInode.asSymlink().getSymlinkString())) {
    throw new FileAlreadyExistsException("Cannot rename symlink " + src
        + " to its target " + dst);
  }
  // dst cannot be a directory or a file under src
  if (dst.startsWith(src)
      && dst.charAt(src.length()) == Path.SEPARATOR_CHAR) {
    error = "Rename destination " + dst
        + " is a directory or file under source " + src;
    NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
        + error);
    throw new IOException(error);
  }
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testRenameDirectoryIfDstPathExists() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path srcDir = new Path(fsHome, "/tmp/");
    Path dstDir = new Path(fsHome, "/tmpNew/");

    FSDataOutputStream os = fs.create(new Path(srcDir, "file1"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    os = fs.create(new Path(dstDir, "file2"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    try {
        fs.rename(srcDir, dstDir);

        fail("FileAlreadyExistsException expected.");
    }
    catch (FileAlreadyExistsException ignore) {
        // No-op.
    }

    // Check all the files stay unchanged:
    assertPathExists(fs, dstDir);
    assertPathExists(fs, new Path(dstDir, "file2"));

    assertPathExists(fs, srcDir);
    assertPathExists(fs, new Path(srcDir, "file1"));
}
 

public void checkOutputSpecs(JobContext job) 
    throws FileAlreadyExistsException, IOException{
  super.checkOutputSpecs(job);
  // creating dummy TaskAttemptID
  TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.JOB_SETUP, 0, 0);
  getOutputCommitter(new TaskAttemptContextImpl(job.getConfiguration(), tid)).
    setupJob(job);
}
 

/**
 * @param permission Currently ignored.
 */
@Override
public FSDataOutputStream create(Path file, FsPermission permission,
    boolean overwrite, int bufferSize,
    short replication, long blockSize, Progressable progress)
  throws IOException {

  INode inode = store.retrieveINode(makeAbsolute(file));
  if (inode != null) {
    if (overwrite) {
      delete(file, true);
    } else {
      throw new FileAlreadyExistsException("File already exists: " + file);
    }
  } else {
    Path parent = file.getParent();
    if (parent != null) {
      if (!mkdirs(parent)) {
        throw new IOException("Mkdirs failed to create " + parent.toString());
      }
    }      
  }
  return new FSDataOutputStream
      (new S3OutputStream(getConf(), store, makeAbsolute(file),
                          blockSize, progress, bufferSize),
       statistics);
}
 

@Override
public boolean mkdirs(Path dir, FsPermission permission)
    throws AccessControlException, FileAlreadyExistsException {
  if (theInternalDir.isRoot && dir == null) {
    throw new FileAlreadyExistsException("/ already exits");
  }
  // Note dir starts with /
  if (theInternalDir.children.containsKey(dir.toString().substring(1))) {
    return true; // this is the stupid semantics of FileSystem
  }
  throw readOnlyMountTable("mkdirs",  dir);
}
 

private static void validateOverwrite(
    String src, String dst, boolean overwrite, INode srcInode, INode dstInode)
    throws IOException {
  String error;// It's OK to rename a file to a symlink and vice versa
  if (dstInode.isDirectory() != srcInode.isDirectory()) {
    error = "Source " + src + " and destination " + dst
        + " must both be directories";
    NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
        + error);
    throw new IOException(error);
  }
  if (!overwrite) { // If destination exists, overwrite flag must be true
    error = "rename destination " + dst + " already exists";
    NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
        + error);
    throw new FileAlreadyExistsException(error);
  }
  if (dstInode.isDirectory()) {
    final ReadOnlyList<INode> children = dstInode.asDirectory()
        .getChildrenList(Snapshot.CURRENT_STATE_ID);
    if (!children.isEmpty()) {
      error = "rename destination directory is not empty: " + dst;
      NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
          + error);
      throw new IOException(error);
    }
  }
}
 

private void createDir(Path dir) throws IOException {
  try {
    FileStatus status = fs.getFileStatus(dir);
    if (!status.isDirectory()) {
      throw new FileAlreadyExistsException("Unexpected file in store: "
          + dir);
    }
    if (!status.getPermission().equals(DIR_PERMISSIONS)) {
      fs.setPermission(dir, DIR_PERMISSIONS);
    }
  } catch (FileNotFoundException e) {
    fs.mkdirs(dir, DIR_PERMISSIONS);
  }
}
 

@Override
public void renameInternal(final Path src, final Path dst)
    throws AccessControlException, FileAlreadyExistsException,
    FileNotFoundException, ParentNotDirectoryException,
    UnresolvedLinkException, IOException {
  renameInternal(src, dst, false);
}