org.apache.hadoop.fs.Options Java Examples

private DataOutputStream getOutputStream(FileSystemWALPointer pointer) throws IOException
{
  Preconditions.checkArgument(outputStream == null, "output stream is not null");

  if (pointer.offset > 0 && (fileSystemWAL.fileContext.getDefaultFileSystem() instanceof LocalFs ||
      fileSystemWAL.fileContext.getDefaultFileSystem() instanceof RawLocalFs)) {
    //On local file system the stream is always closed and never flushed so we open it again in append mode if the
    //offset > 0. This block is entered only when appending to wal while writing on local fs.
    return fileSystemWAL.fileContext.create(new Path(fileSystemWAL.tempPartFiles.get(pointer.partNum)),
        EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND), Options.CreateOpts.CreateParent.createParent());
  }

  String partFile = fileSystemWAL.getPartFilePath(pointer.partNum);
  String tmpFilePath = createTmpFilePath(partFile);
  fileSystemWAL.tempPartFiles.put(pointer.partNum, tmpFilePath);

  Preconditions.checkArgument(pointer.offset == 0, "offset > 0");
  LOG.debug("open {} => {}", pointer.partNum, tmpFilePath);
  outputStream = fileSystemWAL.fileContext.create(new Path(tmpFilePath),
      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());
  return outputStream;
}
 

/** @throws Exception If failed. */
@Test
public void testOpen() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "someFile");

    int cnt = 2 * 1024;

    try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()))) {

        for (long i = 0; i < cnt; i++)
            out.writeLong(i);
    }

    assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());

    try (FSDataInputStream in = fs.open(file, 1024)) {

        for (long i = 0; i < cnt; i++)
            assertEquals(i, in.readLong());
    }
}
 

@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
    throws IOException {
  checkNNStartup();
  if(stateChangeLog.isDebugEnabled()) {
    stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
  }
  if (!checkPathLength(dst)) {
    throw new IOException("rename: Pathname too long.  Limit "
        + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
  }
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return; // Return previous response
  }
  boolean success = false;
  try {
    namesystem.renameTo(src, dst, cacheEntry != null, options);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
  metrics.incrFilesRenamed();
}
 

/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckNonRecursiveness() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path tmpDir = new Path(fsHome, "/tmp");

    fs.setOwner(file, "fUser", "fGroup");
    fs.setOwner(tmpDir, "dUser", "dGroup");

    assertEquals("dUser", fs.getFileStatus(tmpDir).getOwner());
    assertEquals("dGroup", fs.getFileStatus(tmpDir).getGroup());

    assertEquals("fUser", fs.getFileStatus(file).getOwner());
    assertEquals("fGroup", fs.getFileStatus(file).getGroup());
}
 

/** @throws Exception If failed. */
@Test
public void testOpenIfPathIsAlreadyOpened() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "someFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    FSDataInputStream is1 = fs.open(file);
    FSDataInputStream is2 = fs.open(file);

    is1.close();
    is2.close();
}
 

/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
  try {
    namenode.rename2(src, dst, options);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   DSQuotaExceededException.class,
                                   FileAlreadyExistsException.class,
                                   FileNotFoundException.class,
                                   ParentNotDirectoryException.class,
                                   SafeModeException.class,
                                   NSQuotaExceededException.class,
                                   UnresolvedPathException.class,
                                   SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}
 

/** @throws Exception If failed. */
@Test
public void testAppendIfPathPointsToDirectory() throws Exception {
    final Path fsHome = new Path(primaryFsUri);
    final Path dir = new Path(fsHome, "/tmp");
    Path file = new Path(dir, "my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            return fs.create(new Path(fsHome, dir), EnumSet.of(CreateFlag.APPEND),
                Options.CreateOpts.perms(FsPermission.getDefault()));
        }
    }, IOException.class, null);
}
 

/** @throws Exception If failed. */
@Test
public void testAppendIfFileIsAlreadyBeingOpenedToWrite() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "someFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    FSDataOutputStream appendOs = fs.create(file, EnumSet.of(CreateFlag.APPEND),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            return fs.create(file, EnumSet.of(CreateFlag.APPEND),
                Options.CreateOpts.perms(FsPermission.getDefault()));
        }
    }, IOException.class, null);

    appendOs.close();
}
 

/** @throws Exception If failed. */
@Test
public void testRenameCheckParametersDstPathIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "someFile");

    fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault())).close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.rename(file, null);

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: f");
}
 

/** @throws Exception If failed. */
@Test
public void testSetOwner() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    assertEquals(getClientFsUser(), fs.getFileStatus(file).getOwner());

    fs.setOwner(file, "aUser", "aGroup");

    assertEquals("aUser", fs.getFileStatus(file).getOwner());
    assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
 

/** @throws Exception If failed. */
@Test
public void testSetOwnerIfOutputStreamIsNotClosed() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "myFile");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    fs.setOwner(file, "aUser", "aGroup");

    os.close();

    assertEquals("aUser", fs.getFileStatus(file).getOwner());
    assertEquals("aGroup", fs.getFileStatus(file).getGroup());
}
 

/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersGroupIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(file, "aUser", null);

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: grpName");
}
 

/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersUserIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(file, null, "aGroup");

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: username");
}
 

/** @throws Exception If failed. */
@Test
public void testSetPermissionIfOutputStreamIsNotClosed() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "myFile");

    FsPermission perm = new FsPermission((short)123);

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    fs.setPermission(file, perm);

    os.close();

    assertEquals(perm, fs.getFileStatus(file).getPermission());
}
 

/** @throws Exception If failed. */
@Test
public void testRenameDirectory() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/tmp/");
    Path newDir = new Path(fsHome, "/tmpNew/");

    FSDataOutputStream os = fs.create(new Path(dir, "myFile"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    fs.rename(dir, newDir);

    assertPathDoesNotExist(fs, dir);
    assertPathExists(fs, newDir);
}
 

private void testCopyPartialHelper(int dataSize, int offset, long size) throws IOException
  {
    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
    File file = new File("target/IOUtilsTest/testCopyPartial/input");
    createDataFile(file, dataSize);

    FileContext fileContext = FileContext.getFileContext();
    DataInputStream inputStream = fileContext.open(new Path(file.getAbsolutePath()));

    Path output = new Path("target/IOUtilsTest/testCopyPartial/output");
    DataOutputStream outputStream = fileContext.create(output, EnumSet
        .of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent());

    if (offset == 0) {
      IOUtils.copyPartial(inputStream, size, outputStream);
    } else {
      IOUtils.copyPartial(inputStream, offset, size, outputStream);
    }

    outputStream.close();

    Assert.assertTrue("output exists", fileContext.util().exists(output));
    Assert.assertEquals("output size", size, fileContext.getFileStatus(output).getLen());
//    FileUtils.deleteQuietly(new File("target/IOUtilsTest"));
  }
 

@Override // ClientProtocol
public void rename2(String src, String dst, Options.Rename... options)
    throws IOException {
  checkNNStartup();
  if(stateChangeLog.isDebugEnabled()) {
    stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
  }
  if (!checkPathLength(dst)) {
    throw new IOException("rename: Pathname too long.  Limit "
        + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
  }
  CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
  if (cacheEntry != null && cacheEntry.isSuccess()) {
    return; // Return previous response
  }
  boolean success = false;
  try {
    namesystem.renameTo(src, dst, cacheEntry != null, options);
    success = true;
  } finally {
    RetryCache.setState(cacheEntry, success);
  }
  metrics.incrFilesRenamed();
}
 

/** @throws Exception If failed. */
@Test
public void testCreateBase() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path dir = new Path(fsHome, "/someDir1/someDir2/someDir3");
    Path file = new Path(dir, "someFile");

    assertPathDoesNotExist(fs, file);

    FsPermission fsPerm = new FsPermission((short)644);

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(fsPerm));

    // Try to write something in file.
    os.write("abc".getBytes());

    os.close();

    // Check file status.
    FileStatus fileStatus = fs.getFileStatus(file);

    assertFalse(fileStatus.isDirectory());
    assertEquals(file, fileStatus.getPath());
    assertEquals(fsPerm, fileStatus.getPermission());
}
 

/**
 * Rename file or directory.
 * @see ClientProtocol#rename2(String, String, Options.Rename...)
 */
public void rename(String src, String dst, Options.Rename... options)
    throws IOException {
  checkOpen();
  TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
  try {
    namenode.rename2(src, dst, options);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException(AccessControlException.class,
                                   DSQuotaExceededException.class,
                                   FileAlreadyExistsException.class,
                                   FileNotFoundException.class,
                                   ParentNotDirectoryException.class,
                                   SafeModeException.class,
                                   NSQuotaExceededException.class,
                                   UnresolvedPathException.class,
                                   SnapshotAccessControlException.class);
  } finally {
    scope.close();
  }
}
 

/** @throws Exception If failed. */
@Test
public void testDeleteFailsIfNonRecursive() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");

    FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    final Path someDir2 = new Path(fsHome, "/someDir1/someDir2");

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.delete(someDir2, false);

            return null;
        }
    }, PathIsNotEmptyDirectoryException.class, null);

    assertPathExists(fs, someDir2);
    assertPathExists(fs, someDir3);
}
 

/**
 * A wrapper around {@link FileSystem#rename(Path, Path)} which throws {@link IOException} if
 * {@link FileSystem#rename(Path, Path)} returns False.
 */
public static void renamePath(FileSystem fs, Path oldName, Path newName, boolean overwrite) throws IOException {
  //In default implementation of rename with rewrite option in FileSystem, if the parent dir of dst does not exist, it will throw exception,
  //Which will fail some of our job unintentionally. So we only call that method when fs is an instance of DistributedFileSystem to avoid inconsistency problem
  if(fs instanceof DistributedFileSystem) {
    Options.Rename renameOptions = (overwrite) ? Options.Rename.OVERWRITE : Options.Rename.NONE;
    ((DistributedFileSystem) fs).rename(oldName, newName, renameOptions);
  } else {
    if (!fs.exists(oldName)) {
      throw new FileNotFoundException(String.format("Failed to rename %s to %s: src not found", oldName, newName));
    }
    if (fs.exists(newName)) {
      if (overwrite) {
        HadoopUtils.moveToTrash(fs, newName);
      } else {
        throw new FileAlreadyExistsException(String.format("Failed to rename %s to %s: dst already exists", oldName, newName));
      }
    }
    if (!fs.rename(oldName, newName)) {
      throw new IOException(String.format("Failed to rename %s to %s", oldName, newName));
    }
  }
}
 

/** @throws Exception If failed. */
@Test
public void testDeleteRecursivelyFromRoot() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path someDir3 = new Path(fsHome, "/someDir1/someDir2/someDir3");

    FSDataOutputStream os = fs.create(someDir3, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path root = new Path(fsHome, "/");

    assertFalse(fs.delete(root, true));

    assertTrue(fs.delete(new Path(fsHome, "/someDir1"), true));

    assertPathDoesNotExist(fs, someDir3);
    assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));
    assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1"));
    assertPathExists(fs, root);
}
 

/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckDefaultPermission() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    fs.setPermission(file, null);

    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file.getParent()).getPermission());
}
 

/** @throws Exception If failed. */
@Test
public void testSetPermissionCheckNonRecursiveness() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    Path tmpDir = new Path(fsHome, "/tmp");

    FsPermission perm = new FsPermission((short)123);

    fs.setPermission(tmpDir, perm);

    assertEquals(perm, fs.getFileStatus(tmpDir).getPermission());
    assertEquals(FsPermission.getDefault(), fs.getFileStatus(file).getPermission());
}
 

/** @throws Exception If failed. */
@SuppressWarnings("OctalInteger")
@Test
public void testSetPermission() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    for (short i = 0; i <= 0777; i += 7) {
        FsPermission perm = new FsPermission(i);

        fs.setPermission(file, perm);

        assertEquals(perm, fs.getFileStatus(file).getPermission());
    }
}
 

/** @throws Exception If failed. */
@Test
public void testSetOwnerCheckParametersPathIsNull() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    final Path file = new Path(fsHome, "/tmp/my");

    FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fs.setOwner(null, "aUser", "aGroup");

            return null;
        }
    }, NullPointerException.class, "Ouch! Argument cannot be null: p");
}
 

/** @throws Exception If failed. */
@Test
public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path srcFile = new Path(fsHome, "srcFile");
    Path dstFile = new Path(fsHome, "dstFile");

    FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    int cnt = 1024;

    for (int i = 0; i < cnt; i++)
        os.writeInt(i);

    os.close();

    FSDataInputStream is = fs.open(srcFile);

    for (int i = 0; i < cnt; i++) {
        if (i == 100)
            // Rename file during the read process.
            fs.rename(srcFile, dstFile);

        assertEquals(i, is.readInt());
    }

    assertPathDoesNotExist(fs, srcFile);
    assertPathExists(fs, dstFile);

    os.close();
    is.close();
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testRenameDirectoryIfDstPathExists() throws Exception {
    Path fsHome = new Path(primaryFsUri);
    Path srcDir = new Path(fsHome, "/tmp/");
    Path dstDir = new Path(fsHome, "/tmpNew/");

    FSDataOutputStream os = fs.create(new Path(srcDir, "file1"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    os = fs.create(new Path(dstDir, "file2"), EnumSet.noneOf(CreateFlag.class),
        Options.CreateOpts.perms(FsPermission.getDefault()));

    os.close();

    try {
        fs.rename(srcDir, dstDir);

        fail("FileAlreadyExistsException expected.");
    }
    catch (FileAlreadyExistsException ignore) {
        // No-op.
    }

    // Check all the files stay unchanged:
    assertPathExists(fs, dstDir);
    assertPathExists(fs, new Path(dstDir, "file2"));

    assertPathExists(fs, srcDir);
    assertPathExists(fs, new Path(srcDir, "file1"));
}
 

@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void save(Object object, int operatorId, long windowId) throws IOException
{
  String operatorIdStr = String.valueOf(operatorId);
  Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE);
  String window = Long.toHexString(windowId);
  boolean stateSaved = false;
  FSDataOutputStream stream = null;
  try {
    stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
      Options.CreateOpts.CreateParent.createParent());
    store(stream, object);
    stateSaved = true;
  } catch (Throwable t) {
    logger.debug("while saving {} {}", operatorId, window, t);
    stateSaved = false;
    throw Throwables.propagate(t);
  } finally {
    try {
      if (stream != null) {
        stream.close();
      }
    } catch (IOException ie) {
      stateSaved = false;
      throw new RuntimeException(ie);
    } finally {
      if (stateSaved) {
        logger.debug("Saving {}: {}", operatorId, window);
        fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window),
            Options.Rename.OVERWRITE);
      }
    }
  }
}
 

protected void finalize(int partNum) throws IOException
{
  String tmpToFinalize = fileSystemWAL.tempPartFiles.remove(partNum);
  Path tmpPath = new Path(tmpToFinalize);
  if (fileSystemWAL.fileContext.util().exists(tmpPath)) {
    LOG.debug("finalize {} of part {}", tmpPath, partNum);
    fileSystemWAL.fileContext.rename(tmpPath, new Path(fileSystemWAL.getPartFilePath(partNum)),
        Options.Rename.OVERWRITE);
    latestFinalizedPart = partNum;
  }
}