org.apache.hadoop.fs.contract.ContractTestUtils Java Examples

private void testDeleteCreatesFakeParentDir() throws Exception {
  Path grandparent = new Path("/testDeleteCreatesFakeParentDir");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);
  rootItemCount++; // grandparent

  // Verify that parent dir key does not exist
  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // Delete the child key
  fs.delete(child, false);

  // Deleting the only child should create the parent dir key if it does
  // not exist
  String parentKey = o3fs.pathToKey(parent) + "/";
  OzoneKeyDetails parentKeyInfo = getKey(parent, true);
  assertEquals(parentKey, parentKeyInfo.getName());
}
 

private void testListStatus() throws Exception {
  Path parent = new Path("/testListStatus");
  Path file1 = new Path(parent, "key1");
  Path file2 = new Path(parent, "key2");
  ContractTestUtils.touch(fs, file1);
  ContractTestUtils.touch(fs, file2);
  rootItemCount++; // parent

  // ListStatus on a directory should return all subdirs along with
  // files, even if there exists a file and sub-dir with the same name.
  FileStatus[] fileStatuses = o3fs.listStatus(parent);
  assertEquals("FileStatus did not return all children of the directory",
      2, fileStatuses.length);

  // ListStatus should return only the immediate children of a directory.
  Path file3 = new Path(parent, "dir1/key3");
  Path file4 = new Path(parent, "dir1/key4");
  ContractTestUtils.touch(fs, file3);
  ContractTestUtils.touch(fs, file4);
  fileStatuses = o3fs.listStatus(parent);
  assertEquals("FileStatus did not return all children of the directory",
      3, fileStatuses.length);
}
 

private void testCreateDoesNotAddParentDirKeys() throws Exception {
  Path grandparent = new Path("/testCreateDoesNotAddParentDirKeys");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);
  rootItemCount++; // grandparent

  OzoneKeyDetails key = getKey(child, false);
  assertEquals(key.getName(), o3fs.pathToKey(child));

  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // List status on the parent should show the child file
  assertEquals("List status of parent should include the 1 child file", 1L,
      fs.listStatus(parent).length);
  assertTrue("Parent directory does not appear to be a directory",
      fs.getFileStatus(parent).isDirectory());
}
 

@Test
public void testConcatMultiple() throws Throwable {
  int numFiles = GoogleCloudStorage.MAX_COMPOSE_OBJECTS * 3 / 2;
  Path testPath = path("test");

  byte[][] blocks = new byte[numFiles][0];
  Path[] srcs = new Path[numFiles];
  for (int i = 0; i < numFiles; i++) {
    Path srcFile = new Path(testPath, "" + i);
    blocks[i] = dataset(TEST_FILE_LEN, i, 255);
    createFile(getFileSystem(), srcFile, true, blocks[i]);
    srcs[i] = srcFile;
  }
  Path target = new Path(testPath, "target");

  createFile(getFileSystem(), target, false, new byte[0]);
  getFileSystem().concat(target, srcs);
  assertFileHasLength(getFileSystem(), target, TEST_FILE_LEN * numFiles);
  ContractTestUtils.validateFileContent(
      ContractTestUtils.readDataset(getFileSystem(), target, TEST_FILE_LEN * numFiles), blocks);
}
 

@Test
public void testOpenCreate() throws IOException {
  Path dir = new Path("/tests3a");
  ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);


  /*
  Enable to test the multipart upload
  try {
    ContractTestUtils.createAndVerifyFile(fs, dir,
        (long)6 * 1024 * 1024 * 1024);
  } catch (IOException e) {
    fail(e.getMessage());
  }
  */
}
 

@Test
public void testOpenCreate() throws IOException {
  Path dir = new Path("/tests3a");
  ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
  ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);


  /*
  Enable to test the multipart upload
  try {
    ContractTestUtils.createAndVerifyFile(fs, dir,
        (long)6 * 1024 * 1024 * 1024);
  } catch (IOException e) {
    fail(e.getMessage());
  }
  */
}
 

/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 

/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 

@Override
public void testRenameFileBeingAppended() throws Throwable {
  try {
    super.testRenameFileBeingAppended();
    fail("Expected a FileNotFoundException");
  } catch (FileNotFoundException e) {
    // downgrade
    ContractTestUtils.downgrade("Renaming an open file" +
                                "still creates the old path", e);

  }
}
 

/**
 *  tweak some of the contract parameters based on the local system
 *  state
 */
protected void adjustContractToLocalEnvironment() {
  if (Shell.WINDOWS) {
    //NTFS doesn't do case sensitivity, and its permissions are ACL-based
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false);
    getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false);
  } else if (ContractTestUtils.isOSX()) {
    //OSX HFS+ is not case sensitive
    getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE),
                         false);
  }
}
 

/**
 * Tests listStatus on a path with subdirs.
 */
private void testListStatusOnSubDirs() throws Exception {
  // Create the following key structure
  //      /dir1/dir11/dir111
  //      /dir1/dir12
  //      /dir1/dir12/file121
  //      /dir2
  // ListStatus on /dir1 should return all its immediated subdirs only
  // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
  // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
  // listStatus.
  Path dir1 = new Path("/dir1");
  Path dir11 = new Path(dir1, "dir11");
  Path dir111 = new Path(dir11, "dir111");
  Path dir12 = new Path(dir1, "dir12");
  Path file121 = new Path(dir12, "file121");
  Path dir2 = new Path("/dir2");
  fs.mkdirs(dir111);
  fs.mkdirs(dir12);
  ContractTestUtils.touch(fs, file121);
  fs.mkdirs(dir2);

  FileStatus[] fileStatuses = o3fs.listStatus(dir1);
  assertEquals("FileStatus should return only the immediate children", 2,
      fileStatuses.length);

  // Verify that the two children of /dir1 returned by listStatus operation
  // are /dir1/dir11 and /dir1/dir12.
  String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
  String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
  assertTrue(fileStatus1.equals(dir11.toString()) ||
      fileStatus1.equals(dir12.toString()));
  assertTrue(fileStatus2.equals(dir11.toString()) ||
      fileStatus2.equals(dir12.toString()));
}
 

public void testSeekOnFileLength() throws IOException {
  Path file = new Path("/file");
  ContractTestUtils.createFile(fs, file, true, "a".getBytes());
  try (FSDataInputStream stream = fs.open(file)) {
    long fileLength = fs.getFileStatus(file).getLen();
    stream.seek(fileLength);
    assertEquals(-1, stream.read());
  }
}
 

@Test
public void testCreateDoesNotAddParentDirKeys() throws Exception {
  Path grandparent = new Path(testBucketPath,
      "testCreateDoesNotAddParentDirKeys");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);

  OzoneKeyDetails key = getKey(child, false);
  OFSPath childOFSPath = new OFSPath(child);
  Assert.assertEquals(key.getName(), childOFSPath.getKeyName());

  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // List status on the parent should show the child file
  Assert.assertEquals(
      "List status of parent should include the 1 child file",
      1L, fs.listStatus(parent).length);
  Assert.assertTrue(
      "Parent directory does not appear to be a directory",
      fs.getFileStatus(parent).isDirectory());
}
 

@Override
public void testRenameFileBeingAppended() throws Throwable {
  try {
    super.testRenameFileBeingAppended();
    fail("Expected a FileNotFoundException");
  } catch (FileNotFoundException e) {
    // downgrade
    ContractTestUtils.downgrade("Renaming an open file" +
                                "still creates the old path", e);

  }
}
 

/**
 * Tests listStatus on a path with subdirs.
 */
@Test
public void testListStatusOnSubDirs() throws Exception {
  // Create the following key structure
  //      /dir1/dir11/dir111
  //      /dir1/dir12
  //      /dir1/dir12/file121
  //      /dir2
  // ListStatus on /dir1 should return all its immediated subdirs only
  // which are /dir1/dir11 and /dir1/dir12. Super child files/dirs
  // (/dir1/dir12/file121 and /dir1/dir11/dir111) should not be returned by
  // listStatus.
  Path dir1 = new Path(testBucketPath, "dir1");
  Path dir11 = new Path(dir1, "dir11");
  Path dir111 = new Path(dir11, "dir111");
  Path dir12 = new Path(dir1, "dir12");
  Path file121 = new Path(dir12, "file121");
  Path dir2 = new Path(testBucketPath, "dir2");
  fs.mkdirs(dir111);
  fs.mkdirs(dir12);
  ContractTestUtils.touch(fs, file121);
  fs.mkdirs(dir2);

  FileStatus[] fileStatuses = ofs.listStatus(dir1);
  Assert.assertEquals(
      "FileStatus should return only the immediate children",
      2, fileStatuses.length);

  // Verify that the two children of /dir1 returned by listStatus operation
  // are /dir1/dir11 and /dir1/dir12.
  String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
  String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
  Assert.assertTrue(fileStatus1.equals(dir11.toString()) ||
      fileStatus1.equals(dir12.toString()));
  Assert.assertTrue(fileStatus2.equals(dir11.toString()) ||
      fileStatus2.equals(dir12.toString()));
}
 

@Test
public void testListStatus() throws Exception {
  Path parent = new Path(testBucketPath, "testListStatus");
  Path file1 = new Path(parent, "key1");
  Path file2 = new Path(parent, "key2");

  FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
  Assert.assertEquals("Should be empty", 0, fileStatuses.length);

  ContractTestUtils.touch(fs, file1);
  ContractTestUtils.touch(fs, file2);

  fileStatuses = ofs.listStatus(testBucketPath);
  Assert.assertEquals("Should have created parent",
      1, fileStatuses.length);
  Assert.assertEquals("Parent path doesn't match",
      fileStatuses[0].getPath().toUri().getPath(), parent.toString());

  // ListStatus on a directory should return all subdirs along with
  // files, even if there exists a file and sub-dir with the same name.
  fileStatuses = ofs.listStatus(parent);
  Assert.assertEquals(
      "FileStatus did not return all children of the directory",
      2, fileStatuses.length);

  // ListStatus should return only the immediate children of a directory.
  Path file3 = new Path(parent, "dir1/key3");
  Path file4 = new Path(parent, "dir1/key4");
  ContractTestUtils.touch(fs, file3);
  ContractTestUtils.touch(fs, file4);
  fileStatuses = ofs.listStatus(parent);
  Assert.assertEquals(
      "FileStatus did not return all children of the directory",
      3, fileStatuses.length);
}
 

@Test
public void testDeleteCreatesFakeParentDir() throws Exception {
  Path grandparent = new Path(testBucketPath,
      "testDeleteCreatesFakeParentDir");
  Path parent = new Path(grandparent, "parent");
  Path child = new Path(parent, "child");
  ContractTestUtils.touch(fs, child);

  // Verify that parent dir key does not exist
  // Creating a child should not add parent keys to the bucket
  try {
    getKey(parent, true);
  } catch (IOException ex) {
    assertKeyNotFoundException(ex);
  }

  // Delete the child key
  Assert.assertTrue(fs.delete(child, false));

  // Deleting the only child should create the parent dir key if it does
  // not exist
  OFSPath parentOFSPath = new OFSPath(parent);
  String parentKey = parentOFSPath.getKeyName() + "/";
  OzoneKeyDetails parentKeyInfo = getKey(parent, true);
  Assert.assertEquals(parentKey, parentKeyInfo.getName());

  // Recursive delete with DeleteIterator
  Assert.assertTrue(fs.delete(grandparent, true));
}
 

@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip("blobstores can't distinguish empty directories from files");
}
 

@Override
public void testRenameFileBeingAppended() throws Throwable {
  ContractTestUtils.skip("blobstores can not rename file that being appended");
}
 

@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 

@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip("blobstores can't distinguish empty directories from files");
}
 

@Override
public void testRenameFileBeingAppended() throws Throwable {
  ContractTestUtils.skip("blobstores can not rename file that being appended");
}
 

@After
public void tearDown() throws Exception {
  ContractTestUtils.rm(fs, getTestPath(), true, true);
}
 

@Test
public void testBulkRenameAndDelete() throws Throwable {
  final Path scaleTestDir = getTestPath();
  final Path srcDir = new Path(scaleTestDir, "src");
  final Path finalDir = new Path(scaleTestDir, "final");
  final long count = getOperationCount();
  ContractTestUtils.rm(fs, scaleTestDir, true, false);

  fs.mkdirs(srcDir);
  fs.mkdirs(finalDir);

  int testBufferSize = fs.getConf()
      .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
          ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
  // use Executor to speed up file creation
  ExecutorService exec = Executors.newFixedThreadPool(16);
  final ExecutorCompletionService<Boolean> completionService =
      new ExecutorCompletionService<Boolean>(exec);
  try {
    final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');

    for (int i = 0; i < count; ++i) {
      final String fileName = "foo-" + i;
      completionService.submit(new Callable<Boolean>() {
        @Override
        public Boolean call() throws IOException {
          ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
              false, data);
          return fs.exists(new Path(srcDir, fileName));
        }
      });
    }
    for (int i = 0; i < count; ++i) {
      final Future<Boolean> future = completionService.take();
      try {
        if (!future.get()) {
          LOG.warn("cannot create file");
        }
      } catch (ExecutionException e) {
        LOG.warn("Error while uploading file", e.getCause());
        throw e;
      }
    }
  } finally {
    exec.shutdown();
  }

  int nSrcFiles = fs.listStatus(srcDir).length;
  fs.rename(srcDir, finalDir);
  assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + 0));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + count / 2));
  ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
      new Path(srcDir, "foo-" + (count - 1)));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + 0));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + count/2));
  ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
      new Path(finalDir, "foo-" + (count-1)));

  ContractTestUtils.assertDeleted(fs, finalDir, true, false);
}
 

@Test
public void testMultiPartUpload() throws IOException {
  ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
      1024);
}
 

@Test
public void testRegularUpload() throws IOException {
  ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
}
 

@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 

@Override
public void testOverwriteEmptyDirectory() throws Throwable {
  ContractTestUtils.skip(
      "blobstores can't distinguish empty directories from files");
}
 

@Override
public void testOpenReadDir() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}
 

@Override
public void testOpenReadDirWithChild() throws Throwable {
  ContractTestUtils.skip("Skipping object-store quirk");
}