org.apache.hadoop.fs.swift.util.SwiftTestUtils Java Examples

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenamedConsistence() throws IOException {
  assumeRenameSupported();
  describe("verify that overwriting a file with new data doesn't impact" +
          " the existing content");

  final Path filePath = new Path("/test/home/user/documents/file.txt");
  final Path newFilePath = new Path("/test/home/user/files/file.txt");
  mkdirs(newFilePath.getParent());
  int len = 1024;
  byte[] dataset = dataset(len, 'A', 26);
  byte[] dataset2 = dataset(len, 'a', 26);
  writeDataset(fs, filePath, dataset, len, len, false);
  rename(filePath, newFilePath, true, false, true);
  SwiftTestUtils.writeAndRead(fs, filePath, dataset2, len, len, false, true);
  byte[] dest = readDataset(fs, newFilePath, len);
  compareByteArrays(dataset, dest, len);
  String reread = readBytesToString(fs, newFilePath, 20);
}
 

/**
 * test that a dir off root has a listStatus() call that
 * works as expected. and that when a child is added. it changes
 *
 * @throws Exception on failures
 */
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDirectoriesOffRootHaveMatchingFileStatus() throws Exception {
  Path test = path("/test");
  fs.delete(test, true);
  mkdirs(test);
  assertExists("created test directory", test);
  FileStatus[] statuses = fs.listStatus(test);
  String statusString = statusToString(test.toString(), statuses);
  assertEquals("Wrong number of elements in file status " + statusString, 0,
               statuses.length);

  Path src = path("/test/file");

  //create a zero byte file
  SwiftTestUtils.touch(fs, src);
  //stat it
  statuses = fs.listStatus(test);
  statusString = statusToString(test.toString(), statuses);
  assertEquals("Wrong number of elements in file status " + statusString, 1,
               statuses.length);
  SwiftFileStatus stat = (SwiftFileStatus) statuses[0];
  assertTrue("isDir(): Not a directory: " + stat, stat.isDir());
  extraStatusAssertions(stat);
}
 

@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeletePartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeletePartitionedFile");

  SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekBigFile() throws Throwable {
  Path testSeekFile = new Path(testPath, "bigseekfile.txt");
  byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
  createFile(testSeekFile, block);
  instream = fs.open(testSeekFile);
  assertEquals(0, instream.getPos());
  //expect that seek to 0 works
  instream.seek(0);
  int result = instream.read();
  assertEquals(0, result);
  assertEquals(1, instream.read());
  assertEquals(2, instream.read());

  //do seek 32KB ahead
  instream.seek(32768);
  assertEquals("@32768", block[32768], (byte) instream.read());
  instream.seek(40000);
  assertEquals("@40000", block[40000], (byte) instream.read());
  instream.seek(8191);
  assertEquals("@8191", block[8191], (byte) instream.read());
  instream.seek(0);
  assertEquals("@0", 0, (byte) instream.read());
}
 

@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeletePartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeletePartitionedFile");

  SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testRenamedConsistence() throws IOException {
  assumeRenameSupported();
  describe("verify that overwriting a file with new data doesn't impact" +
          " the existing content");

  final Path filePath = new Path("/test/home/user/documents/file.txt");
  final Path newFilePath = new Path("/test/home/user/files/file.txt");
  mkdirs(newFilePath.getParent());
  int len = 1024;
  byte[] dataset = dataset(len, 'A', 26);
  byte[] dataset2 = dataset(len, 'a', 26);
  writeDataset(fs, filePath, dataset, len, len, false);
  rename(filePath, newFilePath, true, false, true);
  SwiftTestUtils.writeAndRead(fs, filePath, dataset2, len, len, false, true);
  byte[] dest = readDataset(fs, newFilePath, len);
  compareByteArrays(dataset, dest, len);
  String reread = readBytesToString(fs, newFilePath, 20);
}
 

@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeleteSmallPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeleteSmallPartitionedFile");

  final int len1 = 1024;
  final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
  SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
}
 

@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeleteSmallPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeleteSmallPartitionedFile");

  final int len1 = 1024;
  final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
  SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
}
 

@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testDeletePartitionedFile() throws Throwable {
  final Path path = new Path("/test/testDeletePartitionedFile");

  SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
  assertExists("Exists", path);

  Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
  Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
  String ls = SwiftTestUtils.ls(fs, path);
  assertExists("Partition 0001 Exists in " + ls, part_0001);
  assertExists("Partition 0002 Exists in " + ls, part_0001);
  fs.delete(path, false);
  assertPathDoesNotExist("deleted file still there", path);
  ls = SwiftTestUtils.ls(fs, path);
  assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
  assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekBigFile() throws Throwable {
  Path testSeekFile = new Path(testPath, "bigseekfile.txt");
  byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
  createFile(testSeekFile, block);
  instream = fs.open(testSeekFile);
  assertEquals(0, instream.getPos());
  //expect that seek to 0 works
  instream.seek(0);
  int result = instream.read();
  assertEquals(0, result);
  assertEquals(1, instream.read());
  assertEquals(2, instream.read());

  //do seek 32KB ahead
  instream.seek(32768);
  assertEquals("@32768", block[32768], (byte) instream.read());
  instream.seek(40000);
  assertEquals("@40000", block[40000], (byte) instream.read());
  instream.seek(8191);
  assertEquals("@8191", block[8191], (byte) instream.read());
  instream.seek(0);
  assertEquals("@0", 0, (byte) instream.read());
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
  Path testSeekFile = new Path(testPath, "bigseekfile.txt");
  byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
  createFile(testSeekFile, block);
  instream = fs.open(testSeekFile);
  instream.seek(39999);
  assertTrue(-1 != instream.read());
  assertEquals (40000, instream.getPos());

  byte[] readBuffer = new byte[256];
  instream.read(128, readBuffer, 0, readBuffer.length);
  //have gone back
  assertEquals(40000, instream.getPos());
  //content is the same too
  assertEquals("@40000", block[40000], (byte) instream.read());
  //now verify the picked up data
  for (int i = 0; i < 256; i++) {
    assertEquals("@" + i, block[i + 128], readBuffer[i]);
  }
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testSeekBigFile() throws Throwable {
  Path testSeekFile = new Path(testPath, "bigseekfile.txt");
  byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
  createFile(testSeekFile, block);
  instream = fs.open(testSeekFile);
  assertEquals(0, instream.getPos());
  //expect that seek to 0 works
  instream.seek(0);
  int result = instream.read();
  assertEquals(0, result);
  assertEquals(1, instream.read());
  assertEquals(2, instream.read());

  //do seek 32KB ahead
  instream.seek(32768);
  assertEquals("@32768", block[32768], (byte) instream.read());
  instream.seek(40000);
  assertEquals("@40000", block[40000], (byte) instream.read());
  instream.seek(8191);
  assertEquals("@8191", block[8191], (byte) instream.read());
  instream.seek(0);
  assertEquals("@0", 0, (byte) instream.read());
}
 

/**
 * Test that when a partitioned file is overwritten by a smaller one,
 * all the old partitioned files go away
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testOverwritePartitionedFile() throws Throwable {
  final Path path = new Path("/test/testOverwritePartitionedFile");

  final int len1 = 8192;
  final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     1024);
  out.write(src1, 0, len1);
  out.close();
  long expected = getExpectedPartitionsWritten(len1,
                                               PART_SIZE_BYTES,
                                               false);
  assertPartitionsWritten("initial upload", out, expected);
  assertExists("Exists", path);
  FileStatus status = fs.getFileStatus(path);
  assertEquals("Length", len1, status.getLen());
  //now write a shorter file with a different dataset
  final int len2 = 4095;
  final byte[] src2 = SwiftTestUtils.dataset(len2, 'a', 'z');
  out = fs.create(path,
                  true,
                  getBufferSize(),
                  (short) 1,
                  1024);
  out.write(src2, 0, len2);
  out.close();
  status = fs.getFileStatus(path);
  assertEquals("Length", len2, status.getLen());
  byte[] dest = readDataset(fs, path, len2);
  //compare data
  SwiftTestUtils.compareByteArrays(src2, dest, len2);
}
 

@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testRenamePartitionedFile() throws Throwable {
  Path src = new Path("/test/testRenamePartitionedFileSrc");

  int len = data.length;
  SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
  assertExists("Exists", src);

  String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
  Path srcPart = new Path(src, partOneName);
  Path dest = new Path("/test/testRenamePartitionedFileDest");
  Path destPart = new Path(src, partOneName);
  assertExists("Partition Exists", srcPart);
  fs.rename(src, dest);
  assertPathExists(fs, "dest file missing", dest);
  FileStatus status = fs.getFileStatus(dest);
  assertEquals("Length of renamed file is wrong", len, status.getLen());
  byte[] destData = readDataset(fs, dest, len);
  //compare data
  SwiftTestUtils.compareByteArrays(data, destData, len);
  String srcLs = SwiftTestUtils.ls(fs, src);
  String destLs = SwiftTestUtils.ls(fs, dest);

  assertPathDoesNotExist("deleted file still found in " + srcLs, src);

  assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
}
 

/**
 * Setup creates dirs under test/hadoop
 *
 * @throws Exception
 */
@Override
public void setUp() throws Exception {
  super.setUp();
  byte[] block = SwiftTestUtils.dataset(SEEK_FILE_LEN, 0, 255);

  //delete the test directory
  testPath = path("/test");
  readFile = new Path(testPath, "TestReadPastBuffer.txt");
  createFile(readFile, block);
}
 

@Before
public void setup() throws IOException {
  conf = new Configuration();
  runTests = SwiftTestUtils.hasServiceURI(conf);
  if (runTests) {
    serviceURI = SwiftTestUtils.getServiceURI(conf);
  }
}
 

/**
 * Asserts that a zero byte file has a status of file and not
 * file or symlink
 *
 * @throws Exception on failures
 */
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testZeroByteFilesAreFiles() throws Exception {
  Path src = path("/test/testZeroByteFilesAreFiles");
  //create a zero byte file
  SwiftTestUtils.touch(fs, src);
  SwiftTestUtils.assertIsFile(fs, src);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testBlocksizeNonZeroForFile() throws Throwable {
  Path smallfile = new Path("/test/smallfile");
  SwiftTestUtils.writeTextFile(fs, smallfile, "blocksize", true);
  createFile(smallfile);
  FileStatus status = getFs().getFileStatus(smallfile);
  assertTrue("Zero blocksize in " + status,
             status.getBlockSize() != 0L);
  assertTrue("Zero replication in " + status,
             status.getReplication() != 0L);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateRootDirectory() throws Throwable {
  describe("verify that locating the root directory is an error");
  FileStatus status = fs.getFileStatus(path("/"));
  SwiftTestUtils.assertIsDirectory(status);
  BlockLocation[] locations;
  locations = getFs().getFileBlockLocations(status,
                                            0,
                                            1);
  assertEmptyBlockLocations(locations);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testLocateDirectory() throws Throwable {
  describe("verify that locating a directory is an error");
  createFile(path("/test/filename"));
  FileStatus status = fs.getFileStatus(path("/test"));
  LOG.info("Filesystem is " + fs + "; target is " + status);
  SwiftTestUtils.assertIsDirectory(status);
  BlockLocation[] locations;
  locations = getFs().getFileBlockLocations(status,
                                            0,
                                            1);
  assertEmptyBlockLocations(locations);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testBlocksizeNonZeroForFile() throws Throwable {
  Path smallfile = new Path("/test/smallfile");
  SwiftTestUtils.writeTextFile(fs, smallfile, "blocksize", true);
  createFile(smallfile);
  FileStatus status = getFs().getFileStatus(smallfile);
  assertTrue("Zero blocksize in " + status,
             status.getBlockSize() != 0L);
  assertTrue("Zero replication in " + status,
             status.getReplication() != 0L);
}
 

@Override
@Before
public void setUp() throws Exception {
  Configuration conf = new Configuration();
  //small blocksize for faster remote tests
  conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, 2);
  URI serviceURI = SwiftTestUtils.getServiceURI(conf);
  fSys = FileSystem.get(serviceURI, conf);
  super.setUp();
}
 

/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = fs.listStatus(path);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testFileStatus() throws Throwable {
  Path path = new Path("/test/FileStatus");
  try {
    String text = "Testing File Status "
            + System.currentTimeMillis();
    writeTextFile(fs, path, text, false);
    SwiftTestUtils.assertIsFile(fs, path);
  } finally {
    delete(fs, path);
  }
}
 

@Override
public void testWriteReadAndDeleteEmptyFile() throws Exception {
  try {
    super.testWriteReadAndDeleteEmptyFile();
  } catch (AssertionFailedError e) {
    SwiftTestUtils.downgrade("empty files get mistaken for directories", e);
  }
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteEmptyFileTwice() throws IOException {
  final Path file = new Path("/test/testDeleteEmptyFileTwice");
  createEmptyFile(file);
  assertDeleted(file, true);
  SwiftTestUtils.noteAction("multiple creates, and deletes");
  assertFalse("Delete returned true", fs.delete(file, false));
  createEmptyFile(file);
  assertDeleted(file, true);
  assertFalse("Delete returned true", fs.delete(file, false));
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteEmptyFile() throws IOException {
  final Path file = new Path("/test/testDeleteEmptyFile");
  createEmptyFile(file);
  SwiftTestUtils.noteAction("about to delete");
  assertDeleted(file, true);
}
 

@Before
public void setup() throws IOException {
  conf = new Configuration();
  runTests = SwiftTestUtils.hasServiceURI(conf);
  if (runTests) {
    serviceURI = SwiftTestUtils.getServiceURI(conf);
  }
}
 

/**
 * Setup creates dirs under test/hadoop
 *
 * @throws Exception
 */
@Override
public void setUp() throws Exception {
  super.setUp();
  byte[] block = SwiftTestUtils.dataset(SEEK_FILE_LEN, 0, 255);

  //delete the test directory
  testPath = path("/test");
  readFile = new Path(testPath, "TestReadPastBuffer.txt");
  createFile(readFile, block);
}
 

@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testDeleteEmptyFile() throws IOException {
  final Path file = new Path("/test/testDeleteFile");
  createEmptyFile(file);
  SwiftTestUtils.noteAction("about to delete");
  assertDeleted(file, true);
}