org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem Java Examples

The following examples show how to use org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSwiftFileSystemPartitionedUploads.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up 
/**
 * Test writes partitioned file writing that path is qualified.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testQualifiedPath() throws Throwable {
  final Path path = path("/test/qualifiedPath");
  int len = PART_SIZE_BYTES * 4;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
Example #2
Source File: TestSwiftFileSystemPartitionedUploads.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up 
/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = getStore().listSegments(fs.getFileStatus(path), true);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
Example #3
Source File: TestSwiftFileSystemBasicOps.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up 
private void deleteR(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, true)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
Example #4
Source File: TestSwiftFileSystemBasicOps.java    From sahara-extra with Apache License 2.0 5 votes vote down vote up 
private void delete(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, false)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
Example #5
Source File: TestSwiftFileSystemBasicOps.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void delete(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, false)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
Example #6
Source File: TestSwiftFileSystemBasicOps.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void deleteR(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, true)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
Example #7
Source File: TestSwiftFileSystemPartitionedUploads.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = fs.listStatus(path);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
Example #8
Source File: TestSwiftFileSystemPartitionedUploads.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
public void testManyPartitionedFile() throws Throwable {
  final Path path = new Path("/test/testManyPartitionedFile");

  int len = PART_SIZE_BYTES * 15;
  final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
  FSDataOutputStream out = fs.create(path,
                                     false,
                                     getBufferSize(),
                                     (short) 1,
                                     BLOCK_SIZE);

  out.write(src, 0, src.length);
  int expected =
    getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
  out.close();
  assertPartitionsWritten("write completed", out, expected);
  assertEquals("too few bytes written", len,
               SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded", len,
               SwiftNativeFileSystem.getBytesUploaded(out));
  //now we verify that the data comes back. If it
  //doesn't, it means that the ordering of the partitions
  //isn't right
  byte[] dest = readDataset(fs, path, len);
  //compare data
  SwiftTestUtils.compareByteArrays(src, dest, len);
  //finally, check the data
  FileStatus[] stats = fs.listStatus(path);
  assertEquals("wrong entry count in "
               + SwiftTestUtils.dumpStats(path.toString(), stats),
               expected, stats.length);
}
Example #9
Source File: TestSwiftFileSystemBasicOps.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void deleteR(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, true)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
Example #10
Source File: TestSwiftFileSystemBasicOps.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void delete(SwiftNativeFileSystem fs, Path path) {
  try {
    if (!fs.delete(path, false)) {
      LOG.warn("Failed to delete " + path);
    }
  } catch (IOException e) {
    LOG.warn("deleting " + path, e);
  }
}
Example #11
Source File: TestSwiftFileSystemContract.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
          new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
Example #12
Source File: SwiftFileSystemBaseTest.java    From sahara-extra with Apache License 2.0 4 votes vote down vote up 
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
    new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
Example #13
Source File: TestSwiftFileSystemExtendedContract.java    From sahara-extra with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testGetSchemeImplemented() throws Throwable {
  String scheme = fs.getScheme();
  assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
}
Example #14
Source File: SwiftContract.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public String getScheme() {
  return SwiftNativeFileSystem.SWIFT;
}
Example #15
Source File: SwiftFileSystemBaseTest.java    From big-c with Apache License 2.0 4 votes vote down vote up 
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
    new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
Example #16
Source File: TestSwiftFileSystemContract.java    From big-c with Apache License 2.0 4 votes vote down vote up 
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
          new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
Example #17
Source File: TestSwiftFileSystemExtendedContract.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testGetSchemeImplemented() throws Throwable {
  String scheme = fs.getScheme();
  assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
}
Example #18
Source File: SwiftContract.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public String getScheme() {
  return SwiftNativeFileSystem.SWIFT;
}
Example #19
Source File: SwiftFileSystemBaseTest.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
protected SwiftNativeFileSystem createSwiftFS() throws IOException {
  SwiftNativeFileSystem swiftNativeFileSystem =
    new SwiftNativeFileSystem();
  return swiftNativeFileSystem;
}
Example #20
Source File: TestSwiftFileSystemExtendedContract.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = SWIFT_TEST_TIMEOUT)
public void testGetSchemeImplemented() throws Throwable {
  String scheme = fs.getScheme();
  assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
}
Example #21
Source File: SwiftFileSystemBaseTest.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the number of partitions written from the Swift Native FS APIs
 * @param out output stream
 * @return the number of partitioned files written by the stream
 */
protected int getPartitionsWritten(FSDataOutputStream out) {
  return SwiftNativeFileSystem.getPartitionsWritten(out);
}
Example #22
Source File: SwiftFileSystemBaseTest.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the filesystem
 * @return the current FS
 */
public SwiftNativeFileSystem getFs() {
  return fs;
}
Example #23
Source File: SwiftFileSystemBaseTest.java    From sahara-extra with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the filesystem
 * @return the current FS
 */
public SwiftNativeFileSystem getFs() {
  return fs;
}
Example #24
Source File: SwiftFileSystemBaseTest.java    From sahara-extra with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the number of partitions written from the Swift Native FS APIs
 * @param out output stream
 * @return the number of partitioned files written by the stream
 */
protected int getPartitionsWritten(FSDataOutputStream out) {
  return SwiftNativeFileSystem.getPartitionsWritten(out);
}
Example #25
Source File: SwiftFileSystemBaseTest.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the number of partitions written from the Swift Native FS APIs
 * @param out output stream
 * @return the number of partitioned files written by the stream
 */
protected int getPartitionsWritten(FSDataOutputStream out) {
  return SwiftNativeFileSystem.getPartitionsWritten(out);
}
Example #26
Source File: SwiftFileSystemBaseTest.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the filesystem
 * @return the current FS
 */
public SwiftNativeFileSystem getFs() {
  return fs;
}