Java Code Examples for org.apache.hadoop.hdfs.server.namenode.INodeDirectory#computeQuotaUsage()
The following examples show how to use
org.apache.hadoop.hdfs.server.namenode.INodeDirectory#computeQuotaUsage() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestSnapshotDeletion.java From hadoop with Apache License 2.0 | 6 votes |
|
private void checkQuotaUsageComputation(final Path dirPath,
final long expectedNs, final long expectedDs) throws IOException {
INodeDirectory dirNode = getDir(fsdir, dirPath);
assertTrue(dirNode.isQuotaSet());
QuotaCounts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
q.getNameSpace());
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
q.getStorageSpace());
QuotaCounts counts = new QuotaCounts.Builder().build();
dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false);
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
counts.getNameSpace());
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
counts.getStorageSpace());
}
Example 2
| Source File: TestSnapshotDeletion.java From big-c with Apache License 2.0 | 6 votes |
|
private void checkQuotaUsageComputation(final Path dirPath,
final long expectedNs, final long expectedDs) throws IOException {
INodeDirectory dirNode = getDir(fsdir, dirPath);
assertTrue(dirNode.isQuotaSet());
QuotaCounts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
q.getNameSpace());
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
q.getStorageSpace());
QuotaCounts counts = new QuotaCounts.Builder().build();
dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false);
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
counts.getNameSpace());
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
counts.getStorageSpace());
}
Example 3
| Source File: TestRenameWithSnapshots.java From hadoop with Apache License 2.0 | 4 votes |
|
/**
* Test rename while the rename operation will exceed the quota in the dst
* tree.
*/
@Test
public void testRenameUndo_5() throws Exception {
final Path test = new Path("/test");
final Path dir1 = new Path(test, "dir1");
final Path dir2 = new Path(test, "dir2");
final Path subdir2 = new Path(dir2, "subdir2");
hdfs.mkdirs(dir1);
hdfs.mkdirs(subdir2);
final Path foo = new Path(dir1, "foo");
final Path bar = new Path(foo, "bar");
DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED);
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
// set ns quota of dir2 to 4, so the current remaining is 2 (already has
// dir2, and subdir2)
hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
final Path foo2 = new Path(subdir2, foo.getName());
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new NSQuotaExceededException("fake exception")).when(fsdir2)
.addLastINode((INodesInPath) Mockito.anyObject(),
(INode) Mockito.anyObject(), Mockito.anyBoolean());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/subdir2/foo.
// FSDirectory#verifyQuota4Rename will pass since the remaining quota is 2.
// However, the rename operation will fail since we let addLastINode throw
// NSQuotaExceededException
boolean rename = hdfs.rename(foo, foo2);
assertFalse(rename);
// check the undo
assertTrue(hdfs.exists(foo));
assertTrue(hdfs.exists(bar));
INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
.asDirectory();
List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode fooNode = childrenList.get(0);
assertTrue(fooNode.asDirectory().isWithSnapshot());
INode barNode = fsdir2.getINode4Write(bar.toString());
assertTrue(barNode.getClass() == INodeFile.class);
assertSame(fooNode, barNode.getParent());
List<DirectoryDiff> diffList = dir1Node
.getDiffs().asList();
assertEquals(1, diffList.size());
DirectoryDiff diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
// check dir2
INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable());
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
assertEquals(2, counts.getNameSpace());
assertEquals(0, counts.getStorageSpace());
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode subdir2Node = childrenList.get(0);
assertSame(dir2Node, subdir2Node.getParent());
assertSame(subdir2Node, fsdir2.getINode4Write(subdir2.toString()));
diffList = dir2Node.getDiffs().asList();
assertEquals(1, diffList.size());
diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
}
Example 4
| Source File: TestRenameWithSnapshots.java From hadoop with Apache License 2.0 | 4 votes |
|
/**
* Test the rename undo when removing dst node fails
*/
@Test
public void testRenameUndo_6() throws Exception {
final Path test = new Path("/test");
final Path dir1 = new Path(test, "dir1");
final Path dir2 = new Path(test, "dir2");
final Path sub_dir2 = new Path(dir2, "subdir");
final Path subsub_dir2 = new Path(sub_dir2, "subdir");
hdfs.mkdirs(dir1);
hdfs.mkdirs(subsub_dir2);
final Path foo = new Path(dir1, "foo");
hdfs.mkdirs(foo);
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
// set ns quota of dir2 to 4, so the current remaining is 1 (already has
// dir2, sub_dir2, and subsub_dir2)
hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new RuntimeException("fake exception")).when(fsdir2)
.removeLastINode((INodesInPath) Mockito.anyObject());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/sub_dir2/subsub_dir2.
// FSDirectory#verifyQuota4Rename will pass since foo only be counted
// as 1 in NS quota. However, the rename operation will fail when removing
// subsub_dir2.
try {
hdfs.rename(foo, subsub_dir2, Rename.OVERWRITE);
fail("Expect QuotaExceedException");
} catch (Exception e) {
String msg = "fake exception";
GenericTestUtils.assertExceptionContains(msg, e);
}
// check the undo
assertTrue(hdfs.exists(foo));
INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
.asDirectory();
List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode fooNode = childrenList.get(0);
assertTrue(fooNode.asDirectory().isWithSnapshot());
assertSame(dir1Node, fooNode.getParent());
List<DirectoryDiff> diffList = dir1Node
.getDiffs().asList();
assertEquals(1, diffList.size());
DirectoryDiff diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
// check dir2
INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable());
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
assertEquals(3, counts.getNameSpace());
assertEquals(0, counts.getStorageSpace());
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode subdir2Node = childrenList.get(0);
assertSame(dir2Node, subdir2Node.getParent());
assertSame(subdir2Node, fsdir2.getINode4Write(sub_dir2.toString()));
INode subsubdir2Node = fsdir2.getINode4Write(subsub_dir2.toString());
assertTrue(subsubdir2Node.getClass() == INodeDirectory.class);
assertSame(subdir2Node, subsubdir2Node.getParent());
diffList = ( dir2Node).getDiffs().asList();
assertEquals(1, diffList.size());
diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
}
Example 5
| Source File: TestRenameWithSnapshots.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Test rename while the rename operation will exceed the quota in the dst
* tree.
*/
@Test
public void testRenameUndo_5() throws Exception {
final Path test = new Path("/test");
final Path dir1 = new Path(test, "dir1");
final Path dir2 = new Path(test, "dir2");
final Path subdir2 = new Path(dir2, "subdir2");
hdfs.mkdirs(dir1);
hdfs.mkdirs(subdir2);
final Path foo = new Path(dir1, "foo");
final Path bar = new Path(foo, "bar");
DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED);
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
// set ns quota of dir2 to 4, so the current remaining is 2 (already has
// dir2, and subdir2)
hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
final Path foo2 = new Path(subdir2, foo.getName());
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new NSQuotaExceededException("fake exception")).when(fsdir2)
.addLastINode((INodesInPath) Mockito.anyObject(),
(INode) Mockito.anyObject(), Mockito.anyBoolean());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/subdir2/foo.
// FSDirectory#verifyQuota4Rename will pass since the remaining quota is 2.
// However, the rename operation will fail since we let addLastINode throw
// NSQuotaExceededException
boolean rename = hdfs.rename(foo, foo2);
assertFalse(rename);
// check the undo
assertTrue(hdfs.exists(foo));
assertTrue(hdfs.exists(bar));
INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
.asDirectory();
List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode fooNode = childrenList.get(0);
assertTrue(fooNode.asDirectory().isWithSnapshot());
INode barNode = fsdir2.getINode4Write(bar.toString());
assertTrue(barNode.getClass() == INodeFile.class);
assertSame(fooNode, barNode.getParent());
List<DirectoryDiff> diffList = dir1Node
.getDiffs().asList();
assertEquals(1, diffList.size());
DirectoryDiff diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
// check dir2
INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable());
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
assertEquals(2, counts.getNameSpace());
assertEquals(0, counts.getStorageSpace());
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode subdir2Node = childrenList.get(0);
assertSame(dir2Node, subdir2Node.getParent());
assertSame(subdir2Node, fsdir2.getINode4Write(subdir2.toString()));
diffList = dir2Node.getDiffs().asList();
assertEquals(1, diffList.size());
diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
}
Example 6
| Source File: TestRenameWithSnapshots.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Test the rename undo when removing dst node fails
*/
@Test
public void testRenameUndo_6() throws Exception {
final Path test = new Path("/test");
final Path dir1 = new Path(test, "dir1");
final Path dir2 = new Path(test, "dir2");
final Path sub_dir2 = new Path(dir2, "subdir");
final Path subsub_dir2 = new Path(sub_dir2, "subdir");
hdfs.mkdirs(dir1);
hdfs.mkdirs(subsub_dir2);
final Path foo = new Path(dir1, "foo");
hdfs.mkdirs(foo);
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
// set ns quota of dir2 to 4, so the current remaining is 1 (already has
// dir2, sub_dir2, and subsub_dir2)
hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new RuntimeException("fake exception")).when(fsdir2)
.removeLastINode((INodesInPath) Mockito.anyObject());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/sub_dir2/subsub_dir2.
// FSDirectory#verifyQuota4Rename will pass since foo only be counted
// as 1 in NS quota. However, the rename operation will fail when removing
// subsub_dir2.
try {
hdfs.rename(foo, subsub_dir2, Rename.OVERWRITE);
fail("Expect QuotaExceedException");
} catch (Exception e) {
String msg = "fake exception";
GenericTestUtils.assertExceptionContains(msg, e);
}
// check the undo
assertTrue(hdfs.exists(foo));
INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
.asDirectory();
List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode fooNode = childrenList.get(0);
assertTrue(fooNode.asDirectory().isWithSnapshot());
assertSame(dir1Node, fooNode.getParent());
List<DirectoryDiff> diffList = dir1Node
.getDiffs().asList();
assertEquals(1, diffList.size());
DirectoryDiff diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
// check dir2
INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable());
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
assertEquals(3, counts.getNameSpace());
assertEquals(0, counts.getStorageSpace());
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
INode subdir2Node = childrenList.get(0);
assertSame(dir2Node, subdir2Node.getParent());
assertSame(subdir2Node, fsdir2.getINode4Write(sub_dir2.toString()));
INode subsubdir2Node = fsdir2.getINode4Write(subsub_dir2.toString());
assertTrue(subsubdir2Node.getClass() == INodeDirectory.class);
assertSame(subdir2Node, subsubdir2Node.getParent());
diffList = ( dir2Node).getDiffs().asList();
assertEquals(1, diffList.size());
diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
}
