Java Code Examples for org.apache.hadoop.fs.StorageType#getTypesSupportingQuota()
The following examples show how to use
org.apache.hadoop.fs.StorageType#getTypesSupportingQuota() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: DirectoryWithQuotaFeature.java From hadoop with Apache License 2.0 | 6 votes |
|
private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
throws QuotaByStorageTypeExceededException {
if (!isQuotaByStorageTypeSet()) {
return;
}
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (!isQuotaByStorageTypeSet(t)) {
continue;
}
if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
typeDelta.get(t))) {
throw new QuotaByStorageTypeExceededException(
quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
}
}
}
Example 2
| Source File: PBHelper.java From big-c with Apache License 2.0 | 6 votes |
|
public static ContentSummaryProto convert(ContentSummary cs) {
if (cs == null) return null;
ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
builder.setLength(cs.getLength()).
setFileCount(cs.getFileCount()).
setDirectoryCount(cs.getDirectoryCount()).
setQuota(cs.getQuota()).
setSpaceConsumed(cs.getSpaceConsumed()).
setSpaceQuota(cs.getSpaceQuota());
if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
for (StorageType t: StorageType.getTypesSupportingQuota()) {
HdfsProtos.StorageTypeQuotaInfoProto info =
HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
setType(convertStorageType(t)).
setConsumed(cs.getTypeConsumed(t)).
setQuota(cs.getTypeQuota(t)).
build();
isb.addTypeQuotaInfo(info);
}
builder.setTypeQuotaInfos(isb);
}
return builder.build();
}
Example 3
| Source File: PBHelper.java From hadoop with Apache License 2.0 | 6 votes |
|
public static ContentSummaryProto convert(ContentSummary cs) {
if (cs == null) return null;
ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
builder.setLength(cs.getLength()).
setFileCount(cs.getFileCount()).
setDirectoryCount(cs.getDirectoryCount()).
setQuota(cs.getQuota()).
setSpaceConsumed(cs.getSpaceConsumed()).
setSpaceQuota(cs.getSpaceQuota());
if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
for (StorageType t: StorageType.getTypesSupportingQuota()) {
HdfsProtos.StorageTypeQuotaInfoProto info =
HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
setType(convertStorageType(t)).
setConsumed(cs.getTypeConsumed(t)).
setQuota(cs.getTypeQuota(t)).
build();
isb.addTypeQuotaInfo(info);
}
builder.setTypeQuotaInfos(isb);
}
return builder.build();
}
Example 4
| Source File: DirectoryWithQuotaFeature.java From big-c with Apache License 2.0 | 6 votes |
|
private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
throws QuotaByStorageTypeExceededException {
if (!isQuotaByStorageTypeSet()) {
return;
}
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (!isQuotaByStorageTypeSet(t)) {
continue;
}
if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
typeDelta.get(t))) {
throw new QuotaByStorageTypeExceededException(
quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
}
}
}
Example 5
| Source File: ContentSummary.java From big-c with Apache License 2.0 | 5 votes |
|
/** Returns true if any storage type consumption information is available*/
public boolean isTypeConsumedAvailable() {
if (typeConsumed == null) {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
if (typeConsumed[t.ordinal()] > 0) {
return true;
}
}
return false;
}
Example 6
| Source File: ContentSummary.java From big-c with Apache License 2.0 | 5 votes |
|
/** Returns true if any storage type quota has been set*/
public boolean isTypeQuotaSet() {
if (typeQuota == null) {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
if (typeQuota[t.ordinal()] > 0) {
return true;
}
}
return false;
}
Example 7
| Source File: FSImageFormatPBINode.java From big-c with Apache License 2.0 | 5 votes |
|
private static QuotaByStorageTypeFeatureProto.Builder
buildQuotaByStorageTypeEntries(QuotaCounts q) {
QuotaByStorageTypeFeatureProto.Builder b =
QuotaByStorageTypeFeatureProto.newBuilder();
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (q.getTypeSpace(t) >= 0) {
QuotaByStorageTypeEntryProto.Builder eb =
QuotaByStorageTypeEntryProto.newBuilder().
setStorageType(PBHelper.convertStorageType(t)).
setQuota(q.getTypeSpace(t));
b.addQuotas(eb);
}
}
return b;
}
Example 8
| Source File: DirectoryWithQuotaFeature.java From big-c with Apache License 2.0 | 5 votes |
|
private String typeSpaceString() {
StringBuilder sb = new StringBuilder();
for (StorageType t : StorageType.getTypesSupportingQuota()) {
sb.append("StorageType: " + t +
(quota.getTypeSpace(t) < 0? "-":
usage.getTypeSpace(t) + "/" + usage.getTypeSpace(t)));
}
return sb.toString();
}
Example 9
| Source File: ContentSummary.java From hadoop with Apache License 2.0 | 5 votes |
|
/** Returns true if any storage type consumption information is available*/
public boolean isTypeConsumedAvailable() {
if (typeConsumed == null) {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
if (typeConsumed[t.ordinal()] > 0) {
return true;
}
}
return false;
}
Example 10
| Source File: ContentSummary.java From hadoop with Apache License 2.0 | 5 votes |
|
/** Returns true if any storage type quota has been set*/
public boolean isTypeQuotaSet() {
if (typeQuota == null) {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
if (typeQuota[t.ordinal()] > 0) {
return true;
}
}
return false;
}
Example 11
| Source File: FSImageFormatPBINode.java From hadoop with Apache License 2.0 | 5 votes |
|
private static QuotaByStorageTypeFeatureProto.Builder
buildQuotaByStorageTypeEntries(QuotaCounts q) {
QuotaByStorageTypeFeatureProto.Builder b =
QuotaByStorageTypeFeatureProto.newBuilder();
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (q.getTypeSpace(t) >= 0) {
QuotaByStorageTypeEntryProto.Builder eb =
QuotaByStorageTypeEntryProto.newBuilder().
setStorageType(PBHelper.convertStorageType(t)).
setQuota(q.getTypeSpace(t));
b.addQuotas(eb);
}
}
return b;
}
Example 12
| Source File: DirectoryWithQuotaFeature.java From hadoop with Apache License 2.0 | 5 votes |
|
private String typeSpaceString() {
StringBuilder sb = new StringBuilder();
for (StorageType t : StorageType.getTypesSupportingQuota()) {
sb.append("StorageType: " + t +
(quota.getTypeSpace(t) < 0? "-":
usage.getTypeSpace(t) + "/" + usage.getTypeSpace(t)));
}
return sb.toString();
}
Example 13
| Source File: TestQuotaByStorageType.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test
public void testQuotaByStorageTypePersistenceInFsImage() throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
final Path testDir = new Path(dir, METHOD_NAME);
Path createdFile1 = new Path(testDir, "created_file1.data");
dfs.mkdirs(testDir);
// set storage policy on testDir to ONESSD
dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
// set quota by storage type on testDir
final long SSD_QUOTA = BLOCKSIZE * 4;
dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
INode testDirNode = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
// Create file of size 2 * BLOCKSIZE under testDir
long file1Len = BLOCKSIZE * 2;
int bufLen = BLOCKSIZE / 16;
DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
// Verify SSD consumed before namenode restart
long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumed);
// Restart the namenode with checkpoint to make sure fsImage is correct
dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
dfs.saveNamespace();
dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
cluster.restartNameNode(true);
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (t != StorageType.SSD) {
assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
}
}
long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumedAfterNNRestart);
}
Example 14
| Source File: FSImage.java From big-c with Apache License 2.0 | 4 votes |
|
private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) {
final long parentNamespace = counts.getNameSpace();
final long parentStoragespace = counts.getStorageSpace();
final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();
dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts);
for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId);
if (child.isDirectory()) {
updateCountForQuotaRecursively(bsps, childPolicyId,
child.asDirectory(), counts);
} else {
// file or symlink: count here to reduce recursive calls.
child.computeQuotaUsage(bsps, childPolicyId, counts, false,
Snapshot.CURRENT_STATE_ID);
}
}
if (dir.isQuotaSet()) {
// check if quota is violated. It indicates a software bug.
final QuotaCounts q = dir.getQuotaCounts();
final long namespace = counts.getNameSpace() - parentNamespace;
final long nsQuota = q.getNameSpace();
if (Quota.isViolated(nsQuota, namespace)) {
LOG.warn("Namespace quota violation in image for "
+ dir.getFullPathName()
+ " quota = " + nsQuota + " < consumed = " + namespace);
}
final long ssConsumed = counts.getStorageSpace() - parentStoragespace;
final long ssQuota = q.getStorageSpace();
if (Quota.isViolated(ssQuota, ssConsumed)) {
LOG.warn("Storagespace quota violation in image for "
+ dir.getFullPathName()
+ " quota = " + ssQuota + " < consumed = " + ssConsumed);
}
final EnumCounters<StorageType> typeSpaces =
new EnumCounters<StorageType>(StorageType.class);
for (StorageType t : StorageType.getTypesSupportingQuota()) {
final long typeSpace = counts.getTypeSpaces().get(t) -
parentTypeSpaces.get(t);
final long typeQuota = q.getTypeSpaces().get(t);
if (Quota.isViolated(typeQuota, typeSpace)) {
LOG.warn("Storage type quota violation in image for "
+ dir.getFullPathName()
+ " type = " + t.toString() + " quota = "
+ typeQuota + " < consumed " + typeSpace);
}
}
dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, ssConsumed,
typeSpaces);
}
}
Example 15
| Source File: TestQuotaByStorageType.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test
public void testQuotaByStorageTypePersistenceInEditLog() throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
final Path testDir = new Path(dir, METHOD_NAME);
Path createdFile1 = new Path(testDir, "created_file1.data");
dfs.mkdirs(testDir);
// set storage policy on testDir to ONESSD
dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
// set quota by storage type on testDir
final long SSD_QUOTA = BLOCKSIZE * 4;
dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
INode testDirNode = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
// Create file of size 2 * BLOCKSIZE under testDir
long file1Len = BLOCKSIZE * 2;
int bufLen = BLOCKSIZE / 16;
DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
// Verify SSD consumed before namenode restart
long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumed);
// Restart namenode to make sure the editlog is correct
cluster.restartNameNode(true);
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
// Verify quota is still set
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (t != StorageType.SSD) {
assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
}
}
long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumedAfterNNRestart);
}
Example 16
| Source File: TestQuotaByStorageType.java From big-c with Apache License 2.0 | 4 votes |
|
@Test
public void testQuotaByStorageTypePersistenceInEditLog() throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
final Path testDir = new Path(dir, METHOD_NAME);
Path createdFile1 = new Path(testDir, "created_file1.data");
dfs.mkdirs(testDir);
// set storage policy on testDir to ONESSD
dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
// set quota by storage type on testDir
final long SSD_QUOTA = BLOCKSIZE * 4;
dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
INode testDirNode = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
// Create file of size 2 * BLOCKSIZE under testDir
long file1Len = BLOCKSIZE * 2;
int bufLen = BLOCKSIZE / 16;
DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
// Verify SSD consumed before namenode restart
long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumed);
// Restart namenode to make sure the editlog is correct
cluster.restartNameNode(true);
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
// Verify quota is still set
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (t != StorageType.SSD) {
assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
}
}
long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumedAfterNNRestart);
}
Example 17
| Source File: TestQuotaByStorageType.java From big-c with Apache License 2.0 | 4 votes |
|
@Test
public void testQuotaByStorageTypePersistenceInFsImage() throws IOException {
final String METHOD_NAME = GenericTestUtils.getMethodName();
final Path testDir = new Path(dir, METHOD_NAME);
Path createdFile1 = new Path(testDir, "created_file1.data");
dfs.mkdirs(testDir);
// set storage policy on testDir to ONESSD
dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
// set quota by storage type on testDir
final long SSD_QUOTA = BLOCKSIZE * 4;
dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
INode testDirNode = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
// Create file of size 2 * BLOCKSIZE under testDir
long file1Len = BLOCKSIZE * 2;
int bufLen = BLOCKSIZE / 16;
DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
// Verify SSD consumed before namenode restart
long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumed);
// Restart the namenode with checkpoint to make sure fsImage is correct
dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
dfs.saveNamespace();
dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
cluster.restartNameNode(true);
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory());
assertTrue(testDirNode.isQuotaSet());
QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
for (StorageType t: StorageType.getTypesSupportingQuota()) {
if (t != StorageType.SSD) {
assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
}
}
long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
assertEquals(file1Len, ssdConsumedAfterNNRestart);
}
Example 18
| Source File: FSImage.java From hadoop with Apache License 2.0 | 4 votes |
|
private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) {
final long parentNamespace = counts.getNameSpace();
final long parentStoragespace = counts.getStorageSpace();
final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();
dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts);
for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId);
if (child.isDirectory()) {
updateCountForQuotaRecursively(bsps, childPolicyId,
child.asDirectory(), counts);
} else {
// file or symlink: count here to reduce recursive calls.
child.computeQuotaUsage(bsps, childPolicyId, counts, false,
Snapshot.CURRENT_STATE_ID);
}
}
if (dir.isQuotaSet()) {
// check if quota is violated. It indicates a software bug.
final QuotaCounts q = dir.getQuotaCounts();
final long namespace = counts.getNameSpace() - parentNamespace;
final long nsQuota = q.getNameSpace();
if (Quota.isViolated(nsQuota, namespace)) {
LOG.warn("Namespace quota violation in image for "
+ dir.getFullPathName()
+ " quota = " + nsQuota + " < consumed = " + namespace);
}
final long ssConsumed = counts.getStorageSpace() - parentStoragespace;
final long ssQuota = q.getStorageSpace();
if (Quota.isViolated(ssQuota, ssConsumed)) {
LOG.warn("Storagespace quota violation in image for "
+ dir.getFullPathName()
+ " quota = " + ssQuota + " < consumed = " + ssConsumed);
}
final EnumCounters<StorageType> typeSpaces =
new EnumCounters<StorageType>(StorageType.class);
for (StorageType t : StorageType.getTypesSupportingQuota()) {
final long typeSpace = counts.getTypeSpaces().get(t) -
parentTypeSpaces.get(t);
final long typeQuota = q.getTypeSpaces().get(t);
if (Quota.isViolated(typeQuota, typeSpace)) {
LOG.warn("Storage type quota violation in image for "
+ dir.getFullPathName()
+ " type = " + t.toString() + " quota = "
+ typeQuota + " < consumed " + typeSpace);
}
}
dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, ssConsumed,
typeSpaces);
}
}
