Java Code Examples for org.apache.hadoop.hdfs.server.namenode.INodeDirectory#getDirectoryWithSnapshotFeature()

The following examples show how to use org.apache.hadoop.hdfs.server.namenode.INodeDirectory#getDirectoryWithSnapshotFeature() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSImageFormatPBSnapshot.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void serializeDirDiffList(INodeDirectory dir,
    final List<INodeReference> refList, OutputStream out)
    throws IOException {
  DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
  if (sf != null) {
    List<DirectoryDiff> diffList = sf.getDiffs().asList();
    SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry
        .newBuilder().setInodeId(dir.getId()).setType(Type.DIRECTORYDIFF)
        .setNumOfDiff(diffList.size()).build();
    entry.writeDelimitedTo(out);
    for (int i = diffList.size() - 1; i >= 0; i--) { // reverse order!
      DirectoryDiff diff = diffList.get(i);
      SnapshotDiffSection.DirectoryDiff.Builder db = SnapshotDiffSection.
          DirectoryDiff.newBuilder().setSnapshotId(diff.getSnapshotId())
                       .setChildrenSize(diff.getChildrenSize())
                       .setIsSnapshotRoot(diff.isSnapshotRoot());
      INodeDirectoryAttributes copy = diff.snapshotINode;
      if (!diff.isSnapshotRoot() && copy != null) {
        db.setName(ByteString.copyFrom(copy.getLocalNameBytes()))
            .setSnapshotCopy(
                buildINodeDirectory(copy, parent.getSaverContext()));
      }
      // process created list and deleted list
      List<INode> created = diff.getChildrenDiff()
          .getList(ListType.CREATED);
      db.setCreatedListSize(created.size());
      List<INode> deleted = diff.getChildrenDiff().getList(ListType.DELETED);
      for (INode d : deleted) {
        if (d.isReference()) {
          refList.add(d.asReference());
          db.addDeletedINodeRef(refList.size() - 1);
        } else {
          db.addDeletedINode(d.getId());
        }
      }
      db.build().writeDelimitedTo(out);
      saveCreatedList(created, out);
    }
  }
}
Example 2
Source File: FSImageFormatPBSnapshot.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void serializeDirDiffList(INodeDirectory dir,
    final List<INodeReference> refList, OutputStream out)
    throws IOException {
  DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
  if (sf != null) {
    List<DirectoryDiff> diffList = sf.getDiffs().asList();
    SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry
        .newBuilder().setInodeId(dir.getId()).setType(Type.DIRECTORYDIFF)
        .setNumOfDiff(diffList.size()).build();
    entry.writeDelimitedTo(out);
    for (int i = diffList.size() - 1; i >= 0; i--) { // reverse order!
      DirectoryDiff diff = diffList.get(i);
      SnapshotDiffSection.DirectoryDiff.Builder db = SnapshotDiffSection.
          DirectoryDiff.newBuilder().setSnapshotId(diff.getSnapshotId())
                       .setChildrenSize(diff.getChildrenSize())
                       .setIsSnapshotRoot(diff.isSnapshotRoot());
      INodeDirectoryAttributes copy = diff.snapshotINode;
      if (!diff.isSnapshotRoot() && copy != null) {
        db.setName(ByteString.copyFrom(copy.getLocalNameBytes()))
            .setSnapshotCopy(
                buildINodeDirectory(copy, parent.getSaverContext()));
      }
      // process created list and deleted list
      List<INode> created = diff.getChildrenDiff()
          .getList(ListType.CREATED);
      db.setCreatedListSize(created.size());
      List<INode> deleted = diff.getChildrenDiff().getList(ListType.DELETED);
      for (INode d : deleted) {
        if (d.isReference()) {
          refList.add(d.asReference());
          db.addDeletedINodeRef(refList.size() - 1);
        } else {
          db.addDeletedINode(d.getId());
        }
      }
      db.build().writeDelimitedTo(out);
      saveCreatedList(created, out);
    }
  }
}
Example 3
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Destroy a subtree under a DstReference node.
 */
public static void destroyDstSubtree(
    final BlockStoragePolicySuite bsps, INode inode, final int snapshot,
    final int prior, final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) throws QuotaExceededException {
  Preconditions.checkArgument(prior != Snapshot.NO_SNAPSHOT_ID);
  if (inode.isReference()) {
    if (inode instanceof INodeReference.WithName
        && snapshot != Snapshot.CURRENT_STATE_ID) {
      // this inode has been renamed before the deletion of the DstReference
      // subtree
      inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
    } else { 
      // for DstReference node, continue this process to its subtree
      destroyDstSubtree(bsps, inode.asReference().getReferredINode(), snapshot,
          prior, collectedBlocks, removedINodes);
    }
  } else if (inode.isFile()) {
    inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
  } else if (inode.isDirectory()) {
    Map<INode, INode> excludedNodes = null;
    INodeDirectory dir = inode.asDirectory();
    DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
    if (sf != null) {
      DirectoryDiffList diffList = sf.getDiffs();
      DirectoryDiff priorDiff = diffList.getDiffById(prior);
      if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
        List<INode> dList = priorDiff.diff.getList(ListType.DELETED);
        excludedNodes = cloneDiffList(dList);
      }
      
      if (snapshot != Snapshot.CURRENT_STATE_ID) {
        diffList.deleteSnapshotDiff(bsps, snapshot, prior, dir, collectedBlocks,
            removedINodes);
      }
      priorDiff = diffList.getDiffById(prior);
      if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
        priorDiff.diff.destroyCreatedList(bsps, dir, collectedBlocks,
            removedINodes);
      }
    }
    for (INode child : inode.asDirectory().getChildrenList(prior)) {
      if (excludedNodes != null && excludedNodes.containsKey(child)) {
        continue;
      }
      destroyDstSubtree(bsps, child, snapshot, prior, collectedBlocks,
          removedINodes);
    }
  }
}
Example 4
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Clean an inode while we move it from the deleted list of post to the
 * deleted list of prior.
 * @param bsps The block storage policy suite.
 * @param inode The inode to clean.
 * @param post The post snapshot.
 * @param prior The id of the prior snapshot.
 * @param collectedBlocks Used to collect blocks for later deletion.
 * @return Quota usage update.
 */
private static QuotaCounts cleanDeletedINode(
    final BlockStoragePolicySuite bsps, INode inode,
    final int post, final int prior,
    final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  QuotaCounts counts = new QuotaCounts.Builder().build();
  Deque<INode> queue = new ArrayDeque<INode>();
  queue.addLast(inode);
  while (!queue.isEmpty()) {
    INode topNode = queue.pollFirst();
    if (topNode instanceof INodeReference.WithName) {
      INodeReference.WithName wn = (INodeReference.WithName) topNode;
      if (wn.getLastSnapshotId() >= post) {
        INodeReference.WithCount wc =
            (INodeReference.WithCount) wn.getReferredINode();
        if (wc.getLastWithName() == wn && wc.getParentReference() == null) {
          // this wn is the last wn inside of the wc, also the dstRef node has
          // been deleted. In this case, we should treat the referred file/dir
          // as normal case
          queue.add(wc.getReferredINode());
        } else {
          wn.cleanSubtree(bsps, post, prior, collectedBlocks, removedINodes);
        }
      }
      // For DstReference node, since the node is not in the created list of
      // prior, we should treat it as regular file/dir
    } else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) {
      INodeFile file = topNode.asFile();
      counts.add(file.getDiffs().deleteSnapshotDiff(bsps, post, prior, file,
          collectedBlocks, removedINodes));
    } else if (topNode.isDirectory()) {
      INodeDirectory dir = topNode.asDirectory();
      ChildrenDiff priorChildrenDiff = null;
      DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
      if (sf != null) {
        // delete files/dirs created after prior. Note that these
        // files/dirs, along with inode, were deleted right after post.
        DirectoryDiff priorDiff = sf.getDiffs().getDiffById(prior);
        if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
          priorChildrenDiff = priorDiff.getChildrenDiff();
          counts.add(priorChildrenDiff.destroyCreatedList(bsps, dir,
              collectedBlocks, removedINodes));
        }
      }
      
      for (INode child : dir.getChildrenList(prior)) {
        if (priorChildrenDiff != null
            && priorChildrenDiff.search(ListType.DELETED,
                child.getLocalNameBytes()) != null) {
          continue;
        }
        queue.addLast(child);
      }
    }
  }
  return counts;
}
Example 5
Source File: DirectoryWithSnapshotFeature.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Destroy a subtree under a DstReference node.
 */
public static void destroyDstSubtree(
    final BlockStoragePolicySuite bsps, INode inode, final int snapshot,
    final int prior, final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) throws QuotaExceededException {
  Preconditions.checkArgument(prior != Snapshot.NO_SNAPSHOT_ID);
  if (inode.isReference()) {
    if (inode instanceof INodeReference.WithName
        && snapshot != Snapshot.CURRENT_STATE_ID) {
      // this inode has been renamed before the deletion of the DstReference
      // subtree
      inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
    } else { 
      // for DstReference node, continue this process to its subtree
      destroyDstSubtree(bsps, inode.asReference().getReferredINode(), snapshot,
          prior, collectedBlocks, removedINodes);
    }
  } else if (inode.isFile()) {
    inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
  } else if (inode.isDirectory()) {
    Map<INode, INode> excludedNodes = null;
    INodeDirectory dir = inode.asDirectory();
    DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
    if (sf != null) {
      DirectoryDiffList diffList = sf.getDiffs();
      DirectoryDiff priorDiff = diffList.getDiffById(prior);
      if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
        List<INode> dList = priorDiff.diff.getList(ListType.DELETED);
        excludedNodes = cloneDiffList(dList);
      }
      
      if (snapshot != Snapshot.CURRENT_STATE_ID) {
        diffList.deleteSnapshotDiff(bsps, snapshot, prior, dir, collectedBlocks,
            removedINodes);
      }
      priorDiff = diffList.getDiffById(prior);
      if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
        priorDiff.diff.destroyCreatedList(bsps, dir, collectedBlocks,
            removedINodes);
      }
    }
    for (INode child : inode.asDirectory().getChildrenList(prior)) {
      if (excludedNodes != null && excludedNodes.containsKey(child)) {
        continue;
      }
      destroyDstSubtree(bsps, child, snapshot, prior, collectedBlocks,
          removedINodes);
    }
  }
}
Example 6
Source File: DirectoryWithSnapshotFeature.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Clean an inode while we move it from the deleted list of post to the
 * deleted list of prior.
 * @param bsps The block storage policy suite.
 * @param inode The inode to clean.
 * @param post The post snapshot.
 * @param prior The id of the prior snapshot.
 * @param collectedBlocks Used to collect blocks for later deletion.
 * @return Quota usage update.
 */
private static QuotaCounts cleanDeletedINode(
    final BlockStoragePolicySuite bsps, INode inode,
    final int post, final int prior,
    final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  QuotaCounts counts = new QuotaCounts.Builder().build();
  Deque<INode> queue = new ArrayDeque<INode>();
  queue.addLast(inode);
  while (!queue.isEmpty()) {
    INode topNode = queue.pollFirst();
    if (topNode instanceof INodeReference.WithName) {
      INodeReference.WithName wn = (INodeReference.WithName) topNode;
      if (wn.getLastSnapshotId() >= post) {
        wn.cleanSubtree(bsps, post, prior, collectedBlocks, removedINodes);
      }
      // For DstReference node, since the node is not in the created list of
      // prior, we should treat it as regular file/dir
    } else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) {
      INodeFile file = topNode.asFile();
      counts.add(file.getDiffs().deleteSnapshotDiff(bsps, post, prior, file,
          collectedBlocks, removedINodes));
    } else if (topNode.isDirectory()) {
      INodeDirectory dir = topNode.asDirectory();
      ChildrenDiff priorChildrenDiff = null;
      DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
      if (sf != null) {
        // delete files/dirs created after prior. Note that these
        // files/dirs, along with inode, were deleted right after post.
        DirectoryDiff priorDiff = sf.getDiffs().getDiffById(prior);
        if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
          priorChildrenDiff = priorDiff.getChildrenDiff();
          counts.add(priorChildrenDiff.destroyCreatedList(bsps, dir,
              collectedBlocks, removedINodes));
        }
      }
      
      for (INode child : dir.getChildrenList(prior)) {
        if (priorChildrenDiff != null
            && priorChildrenDiff.search(ListType.DELETED,
                child.getLocalNameBytes()) != null) {
          continue;
        }
        queue.addLast(child);
      }
    }
  }
  return counts;
}