org.apache.hadoop.hdfs.server.namenode.FsImageProto Java Examples

/**
 * Scan the INodeDirectory section to construct the namespace.
 */
private void buildNamespace(InputStream in, List<Long> refIdList) throws IOException {
  int count = 0;
  while (true) {
    FsImageProto.INodeDirectorySection.DirEntry e =
        FsImageProto.INodeDirectorySection.DirEntry.parseDelimitedFrom(in);
    if (e == null) {
      break;
    }
    count++;
    if (LOG.isDebugEnabled() && count % 10000 == 0) {
      LOG.debug("Scanned {} directories.", count);
    }
    long parentId = e.getParent();
    for (int i = 0; i < e.getChildrenCount(); i++) {
      long childId = e.getChildren(i);
      metadataMap.putDirChild(parentId, childId);
    }
    for (int i = e.getChildrenCount();
         i < e.getChildrenCount() + e.getRefChildrenCount(); i++) {
      int refId = e.getRefChildren(i - e.getChildrenCount());
      metadataMap.putDirChild(parentId, refIdList.get(refId));
    }
  }
  LOG.info("Scanned {} INode directories to build namespace.", count);
}
 

private FsImageProto.INodeSection.INode fromINodeId(final long id)
        throws IOException {
  int l = 0, r = inodes.length;
  while (l < r) {
    int mid = l + (r - l) / 2;
    FsImageProto.INodeSection.INode n = FsImageProto.INodeSection.INode
            .parseFrom(inodes[mid]);
    long nid = n.getId();
    if (id > nid) {
      l = mid + 1;
    } else if (id < nid) {
      r = mid;
    } else {
      return n;
    }
  }
  return null;
}
 

private PermissionStatus getPermissionStatus(String path) throws IOException {
  long id = lookup(path);
  FsImageProto.INodeSection.INode inode = fromINodeId(id);
  switch (inode.getType()) {
    case FILE: {
      FsImageProto.INodeSection.INodeFile f = inode.getFile();
      return FSImageFormatPBINode.Loader.loadPermission(
          f.getPermission(), stringTable);
    }
    case DIRECTORY: {
      FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory();
      return FSImageFormatPBINode.Loader.loadPermission(
          d.getPermission(), stringTable);
    }
    case SYMLINK: {
      FsImageProto.INodeSection.INodeSymlink s = inode.getSymlink();
      return FSImageFormatPBINode.Loader.loadPermission(
          s.getPermission(), stringTable);
    }
    default: {
      return null;
    }
  }
}
 

private List<AclEntry> getAclEntryList(String path) throws IOException {
  long id = lookup(path);
  FsImageProto.INodeSection.INode inode = fromINodeId(id);
  switch (inode.getType()) {
    case FILE: {
      FsImageProto.INodeSection.INodeFile f = inode.getFile();
      return FSImageFormatPBINode.Loader.loadAclEntries(
          f.getAcl(), stringTable);
    }
    case DIRECTORY: {
      FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory();
      return FSImageFormatPBINode.Loader.loadAclEntries(
          d.getAcl(), stringTable);
    }
    default: {
      return new ArrayList<AclEntry>();
    }
  }
}
 

private List<Map<String, Object>> getFileStatusList(String path)
        throws IOException {
  List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
  long id = lookup(path);
  FsImageProto.INodeSection.INode inode = fromINodeId(id);
  if (inode.getType() == FsImageProto.INodeSection.INode.Type.DIRECTORY) {
    if (!dirmap.containsKey(id)) {
      // if the directory is empty, return empty list
      return list;
    }
    long[] children = dirmap.get(id);
    for (long cid : children) {
      list.add(getFileStatus(fromINodeId(cid), true));
    }
  } else {
    list.add(getFileStatus(inode, false));
  }
  return list;
}
 

private static byte[][] loadINodeSection(InputStream in)
        throws IOException {
  FsImageProto.INodeSection s = FsImageProto.INodeSection
      .parseDelimitedFrom(in);
  LOG.info("Loading " + s.getNumInodes() + " inodes.");
  final byte[][] inodes = new byte[(int) s.getNumInodes()][];

  for (int i = 0; i < s.getNumInodes(); ++i) {
    int size = CodedInputStream.readRawVarint32(in.read(), in);
    byte[] bytes = new byte[size];
    IOUtils.readFully(in, bytes, 0, size);
    inodes[i] = bytes;
  }
  LOG.debug("Sorting inodes");
  Arrays.sort(inodes, INODE_BYTES_COMPARATOR);
  LOG.debug("Finished sorting inodes");
  return inodes;
}
 

private static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
    throws IOException {
  LOG.info("Loading inode references");
  ImmutableList.Builder<Long> builder = ImmutableList.builder();
  long counter = 0;
  while (true) {
    FsImageProto.INodeReferenceSection.INodeReference e =
        FsImageProto.INodeReferenceSection.INodeReference
            .parseDelimitedFrom(in);
    if (e == null) {
      break;
    }
    ++counter;
    builder.add(e.getReferredId());
  }
  LOG.info("Loaded " + counter + " inode references");
  return builder.build();
}
 

@Override
public int compare(byte[] o1, byte[] o2) {
  try {
    final FsImageProto.INodeSection.INode l = FsImageProto.INodeSection
            .INode.parseFrom(o1);
    final FsImageProto.INodeSection.INode r = FsImageProto.INodeSection
            .INode.parseFrom(o2);
    if (l.getId() < r.getId()) {
      return -1;
    } else if (l.getId() > r.getId()) {
      return 1;
    } else {
      return 0;
    }
  } catch (InvalidProtocolBufferException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Scan the INodeDirectory section to construct the namespace.
 */
private void buildNamespace(InputStream in) throws IOException {
  int count = 0;
  while (true) {
    FsImageProto.INodeDirectorySection.DirEntry e =
        FsImageProto.INodeDirectorySection.DirEntry.parseDelimitedFrom(in);
    if (e == null) {
      break;
    }
    count++;
    if (LOG.isDebugEnabled() && count % 10000 == 0) {
      LOG.debug("Scanned {} directories.", count);
    }
    long parentId = e.getParent();
    // Referred INode is not support for now.
    for (int i = 0; i < e.getChildrenCount(); i++) {
      long childId = e.getChildren(i);
      metadataMap.putDirChild(parentId, childId);
    }
    Preconditions.checkState(e.getRefChildrenCount() == 0);
  }
  LOG.info("Scanned {} INode directories to build namespace.", count);
}
 

private FsImageProto.INodeSection.INode fromINodeId(final long id)
        throws IOException {
  int l = 0, r = inodes.length;
  while (l < r) {
    int mid = l + (r - l) / 2;
    FsImageProto.INodeSection.INode n = FsImageProto.INodeSection.INode
            .parseFrom(inodes[mid]);
    long nid = n.getId();
    if (id > nid) {
      l = mid + 1;
    } else if (id < nid) {
      r = mid;
    } else {
      return n;
    }
  }
  return null;
}
 

private PermissionStatus getPermissionStatus(String path) throws IOException {
  long id = lookup(path);
  FsImageProto.INodeSection.INode inode = fromINodeId(id);
  switch (inode.getType()) {
    case FILE: {
      FsImageProto.INodeSection.INodeFile f = inode.getFile();
      return FSImageFormatPBINode.Loader.loadPermission(
          f.getPermission(), stringTable);
    }
    case DIRECTORY: {
      FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory();
      return FSImageFormatPBINode.Loader.loadPermission(
          d.getPermission(), stringTable);
    }
    case SYMLINK: {
      FsImageProto.INodeSection.INodeSymlink s = inode.getSymlink();
      return FSImageFormatPBINode.Loader.loadPermission(
          s.getPermission(), stringTable);
    }
    default: {
      return null;
    }
  }
}
 

private List<AclEntry> getAclEntryList(String path) throws IOException {
  long id = lookup(path);
  FsImageProto.INodeSection.INode inode = fromINodeId(id);
  switch (inode.getType()) {
    case FILE: {
      FsImageProto.INodeSection.INodeFile f = inode.getFile();
      return FSImageFormatPBINode.Loader.loadAclEntries(
          f.getAcl(), stringTable);
    }
    case DIRECTORY: {
      FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory();
      return FSImageFormatPBINode.Loader.loadAclEntries(
          d.getAcl(), stringTable);
    }
    default: {
      return new ArrayList<AclEntry>();
    }
  }
}
 

static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
    throws IOException {
  LOG.info("Loading inode references");
  ImmutableList.Builder<Long> builder = ImmutableList.builder();
  long counter = 0;
  while (true) {
    FsImageProto.INodeReferenceSection.INodeReference e =
        FsImageProto.INodeReferenceSection.INodeReference
            .parseDelimitedFrom(in);
    if (e == null) {
      break;
    }
    ++counter;
    builder.add(e.getReferredId());
  }
  LOG.info("Loaded " + counter + " inode references");
  return builder.build();
}
 

private List<Map<String, Object>> getFileStatusList(String path)
        throws IOException {
  List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
  long id = lookup(path);
  FsImageProto.INodeSection.INode inode = fromINodeId(id);
  if (inode.getType() == FsImageProto.INodeSection.INode.Type.DIRECTORY) {
    if (!dirmap.containsKey(id)) {
      // if the directory is empty, return empty list
      return list;
    }
    long[] children = dirmap.get(id);
    for (long cid : children) {
      list.add(getFileStatus(fromINodeId(cid), true));
    }
  } else {
    list.add(getFileStatus(inode, false));
  }
  return list;
}
 

private static byte[][] loadINodeSection(InputStream in)
        throws IOException {
  FsImageProto.INodeSection s = FsImageProto.INodeSection
      .parseDelimitedFrom(in);
  LOG.info("Loading " + s.getNumInodes() + " inodes.");
  final byte[][] inodes = new byte[(int) s.getNumInodes()][];

  for (int i = 0; i < s.getNumInodes(); ++i) {
    int size = CodedInputStream.readRawVarint32(in.read(), in);
    byte[] bytes = new byte[size];
    IOUtils.readFully(in, bytes, 0, size);
    inodes[i] = bytes;
  }
  LOG.debug("Sorting inodes");
  Arrays.sort(inodes, INODE_BYTES_COMPARATOR);
  LOG.debug("Finished sorting inodes");
  return inodes;
}
 

@Override
public int compare(byte[] o1, byte[] o2) {
  try {
    final FsImageProto.INodeSection.INode l = FsImageProto.INodeSection
            .INode.parseFrom(o1);
    final FsImageProto.INodeSection.INode r = FsImageProto.INodeSection
            .INode.parseFrom(o2);
    if (l.getId() < r.getId()) {
      return -1;
    } else if (l.getId() > r.getId()) {
      return 1;
    } else {
      return 0;
    }
  } catch (InvalidProtocolBufferException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Return the INodeId of the specified path.
 */
private long lookup(String path) throws IOException {
  Preconditions.checkArgument(path.startsWith("/"));
  long id = INodeId.ROOT_INODE_ID;
  for (int offset = 0, next; offset < path.length(); offset = next) {
    next = path.indexOf('/', offset + 1);
    if (next == -1) {
      next = path.length();
    }
    if (offset + 1 > next) {
      break;
    }

    final String component = path.substring(offset + 1, next);

    if (component.isEmpty()) {
      continue;
    }

    final long[] children = dirmap.get(id);
    if (children == null) {
      throw new FileNotFoundException(path);
    }

    boolean found = false;
    for (long cid : children) {
      FsImageProto.INodeSection.INode child = fromINodeId(cid);
      if (component.equals(child.getName().toStringUtf8())) {
        found = true;
        id = child.getId();
        break;
      }
    }
    if (!found) {
      throw new FileNotFoundException(path);
    }
  }
  return id;
}
 

private static Map<Long, long[]> loadINodeDirectorySection
        (InputStream in, List<Long> refIdList)
    throws IOException {
  LOG.info("Loading inode directory section");
  Map<Long, long[]> dirs = Maps.newHashMap();
  long counter = 0;
  while (true) {
    FsImageProto.INodeDirectorySection.DirEntry e =
        FsImageProto.INodeDirectorySection.DirEntry.parseDelimitedFrom(in);
    // note that in is a LimitedInputStream
    if (e == null) {
      break;
    }
    ++counter;

    long[] l = new long[e.getChildrenCount() + e.getRefChildrenCount()];
    for (int i = 0; i < e.getChildrenCount(); ++i) {
      l[i] = e.getChildren(i);
    }
    for (int i = e.getChildrenCount(); i < l.length; i++) {
      int refId = e.getRefChildren(i - e.getChildrenCount());
      l[i] = refIdList.get(refId);
    }
    dirs.put(e.getParent(), l);
  }
  LOG.info("Loaded " + counter + " directories");
  return dirs;
}
 

/**
 * Return the JSON formatted FileStatus of the specified file.
 * @param path a path specifies a file
 * @return JSON formatted FileStatus
 * @throws IOException if failed to serialize fileStatus to JSON.
 */
String getFileStatus(String path) throws IOException {
  ObjectMapper mapper = new ObjectMapper();
  FsImageProto.INodeSection.INode inode = fromINodeId(lookup(path));
  return "{\"FileStatus\":\n"
      + mapper.writeValueAsString(getFileStatus(inode, false)) + "\n}\n";
}
 

static String[] loadStringTable(InputStream in) throws
IOException {
  FsImageProto.StringTableSection s = FsImageProto.StringTableSection
      .parseDelimitedFrom(in);
  LOG.info("Loading " + s.getNumEntry() + " strings");
  String[] stringTable = new String[s.getNumEntry() + 1];
  for (int i = 0; i < s.getNumEntry(); ++i) {
    FsImageProto.StringTableSection.Entry e = FsImageProto
        .StringTableSection.Entry.parseDelimitedFrom(in);
    stringTable[e.getId()] = e.getStr();
  }
  return stringTable;
}
 

private static Map<Long, long[]> loadINodeDirectorySection
        (InputStream in, List<Long> refIdList)
    throws IOException {
  LOG.info("Loading inode directory section");
  Map<Long, long[]> dirs = Maps.newHashMap();
  long counter = 0;
  while (true) {
    FsImageProto.INodeDirectorySection.DirEntry e =
        FsImageProto.INodeDirectorySection.DirEntry.parseDelimitedFrom(in);
    // note that in is a LimitedInputStream
    if (e == null) {
      break;
    }
    ++counter;

    long[] l = new long[e.getChildrenCount() + e.getRefChildrenCount()];
    for (int i = 0; i < e.getChildrenCount(); ++i) {
      l[i] = e.getChildren(i);
    }
    for (int i = e.getChildrenCount(); i < l.length; i++) {
      int refId = e.getRefChildren(i - e.getChildrenCount());
      l[i] = refIdList.get(refId);
    }
    dirs.put(e.getParent(), l);
  }
  LOG.info("Loaded " + counter + " directories");
  return dirs;
}
 

static String[] loadStringTable(InputStream in) throws
IOException {
  FsImageProto.StringTableSection s = FsImageProto.StringTableSection
      .parseDelimitedFrom(in);
  LOG.info("Loading " + s.getNumEntry() + " strings");
  String[] stringTable = new String[s.getNumEntry() + 1];
  for (int i = 0; i < s.getNumEntry(); ++i) {
    FsImageProto.StringTableSection.Entry e = FsImageProto
        .StringTableSection.Entry.parseDelimitedFrom(in);
    stringTable[e.getId()] = e.getStr();
  }
  return stringTable;
}
 

/**
 * Return the JSON formatted FileStatus of the specified file.
 * @param path a path specifies a file
 * @return JSON formatted FileStatus
 * @throws IOException if failed to serialize fileStatus to JSON.
 */
String getFileStatus(String path) throws IOException {
  ObjectMapper mapper = new ObjectMapper();
  FsImageProto.INodeSection.INode inode = fromINodeId(lookup(path));
  return "{\"FileStatus\":\n"
      + mapper.writeValueAsString(getFileStatus(inode, false)) + "\n}\n";
}
 

static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
  long size = 0;
  for (HdfsProtos.BlockProto p : f.getBlocksList()) {
    size += p.getNumBytes();
  }
  return size;
}
 

private List<XAttr> getXAttrList(String path) throws IOException {
  long id = lookup(path);
  FsImageProto.INodeSection.INode inode = fromINodeId(id);
  switch (inode.getType()) {
    case FILE:
      return FSImageFormatPBINode.Loader.loadXAttrs(
              inode.getFile().getXAttrs(), stringTable);
    case DIRECTORY:
      return FSImageFormatPBINode.Loader.loadXAttrs(inode.getDirectory()
              .getXAttrs(), stringTable);
    default:
      return null;
  }
}
 

@Override
public void onFile(FsImageProto.INodeSection.INode inode, String path) {
    FsImageProto.INodeSection.INodeFile f = inode.getFile();
    pathStats.sumBlocks.add(f.getBlocksCount());
    final long fileSize = FSImageLoader.getFileSize(f);
    pathStats.fileSize.observe(fileSize);
}
 

static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
  long size = 0;
  for (HdfsProtos.BlockProto p : f.getBlocksList()) {
    size += p.getNumBytes();
  }
  return size;
}
 

/**
 * Return the INodeId of the specified path.
 */
private long lookup(String path) throws IOException {
  Preconditions.checkArgument(path.startsWith("/"));
  long id = INodeId.ROOT_INODE_ID;
  for (int offset = 0, next; offset < path.length(); offset = next) {
    next = path.indexOf('/', offset + 1);
    if (next == -1) {
      next = path.length();
    }
    if (offset + 1 > next) {
      break;
    }

    final String component = path.substring(offset + 1, next);

    if (component.isEmpty()) {
      continue;
    }

    final long[] children = dirmap.get(id);
    if (children == null) {
      throw new FileNotFoundException(path);
    }

    boolean found = false;
    for (long cid : children) {
      FsImageProto.INodeSection.INode child = fromINodeId(cid);
      if (component.equals(child.getName().toStringUtf8())) {
        found = true;
        id = child.getId();
        break;
      }
    }
    if (!found) {
      throw new FileNotFoundException(path);
    }
  }
  return id;
}
 

private Map<String, Object> getContentSummaryMap(String path)
        throws IOException {
  long id = lookup(path);
  INode inode = fromINodeId(id);
  long spaceQuota = 0;
  long nsQuota = 0;
  long[] data = new long[4];
  FsImageProto.INodeSection.INodeFile f = inode.getFile();
  switch (inode.getType()) {
    case FILE:
      data[0] = 0;
      data[1] = 1;
      data[2] = getFileSize(f);
      nsQuota = -1;
      data[3] = data[2] * f.getReplication();
      spaceQuota = -1;
      return fillSummaryMap(spaceQuota, nsQuota, data);
    case DIRECTORY:
      fillDirSummary(id, data);
      nsQuota = inode.getDirectory().getNsQuota();
      spaceQuota = inode.getDirectory().getDsQuota();
      return fillSummaryMap(spaceQuota, nsQuota, data);
    case SYMLINK:
      data[0] = 0;
      data[1] = 1;
      data[2] = 0;
      nsQuota = -1;
      data[3] = 0;
      spaceQuota = -1;
      return fillSummaryMap(spaceQuota, nsQuota, data);
    default:
      return null;
  }

}
 

private void fillDirSummary(long id, long[] data) throws IOException {
  data[0]++;
  long[] children = dirmap.get(id);
  if (children == null) {
    return;
  }

  for (long cid : children) {
    INode node = fromINodeId(cid);
    switch (node.getType()) {
      case DIRECTORY:
        fillDirSummary(cid, data);
        break;
      case FILE:
        FsImageProto.INodeSection.INodeFile f = node.getFile();
        long curLength = getFileSize(f);
        data[1]++;
        data[2] += curLength;
        data[3] += (curLength) * (f.getReplication());
        break;
      case SYMLINK:
        data[1]++;
        break;
      default:
        break;
    }
  }
}