Java Code Examples for org.apache.hadoop.hdfs.protocol.LayoutVersion#supports()

/**
 * Construct the reader
 * @param in The stream to read from.
 * @param logVersion The version of the data coming from the stream.
 */
@SuppressWarnings("deprecation")
public Reader(DataInputStream in, int logVersion) {
  this.logVersion = logVersion;
  if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) {
    this.checksum = FSEditLog.getChecksumForRead();
  } else {
    this.checksum = null;
  }

  if (this.checksum != null) {
    this.in = new DataInputStream(
        new CheckedInputStream(in, this.checksum));
  } else {
    this.in = in;
  }
}
 

/**
  * Process an INode
  * 
  * @param in image stream
  * @param v visitor
  * @param skipBlocks skip blocks or not
  * @param parentName the name of its parent node
  * @throws IOException
  */
private void processINode(DataInputStream in, ImageVisitor v,
    boolean skipBlocks, String parentName) throws IOException {
  v.visitEnclosingElement(ImageElement.INODE);
  String pathName = FSImageSerialization.readString(in);
  if (parentName != null) {  // local name
    pathName = "/" + pathName;
    if (!"/".equals(parentName)) { // children of non-root directory
      pathName = parentName + pathName;
    }
  }

  v.visit(ImageElement.INODE_PATH, pathName);
  v.visit(ImageElement.REPLICATION, in.readShort());
  v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
  if(LayoutVersion.supports(Feature.FILE_ACCESS_TIME, imageVersion))
    v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
  v.visit(ImageElement.BLOCK_SIZE, in.readLong());
  int numBlocks = in.readInt();

  processBlocks(in, v, numBlocks, skipBlocks);

  // File or directory
  if (numBlocks > 0 || numBlocks == -1) {
    v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
    if (LayoutVersion.supports(Feature.DISKSPACE_QUOTA, imageVersion))
      v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
  }
  if (numBlocks == -2) {
    v.visit(ImageElement.SYMLINK, Text.readString(in));
  }

  processPermission(in, v);
  v.leaveEnclosingElement(); // INode
}
 

/**
 * Process the INode records stored in the fsimage.
 *
 * @param in Datastream to process
 * @param v Visitor to walk over INodes
 * @param numInodes Number of INodes stored in file
 * @param skipBlocks Process all the blocks within the INode?
 * @throws VisitException
 * @throws IOException
 */
private void processINodes(DataInputStream in, ImageVisitor v,
    long numInodes, boolean skipBlocks) throws IOException {
  v.visitEnclosingElement(ImageElement.INODES,
      ImageElement.NUM_INODES, numInodes);
  
  if (LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION, imageVersion)) {
    processLocalNameINodes(in, v, numInodes, skipBlocks);
  } else { // full path name
    processFullNameINodes(in, v, numInodes, skipBlocks);
  }

  
  v.leaveEnclosingElement(); // INodes
}
 

/** Validate and set clusterId from {@link Properties}*/
protected void setClusterId(Properties props, int layoutVersion,
    StorageDirectory sd) throws InconsistentFSStateException {
  // Set cluster ID in version that supports federation
  if (LayoutVersion.supports(getServiceLayoutFeatureMap(),
      Feature.FEDERATION, layoutVersion)) {
    String cid = getProperty(props, sd, "clusterID");
    if (!(clusterID.equals("") || cid.equals("") || clusterID.equals(cid))) {
      throw new InconsistentFSStateException(sd.getRoot(),
          "cluster Id is incompatible with others.");
    }
    clusterID = cid;
  }
}
 

/** Validate and set clusterId from {@link Properties}*/
protected void setClusterId(Properties props, int layoutVersion,
    StorageDirectory sd) throws InconsistentFSStateException {
  // Set cluster ID in version that supports federation
  if (LayoutVersion.supports(getServiceLayoutFeatureMap(),
      Feature.FEDERATION, layoutVersion)) {
    String cid = getProperty(props, sd, "clusterID");
    if (!(clusterID.equals("") || cid.equals("") || clusterID.equals(cid))) {
      throw new InconsistentFSStateException(sd.getRoot(),
          "cluster Id is incompatible with others.");
    }
    clusterID = cid;
  }
}
 

/**
 * Read an operation from the input stream.
 * 
 * Note that the objects returned from this method may be re-used by future
 * calls to the same method.
 * 
 * @return the operation read from the stream, or null at the end of the file
 * @throws IOException on error.
 */
public FSEditLogOp readOp() throws IOException {
  if (checksum != null) {
    checksum.reset();
  }

  byte opCodeByte;
  try {
    opCodeByte = in.readByte();
  } catch (EOFException eof) {
    // EOF at an opcode boundary is expected
    return null;
  }

  FSEditLogOpCodes opCode = FSEditLogOpCodes.fromByte(opCodeByte);

  if (opCode == OP_INVALID) {
    LOG.info("Reached the end of the edit file...");
    return null;
  }

  FSEditLogOp op = opInstances.get().get(opCode);
  if (op == null) {
    throw new IOException("Read invalid opcode " + opCode);
  }

  if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
    // Read the txid
    op.setTransactionId(in.readLong());
  }

  op.readFields(in, logVersion);
  if (checksum != null)
    readChecksum = checksum.getValue();
  validateChecksum(in, checksum, op.txid);
  return op;
}
 

@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {

  this.length = in.readInt();
  if (-17 < logVersion && length != 2 ||
      logVersion <= -17 && length != 3) {
    throw new IOException("Incorrect data format. "
                          + "Mkdir operation.");
  }
  this.path = FSImageSerialization.readString(in);
  this.timestamp = readLong(in);

  // The disk format stores atimes for directories as well.
  // However, currently this is not being updated/used because of
  // performance reasons.
  if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
    /* unused this.atime = */
    readLong(in);
   }

  if (logVersion <= -11) {
    this.permissions = PermissionStatus.read(in);
  } else {
    this.permissions = null;
  }
}
 

@Override
void readFields(DataInputStream in, int logVersion)
    throws IOException {
  // versions > 0 support per file replication
  // get name and replication
  this.length = in.readInt();
  if (-7 == logVersion && length != 3||
      -17 < logVersion && logVersion < -7 && length != 4 ||
      (logVersion <= -17 && length != 5)) {
    throw new IOException("Incorrect data format."  +
                          " logVersion is " + logVersion +
                          " but writables.length is " +
                          length + ". ");
  }
  this.path = FSImageSerialization.readString(in);

  this.replication = readShort(in);
  this.mtime = readLong(in);

  if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
    this.atime = readLong(in);
  } else {
    this.atime = 0;
  }
  if (logVersion < -7) {
    this.blockSize = readLong(in);
  } else {
    this.blockSize = 0;
  }

  // get blocks
  this.blocks = readBlocks(in, logVersion);

  if (logVersion <= -11) {
    this.permissions = PermissionStatus.read(in);
  } else {
    this.permissions = null;
  }

  // clientname, clientMachine and block locations of last block.
  if (this.opCode == OP_ADD && logVersion <= -12) {
    this.clientName = FSImageSerialization.readString(in);
    this.clientMachine = FSImageSerialization.readString(in);
    if (-13 <= logVersion) {
      readDatanodeDescriptorArray(in);
    }
  } else {
    this.clientName = "";
    this.clientMachine = "";
  }
}
 

/**
 * Loads edits file, uses visitor to process all elements
 */
@Override
public void loadEdits() throws IOException {

  try {
    v.start();
    v.visitEnclosingElement(EditsElement.EDITS);

    IntToken editsVersionToken = v.visitInt(EditsElement.EDITS_VERSION);
    editsVersion = editsVersionToken.value;
    if(!canLoadVersion(editsVersion)) {
      throw new IOException("Cannot process editLog version " +
        editsVersionToken.value);
    }

    FSEditLogOpCodes editsOpCode;
    do {
      v.visitEnclosingElement(EditsElement.RECORD);

      ByteToken opCodeToken;
      try {
        opCodeToken = v.visitByte(EditsElement.OPCODE);
      } catch (EOFException eof) {
        // Getting EOF when reading the opcode is fine --
        // it's just a finalized edits file
        // Just fake the OP_INVALID here.
        opCodeToken = new ByteToken(EditsElement.OPCODE);
        opCodeToken.fromByte(FSEditLogOpCodes.OP_INVALID.getOpCode());
        v.visit(opCodeToken);
      }
      editsOpCode = FSEditLogOpCodes.fromByte(opCodeToken.value);

      v.visitEnclosingElement(EditsElement.DATA);

      visitOpCode(editsOpCode);

      v.leaveEnclosingElement(); // DATA
      
      if (editsOpCode != FSEditLogOpCodes.OP_INVALID && 
          LayoutVersion.supports(Feature.EDITS_CHESKUM, editsVersion)) {
        v.visitInt(EditsElement.CHECKSUM);
      }
      v.leaveEnclosingElement(); // RECORD
    } while(editsOpCode != FSEditLogOpCodes.OP_INVALID);

    v.leaveEnclosingElement(); // EDITS
    v.finish();
  } catch(IOException e) {
    // Tell the visitor to clean up, then re-throw the exception
    v.finishAbnormally();
    throw e;
  }
}
 

/**
 * Visit a transaction ID, if the log version supports it.
 */
private void visitTxId() throws IOException {
  if (LayoutVersion.supports(Feature.STORED_TXIDS, editsVersion)) {
    v.visitLong(EditsElement.TRANSACTION_ID);
  }
}
 

/**
 * load an inode from fsimage except for its name
 * 
 * @param in data input stream from which image is read
 * @return an inode
 */
private INode loadINode(DataInputStream in)
    throws IOException {
  long modificationTime = 0;
  long atime = 0;
  long blockSize = 0;
  
  int imgVersion = getLayoutVersion();
  short replication = in.readShort();
  replication = namesystem.adjustReplication(replication);
  modificationTime = in.readLong();
  if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, imgVersion)) {
    atime = in.readLong();
  }
  if (imgVersion <= -8) {
    blockSize = in.readLong();
  }
  int numBlocks = in.readInt();
  BlockInfo blocks[] = null;

  // for older versions, a blocklist of size 0
  // indicates a directory.
  if ((-9 <= imgVersion && numBlocks > 0) ||
      (imgVersion < -9 && numBlocks >= 0)) {
    blocks = new BlockInfo[numBlocks];
    for (int j = 0; j < numBlocks; j++) {
      blocks[j] = new BlockInfo(replication);
      if (-14 < imgVersion) {
        blocks[j].set(in.readLong(), in.readLong(), 
                      Block.GRANDFATHER_GENERATION_STAMP);
      } else {
        blocks[j].readFields(in);
      }
    }
  }
  // Older versions of HDFS does not store the block size in inode.
  // If the file has more than one block, use the size of the 
  // first block as the blocksize. Otherwise use the default block size.
  //
  if (-8 <= imgVersion && blockSize == 0) {
    if (numBlocks > 1) {
      blockSize = blocks[0].getNumBytes();
    } else {
      long first = ((numBlocks == 1) ? blocks[0].getNumBytes(): 0);
      blockSize = Math.max(namesystem.getDefaultBlockSize(), first);
    }
  }
  
  // get quota only when the node is a directory
  long nsQuota = -1L;
  if (LayoutVersion.supports(Feature.NAMESPACE_QUOTA, imgVersion)
      && blocks == null) {
    nsQuota = in.readLong();
  }
  long dsQuota = -1L;
  if (LayoutVersion.supports(Feature.DISKSPACE_QUOTA, imgVersion)
      && blocks == null) {
    dsQuota = in.readLong();
  }

  PermissionStatus permissions = namesystem.getUpgradePermission();
  if (imgVersion <= -11) {
    permissions = PermissionStatus.read(in);
  }

  return INode.newINode(permissions, blocks, replication,
      modificationTime, atime, nsQuota, dsQuota, blockSize);
  }
 

public static boolean supports(final LayoutFeature f, final int lv) {
  return LayoutVersion.supports(FEATURES, f, lv);
}
 

public boolean versionSupportsFederation(
    Map<Integer, SortedSet<LayoutFeature>> map) {
  return LayoutVersion.supports(map, LayoutVersion.Feature.FEDERATION,
      layoutVersion);
}
 

public static boolean supports(final LayoutFeature f, final int lv) {
  return LayoutVersion.supports(FEATURES, f, lv);
}
 

public static boolean supports(final LayoutFeature f, final int lv) {
  return LayoutVersion.supports(FEATURES, f, lv);
}
 

public boolean versionSupportsFederation(
    Map<Integer, SortedSet<LayoutFeature>> map) {
  return LayoutVersion.supports(map, LayoutVersion.Feature.FEDERATION,
      layoutVersion);
}
 

public static boolean supports(final LayoutFeature f, final int lv) {
  return LayoutVersion.supports(FEATURES, f, lv);
}