org.apache.hadoop.hdfs.server.protocol.DatanodeCommand Java Examples

@Override
public DatanodeCommand cacheReport(DatanodeRegistration registration,
    String poolId, List<Long> blockIds) throws IOException {
  CacheReportRequestProto.Builder builder =
      CacheReportRequestProto.newBuilder()
      .setRegistration(PBHelper.convert(registration))
      .setBlockPoolId(poolId);
  for (Long blockId : blockIds) {
    builder.addBlocks(blockId);
  }
  
  CacheReportResponseProto resp;
  try {
    resp = rpcProxy.cacheReport(NULL_CONTROLLER, builder.build());
  } catch (ServiceException se) {
    throw ProtobufHelper.getRemoteException(se);
  }
  if (resp.hasCmd()) {
    return PBHelper.convert(resp.getCmd());
  }
  return null;
}
 

/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
int replicateBlocks() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0, 0);
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets());
      }
    }
  }
  return 0;
}
 

/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused")
int replicateBlocks() throws IOException {
	// register datanode
	DatanodeCommand[] cmds = nameNode.sendHeartbeat(dnRegistration,
			DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0, 0);
	if (cmds != null) {
		for (DatanodeCommand cmd : cmds) {
			if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
				// Send a copy of a block to another datanode
				BlockCommand bcmd = (BlockCommand) cmd;
				return transferBlocks(bcmd.getBlocks(),
						bcmd.getTargets());
			}
		}
	}
	return 0;
}
 

public DatanodeCommand blockReportNew(DatanodeRegistration nodeReg, BlockReport rep) throws IOException {
  if (runInfo.shutdown || !runInfo.isRunning) {
    return null;
  }
  if (ignoreDatanodes()) {
    LOG.info("Standby fell behind. Telling " + nodeReg.toString() +
              " to back off");
    // Do not process block reports yet as the ingest thread is catching up
    return AvatarDatanodeCommand.BACKOFF;
  }

  if (currentAvatar == Avatar.STANDBY) {
    Collection<Block> failed = super.blockReportWithRetries(nodeReg, rep);

    BlockCommand bCmd = new BlockCommand(DatanodeProtocols.DNA_RETRY,
        failed.toArray(new Block[failed.size()]));
    return bCmd;
  } else {
    return super.blockReport(nodeReg, rep);
  }
}
 

/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
  // register datanode
  StorageReport[] rep = { new StorageReport(storage,
      false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
      rep, 0L, 0L, 0, 0, 0, null).getCommands();
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(),
                              bcmd.getTargetStorageIDs());
      }
    }
  }
  return 0;
}
 

/**
 * Send a heartbeat to the name-node.
 * Ignore reply commands.
 */
void sendHeartbeat() throws IOException {
  // register datanode
  // TODO:FEDERATION currently a single block pool is supported
  StorageReport[] rep = { new StorageReport(storage, false,
      DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep,
      0L, 0L, 0, 0, 0, null).getCommands();
  if(cmds != null) {
    for (DatanodeCommand cmd : cmds ) {
      if(LOG.isDebugEnabled()) {
        LOG.debug("sendHeartbeat Name-node reply: " + cmd.getAction());
      }
    }
  }
}
 

@Override
public CacheReportResponseProto cacheReport(RpcController controller,
    CacheReportRequestProto request) throws ServiceException {
  DatanodeCommand cmd = null;
  try {
    cmd = impl.cacheReport(
        PBHelper.convert(request.getRegistration()),
        request.getBlockPoolId(),
        request.getBlocksList());
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  CacheReportResponseProto.Builder builder =
      CacheReportResponseProto.newBuilder();
  if (cmd != null) {
    builder.setCmd(PBHelper.convert(cmd));
  }
  return builder.build();
}
 

/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
  // register datanode
  StorageReport[] rep = { new StorageReport(storage,
      false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
      rep, 0L, 0L, 0, 0, 0, null).getCommands();
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(),
                              bcmd.getTargetStorageIDs());
      }
    }
  }
  return 0;
}
 

/**
 * Send a heartbeat to the name-node.
 * Ignore reply commands.
 */
void sendHeartbeat() throws IOException {
  // register datanode
  // TODO:FEDERATION currently a single block pool is supported
  StorageReport[] rep = { new StorageReport(storage, false,
      DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep,
      0L, 0L, 0, 0, 0, null).getCommands();
  if(cmds != null) {
    for (DatanodeCommand cmd : cmds ) {
      if(LOG.isDebugEnabled()) {
        LOG.debug("sendHeartbeat Name-node reply: " + cmd.getAction());
      }
    }
  }
}
 

@Override
public DatanodeCommand blockReport(DatanodeRegistration nodeReg,
                                   long[] blocks) throws IOException {
  verifyRequest(nodeReg);
  myMetrics.numBlockReport.inc();
  BlockListAsLongs blist = new BlockListAsLongs(blocks);
  if (stateChangeLog.isDebugEnabled()) {
    stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
           +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");
  }

  namesystem.processReport(nodeReg, blist);
  if (getFSImage().isUpgradeFinalized())
    return DatanodeCommand.FINALIZE;
  return null;
}
 

public static DatanodeCommand convert(DatanodeCommandProto proto) {
  switch (proto.getCmdType()) {
  case BalancerBandwidthCommand:
    return PBHelper.convert(proto.getBalancerCmd());
  case BlockCommand:
    return PBHelper.convert(proto.getBlkCmd());
  case BlockRecoveryCommand:
    return PBHelper.convert(proto.getRecoveryCmd());
  case FinalizeCommand:
    return PBHelper.convert(proto.getFinalizeCmd());
  case KeyUpdateCommand:
    return PBHelper.convert(proto.getKeyUpdateCmd());
  case RegisterCommand:
    return REG_CMD;
  case BlockIdCommand:
    return PBHelper.convert(proto.getBlkIdCmd());
  default:
    return null;
  }
}
 

@Override
public DatanodeCommand cacheReport(DatanodeRegistration registration,
    String poolId, List<Long> blockIds) throws IOException {
  CacheReportRequestProto.Builder builder =
      CacheReportRequestProto.newBuilder()
      .setRegistration(PBHelper.convert(registration))
      .setBlockPoolId(poolId);
  for (Long blockId : blockIds) {
    builder.addBlocks(blockId);
  }
  
  CacheReportResponseProto resp;
  try {
    resp = rpcProxy.cacheReport(NULL_CONTROLLER, builder.build());
  } catch (ServiceException se) {
    throw ProtobufHelper.getRemoteException(se);
  }
  if (resp.hasCmd()) {
    return PBHelper.convert(resp.getCmd());
  }
  return null;
}
 

@Override
public CacheReportResponseProto cacheReport(RpcController controller,
    CacheReportRequestProto request) throws ServiceException {
  DatanodeCommand cmd = null;
  try {
    cmd = impl.cacheReport(
        PBHelper.convert(request.getRegistration()),
        request.getBlockPoolId(),
        request.getBlocksList());
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  CacheReportResponseProto.Builder builder =
      CacheReportResponseProto.newBuilder();
  if (cmd != null) {
    builder.setCmd(PBHelper.convert(cmd));
  }
  return builder.build();
}
 

/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
int replicateBlocks() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, 0, 0);
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets());
      }
    }
  }
  return 0;
}
 

public static DatanodeCommand convert(DatanodeCommandProto proto) {
  switch (proto.getCmdType()) {
  case BalancerBandwidthCommand:
    return PBHelper.convert(proto.getBalancerCmd());
  case BlockCommand:
    return PBHelper.convert(proto.getBlkCmd());
  case BlockRecoveryCommand:
    return PBHelper.convert(proto.getRecoveryCmd());
  case FinalizeCommand:
    return PBHelper.convert(proto.getFinalizeCmd());
  case KeyUpdateCommand:
    return PBHelper.convert(proto.getKeyUpdateCmd());
  case RegisterCommand:
    return REG_CMD;
  case BlockIdCommand:
    return PBHelper.convert(proto.getBlkIdCmd());
  default:
    return null;
  }
}
 

/**
 * This method should not be invoked on the composite 
 * DatanodeProtocols object. You can call these on the individual
 * DatanodeProcol objects.
 */
public DatanodeCommand[] sendHeartbeat(DatanodeRegistration registration,
                                     long capacity,
                                     long dfsUsed, long remaining,
                                     long namespaceUsed,
                                     int xmitsInProgress,
                                     int xceiverCount) throws IOException {
  throw new IOException("sendHeartbeat" + errMessage);
}
 

/**
 * Determines whether a failover has happened and accordingly takes the
 * appropriate action.
 * 
 * @param cmd
 *          the command received from the AvatarNode
 * @return whether or not this service is the primary service
 */
private boolean checkFailover(DatanodeCommand cmd) throws InterruptedException {
  boolean isPrimary = isPrimaryServiceCached();
  if (!isPrimary && isPrimaryService()) {
    // The datanode has received a register command after the failover, this
    // means that the offerservice thread for the datanode was down for a
    // while and it most probably did not clean up its deletion queue, hence
    // force a cleanup.
    if (cmd.getAction() == DatanodeProtocol.DNA_REGISTER) {
      this.clearPrimary();
    }
    this.servicePair.setPrimaryOfferService(this);
  }
  return isPrimaryServiceCached();
}
 

/**
 * Process an array of datanode commands
 * 
 * @param cmds an array of datanode commands
 * @return true if further processing may be required or false otherwise. 
 */
private boolean processCommand(DatanodeCommand[] cmds) {
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      try {
        if (processCommand(cmd) == false) {
          return false;
        }
      } catch (IOException ioe) {
        LOG.warn("Error processing datanode Command", ioe);
      }
    }
  }
  return true;
}
 

/**
 * Send a heartbeat to the name-node.
 * Ignore reply commands.
 */
void sendHeartbeat() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, 0, 0);
  if(cmds != null) {
    for (DatanodeCommand cmd : cmds ) {
      LOG.debug("sendHeartbeat Name-node reply: " + cmd.getAction());
    }
  }
}
 

private static boolean isValidStandbyCommand(DatanodeCommand cmd) {
  for (int validCommand : validStandbyCommands) {
    if (cmd.getAction() == validCommand) {
      return true;
    }
  }
  return false;
}
 

@Override
public DatanodeCommand cacheReport(DatanodeRegistration nodeReg,
    String poolId, List<Long> blockIds) throws IOException {
  checkNNStartup();
  verifyRequest(nodeReg);
  if (blockStateChangeLog.isDebugEnabled()) {
    blockStateChangeLog.debug("*BLOCK* NameNode.cacheReport: "
         + "from " + nodeReg + " " + blockIds.size() + " blocks");
  }
  namesystem.getCacheManager().processCacheReport(nodeReg, blockIds);
  return null;
}
 

/**
 * Data node notify the name node that it is alive 
 * Return an array of block-oriented commands for the datanode to execute.
 * This will be either a transfer or a delete operation.
 */
public DatanodeCommand[] sendHeartbeat(DatanodeRegistration nodeReg,
                                     long capacity,
                                     long dfsUsed,
                                     long remaining,
                                     int xmitsInProgress,
                                     int xceiverCount) throws IOException {
  verifyRequest(nodeReg);
  return namesystem.handleHeartbeat(nodeReg, capacity, dfsUsed, remaining,
      xceiverCount, xmitsInProgress);
}
 

public DatanodeCommand blockReport(DatanodeRegistration nodeReg,
                                   long[] blocks) throws IOException {
  verifyRequest(nodeReg);
  BlockListAsLongs blist = new BlockListAsLongs(blocks);
  stateChangeLog.debug("*BLOCK* NameNode.blockReport: "
         +"from "+nodeReg.getName()+" "+blist.getNumberOfBlocks() +" blocks");

  namesystem.processReport(nodeReg, blist);
  if (getFSImage().isUpgradeFinalized())
    return DatanodeCommand.FINALIZE;
  return null;
}
 

/**
 * Process an array of datanode commands
 * 
 * @param cmds an array of datanode commands
 * @return true if further processing may be required or false otherwise. 
 */
private boolean processCommand(DatanodeCommand[] cmds) {
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      try {
        if (processCommand(cmd) == false) {
          return false;
        }
      } catch (IOException ioe) {
        LOG.warn("Error processing datanode Command", ioe);
      }
    }
  }
  return true;
}
 

/**
 * Data node notify the name node that it is alive
 * Return an array of block-oriented commands for the datanode to execute.
 * This will be either a transfer or a delete operation.
 */
public DatanodeCommand[] sendHeartbeat(DatanodeRegistration nodeReg,
                                     long capacity,
                                     long dfsUsed,
                                     long remaining,
                                     long namespaceUsed,
                                     int xmitsInProgress,
                                     int xceiverCount) throws IOException {
  verifyRequest(nodeReg);
  myMetrics.numHeartbeat.inc();
  return namesystem.handleHeartbeat(nodeReg, capacity, dfsUsed, remaining, namespaceUsed,
      xceiverCount, xmitsInProgress);
}
 

/**
 * Send a heartbeat to the name-node.
 * Ignore reply commands.
 */
void sendHeartbeat() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0, 0);
  if(cmds != null) {
    for (DatanodeCommand cmd : cmds ) {
      LOG.debug("sendHeartbeat Name-node reply: " + cmd.getAction());
    }
  }
}
 

/**
 * Creates a cache or uncache DatanodeCommand from an array of locations
 */
private static DatanodeCommand getResponse(HdfsBlockLocation[] locs,
    int action) {
  String bpid = locs[0].getLocatedBlock().getBlock().getBlockPoolId();
  long[] blocks = new long[locs.length];
  for (int i=0; i<locs.length; i++) {
    blocks[i] = locs[i].getLocatedBlock().getBlock().getBlockId();
  }
  return new BlockIdCommand(action, bpid, blocks);
}
 

private static void setHeartbeatResponse(DatanodeCommand[] cmds)
    throws IOException {
  NNHAStatusHeartbeat ha = new NNHAStatusHeartbeat(HAServiceState.ACTIVE,
      fsImage.getLastAppliedOrWrittenTxId());
  HeartbeatResponse response = new HeartbeatResponse(cmds, ha, null);
  doReturn(response).when(spyNN).sendHeartbeat(
      (DatanodeRegistration) any(),
      (StorageReport[]) any(), anyLong(), anyLong(),
      anyInt(), anyInt(), anyInt(), (VolumeFailureSummary) any());
}
 

@Override
public BlockReportResponseProto blockReport(RpcController controller,
    BlockReportRequestProto request) throws ServiceException {
  DatanodeCommand cmd = null;
  StorageBlockReport[] report = 
      new StorageBlockReport[request.getReportsCount()];
  
  int index = 0;
  for (StorageBlockReportProto s : request.getReportsList()) {
    final BlockListAsLongs blocks;
    if (s.hasNumberOfBlocks()) { // new style buffer based reports
      int num = (int)s.getNumberOfBlocks();
      Preconditions.checkState(s.getBlocksCount() == 0,
          "cannot send both blocks list and buffers");
      blocks = BlockListAsLongs.decodeBuffers(num, s.getBlocksBuffersList());
    } else {
      blocks = BlockListAsLongs.decodeLongs(s.getBlocksList());
    }
    report[index++] = new StorageBlockReport(PBHelper.convert(s.getStorage()),
        blocks);
  }
  try {
    cmd = impl.blockReport(PBHelper.convert(request.getRegistration()),
        request.getBlockPoolId(), report,
        request.hasContext() ?
            PBHelper.convert(request.getContext()) : null);
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  BlockReportResponseProto.Builder builder = 
      BlockReportResponseProto.newBuilder();
  if (cmd != null) {
    builder.setCmd(PBHelper.convert(cmd));
  }
  return builder.build();
}
 

@Override
public HeartbeatResponseProto sendHeartbeat(RpcController controller,
    HeartbeatRequestProto request) throws ServiceException {
  HeartbeatResponse response;
  try {
    final StorageReport[] report = PBHelper.convertStorageReports(
        request.getReportsList());
    VolumeFailureSummary volumeFailureSummary =
        request.hasVolumeFailureSummary() ? PBHelper.convertVolumeFailureSummary(
            request.getVolumeFailureSummary()) : null;
    response = impl.sendHeartbeat(PBHelper.convert(request.getRegistration()),
        report, request.getCacheCapacity(), request.getCacheUsed(),
        request.getXmitsInProgress(),
        request.getXceiverCount(), request.getFailedVolumes(),
        volumeFailureSummary);
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  HeartbeatResponseProto.Builder builder = HeartbeatResponseProto
      .newBuilder();
  DatanodeCommand[] cmds = response.getCommands();
  if (cmds != null) {
    for (int i = 0; i < cmds.length; i++) {
      if (cmds[i] != null) {
        builder.addCmds(PBHelper.convert(cmds[i]));
      }
    }
  }
  builder.setHaStatus(PBHelper.convert(response.getNameNodeHaState()));
  RollingUpgradeStatus rollingUpdateStatus = response
      .getRollingUpdateStatus();
  if (rollingUpdateStatus != null) {
    builder.setRollingUpgradeStatus(PBHelper
        .convertRollingUpgradeStatus(rollingUpdateStatus));
  }
  return builder.build();
}