Java Code Examples for org.apache.hadoop.hdfs.server.protocol.DatanodeCommand#getAction()

The following examples show how to use org.apache.hadoop.hdfs.server.protocol.DatanodeCommand#getAction() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NNThroughputBenchmark.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
  // register datanode
  StorageReport[] rep = { new StorageReport(storage,
      false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
      rep, 0L, 0L, 0, 0, 0, null).getCommands();
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(),
                              bcmd.getTargetStorageIDs());
      }
    }
  }
  return 0;
}
Example 2
Source File: NNThroughputBenchmark.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
  // register datanode
  StorageReport[] rep = { new StorageReport(storage,
      false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
      rep, 0L, 0L, 0, 0, 0, null).getCommands();
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(),
                              bcmd.getTargetStorageIDs());
      }
    }
  }
  return 0;
}
Example 3
Source File: NNThroughputBenchmark.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
int replicateBlocks() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0, 0);
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets());
      }
    }
  }
  return 0;
}
Example 4
Source File: NNThroughputBenchmark.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused")
int replicateBlocks() throws IOException {
	// register datanode
	DatanodeCommand[] cmds = nameNode.sendHeartbeat(dnRegistration,
			DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0, 0);
	if (cmds != null) {
		for (DatanodeCommand cmd : cmds) {
			if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
				// Send a copy of a block to another datanode
				BlockCommand bcmd = (BlockCommand) cmd;
				return transferBlocks(bcmd.getBlocks(),
						bcmd.getTargets());
			}
		}
	}
	return 0;
}
Example 5
Source File: NNThroughputBenchmark.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
int replicateBlocks() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, 0, 0);
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets());
      }
    }
  }
  return 0;
}
Example 6
Source File: OfferService.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
private static boolean isValidStandbyCommand(DatanodeCommand cmd) {
  for (int validCommand : validStandbyCommands) {
    if (cmd.getAction() == validCommand) {
      return true;
    }
  }
  return false;
}
Example 7
Source File: OfferService.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Determines whether a failover has happened and accordingly takes the
 * appropriate action.
 * 
 * @param cmd
 *          the command received from the AvatarNode
 * @return whether or not this service is the primary service
 */
private boolean checkFailover(DatanodeCommand cmd) throws InterruptedException {
  boolean isPrimary = isPrimaryServiceCached();
  if (!isPrimary && isPrimaryService()) {
    // The datanode has received a register command after the failover, this
    // means that the offerservice thread for the datanode was down for a
    // while and it most probably did not clean up its deletion queue, hence
    // force a cleanup.
    if (cmd.getAction() == DatanodeProtocol.DNA_REGISTER) {
      this.clearPrimary();
    }
    this.servicePair.setPrimaryOfferService(this);
  }
  return isPrimaryServiceCached();
}
Example 8
Source File: OfferService.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Process an array of datanode commands
 * 
 * @param cmds an array of datanode commands
 * @return true if further processing may be required or false otherwise.
 */
private boolean processCommand(DatanodeCommand[] cmds) throws InterruptedException {
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      boolean isPrimary = checkFailover(cmd);
      try {
        // The standby service thread is allowed to process only a small set
        // of valid commands.
        if (!isValidStandbyCommand(cmd) && !isPrimaryService()) {
          LOG.warn("Received an invalid command " + cmd
              + " from standby " + this.namenodeAddress);
          continue;
        } else if (cmd.getAction() == DatanodeProtocol.DNA_REGISTER && 
               !isPrimaryService() && !isPrimary) {
          // Standby issued a DNA_REGISTER. 
          this.clearPrimary();
        }
        if (processCommand(cmd) == false) {
          return false;
        }
      } catch (IOException ioe) {
        LOG.warn("Error processing datanode Command", ioe);
      }
    }
  }
  return true;
}
Example 9
Source File: PBHelper.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static DatanodeCommandProto convert(DatanodeCommand datanodeCommand) {
  DatanodeCommandProto.Builder builder = DatanodeCommandProto.newBuilder();
  if (datanodeCommand == null) {
    return builder.setCmdType(DatanodeCommandProto.Type.NullDatanodeCommand)
        .build();
  }
  switch (datanodeCommand.getAction()) {
  case DatanodeProtocol.DNA_BALANCERBANDWIDTHUPDATE:
    builder.setCmdType(DatanodeCommandProto.Type.BalancerBandwidthCommand)
        .setBalancerCmd(
            PBHelper.convert((BalancerBandwidthCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_ACCESSKEYUPDATE:
    builder
        .setCmdType(DatanodeCommandProto.Type.KeyUpdateCommand)
        .setKeyUpdateCmd(PBHelper.convert((KeyUpdateCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_RECOVERBLOCK:
    builder.setCmdType(DatanodeCommandProto.Type.BlockRecoveryCommand)
        .setRecoveryCmd(
            PBHelper.convert((BlockRecoveryCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_FINALIZE:
    builder.setCmdType(DatanodeCommandProto.Type.FinalizeCommand)
        .setFinalizeCmd(PBHelper.convert((FinalizeCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_REGISTER:
    builder.setCmdType(DatanodeCommandProto.Type.RegisterCommand)
        .setRegisterCmd(REG_CMD_PROTO);
    break;
  case DatanodeProtocol.DNA_TRANSFER:
  case DatanodeProtocol.DNA_INVALIDATE:
  case DatanodeProtocol.DNA_SHUTDOWN:
    builder.setCmdType(DatanodeCommandProto.Type.BlockCommand).
      setBlkCmd(PBHelper.convert((BlockCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_CACHE:
  case DatanodeProtocol.DNA_UNCACHE:
    builder.setCmdType(DatanodeCommandProto.Type.BlockIdCommand).
      setBlkIdCmd(PBHelper.convert((BlockIdCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_UNKNOWN: //Not expected
  default:
    builder.setCmdType(DatanodeCommandProto.Type.NullDatanodeCommand);
  }
  return builder.build();
}
Example 10
Source File: PBHelper.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public static DatanodeCommandProto convert(DatanodeCommand datanodeCommand) {
  DatanodeCommandProto.Builder builder = DatanodeCommandProto.newBuilder();
  if (datanodeCommand == null) {
    return builder.setCmdType(DatanodeCommandProto.Type.NullDatanodeCommand)
        .build();
  }
  switch (datanodeCommand.getAction()) {
  case DatanodeProtocol.DNA_BALANCERBANDWIDTHUPDATE:
    builder.setCmdType(DatanodeCommandProto.Type.BalancerBandwidthCommand)
        .setBalancerCmd(
            PBHelper.convert((BalancerBandwidthCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_ACCESSKEYUPDATE:
    builder
        .setCmdType(DatanodeCommandProto.Type.KeyUpdateCommand)
        .setKeyUpdateCmd(PBHelper.convert((KeyUpdateCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_RECOVERBLOCK:
    builder.setCmdType(DatanodeCommandProto.Type.BlockRecoveryCommand)
        .setRecoveryCmd(
            PBHelper.convert((BlockRecoveryCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_FINALIZE:
    builder.setCmdType(DatanodeCommandProto.Type.FinalizeCommand)
        .setFinalizeCmd(PBHelper.convert((FinalizeCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_REGISTER:
    builder.setCmdType(DatanodeCommandProto.Type.RegisterCommand)
        .setRegisterCmd(REG_CMD_PROTO);
    break;
  case DatanodeProtocol.DNA_TRANSFER:
  case DatanodeProtocol.DNA_INVALIDATE:
  case DatanodeProtocol.DNA_SHUTDOWN:
    builder.setCmdType(DatanodeCommandProto.Type.BlockCommand).
      setBlkCmd(PBHelper.convert((BlockCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_CACHE:
  case DatanodeProtocol.DNA_UNCACHE:
    builder.setCmdType(DatanodeCommandProto.Type.BlockIdCommand).
      setBlkIdCmd(PBHelper.convert((BlockIdCommand) datanodeCommand));
    break;
  case DatanodeProtocol.DNA_UNKNOWN: //Not expected
  default:
    builder.setCmdType(DatanodeCommandProto.Type.NullDatanodeCommand);
  }
  return builder.build();
}
Example 11
Source File: OfferService.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
/**
 *
 * @param cmd
 * @return true if further processing may be required or false otherwise.
 * @throws IOException
 */
private boolean processCommand(DatanodeCommand cmd) throws IOException, InterruptedException {
  if (cmd == null)
    return true;
  final BlockCommand bcmd = cmd instanceof BlockCommand? (BlockCommand)cmd: null;

  boolean retValue = true;
  long startTime = System.currentTimeMillis();

  switch(cmd.getAction()) {
  case DatanodeProtocol.DNA_TRANSFER:
    // Send a copy of a block to another datanode
    anode.transferBlocks(servicePair.namespaceId, bcmd.getBlocks(), bcmd.getTargets());
    myMetrics.blocksReplicated.inc(bcmd.getBlocks().length);
    break;
  case DatanodeProtocol.DNA_INVALIDATE:
    //
    // Some local block(s) are obsolete and can be
    // safely garbage-collected.
    //
    Block toDelete[] = bcmd.getBlocks();
    try {
      if (anode.blockScanner != null) {
        //TODO temporary
        anode.blockScanner.deleteBlocks(servicePair.namespaceId, toDelete);
      }
      servicePair.removeReceivedBlocks(toDelete);
      data.invalidate(servicePair.namespaceId, toDelete);
    } catch(IOException e) {
      anode.checkDiskError();
      throw e;
    }
    myMetrics.blocksRemoved.inc(toDelete.length);
    break;
  case DatanodeProtocol.DNA_SHUTDOWN:
    // shut down the data node
    servicePair.shutdown();
    retValue = false;
    break;
  case DatanodeProtocol.DNA_REGISTER:
    // namenode requested a registration - at start or if NN lost contact
    LOG.info("AvatarDatanodeCommand action: DNA_REGISTER");
    if (shouldRun()) {
      servicePair.register(namenode, namenodeAddress);
      firstBlockReportSent = false;
      scheduleBlockReport(0);
    }
    break;
  case DatanodeProtocol.DNA_FINALIZE:
    anode.getStorage().finalizedUpgrade(servicePair.namespaceId);
    break;
  case UpgradeCommand.UC_ACTION_START_UPGRADE:
    // start distributed upgrade here
    servicePair.processUpgradeCommand((UpgradeCommand)cmd);
    break;
  case DatanodeProtocol.DNA_RECOVERBLOCK:
    anode.recoverBlocks(servicePair.namespaceId, bcmd.getBlocks(), bcmd.getTargets());
    break;
  case DatanodeProtocols.DNA_BACKOFF:
    // We can get a BACKOFF request as a response to a full block report.
    setBackoff(true);
    break;
  case DatanodeProtocols.DNA_CLEARPRIMARY:
    LOG.info("CLEAR PRIMARY requested by : " + this.avatarnodeAddress);
    retValue = clearPrimary();
    break;
  case DatanodeProtocols.DNA_RETRY:
    // We will get a RETRY request as a response to only a full block report.
    LOG.info(this.avatarnodeAddress + " has requested the retry of : "
        + bcmd.getBlocks().length + " blocks in response to a full block"
        + " report");
    // Retry the blocks that failed on the Standby.
    processFailedBlocks(bcmd.getBlocks(), bcmd.getBlocks().length);
    break;
  default:
    LOG.warn("Unknown DatanodeCommand action: " + cmd.getAction());
  }

  long endTime = System.currentTimeMillis();
  if (endTime - startTime > 1000) {
    LOG.info("processCommand() took " + (endTime - startTime)
             + " msec to process command " + cmd.getAction()
             + " from " + namenodeAddress);
  } else if (LOG.isDebugEnabled()) {
    LOG.debug("processCommand() took " + (endTime - startTime)
              + " msec to process command " + cmd.getAction()
              + " from " + namenodeAddress);
  }
  return retValue;
}
Example 12
Source File: DataNode.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
/**
 *
 * @param cmd
 * @return true if further processing may be required or false otherwise.
 * @throws IOException
 */
private boolean processCommand(DatanodeCommand cmd) throws IOException {
  if (cmd == null)
    return true;
  final BlockCommand bcmd = cmd instanceof BlockCommand? (BlockCommand)cmd: null;

  boolean retValue = true;
  long startTime = System.currentTimeMillis();

  switch(cmd.getAction()) {
  case DatanodeProtocol.DNA_TRANSFER:
    // Send a copy of a block to another datanode
    transferBlocks(namespaceId,
        bcmd.getBlocks(), bcmd.getTargets());
    myMetrics.blocksReplicated.inc(bcmd.getBlocks().length);
    break;
  case DatanodeProtocol.DNA_INVALIDATE:
    //
    // Some local block(s) are obsolete and can be 
    // safely garbage-collected.
    //
    Block toDelete[] = bcmd.getBlocks();
    try {
      if (blockScanner != null) {
        blockScanner.deleteBlocks(namespaceId, toDelete);
      }        
      data.invalidate(namespaceId, toDelete);
    } catch(IOException e) {
      checkDiskError();
      throw e;
    }
    myMetrics.blocksRemoved.inc(toDelete.length);
    break;
  case DatanodeProtocol.DNA_SHUTDOWN:
    // shut down the data node
    shouldServiceRun = false;
    retValue = false;
    break;
  case DatanodeProtocol.DNA_REGISTER:
    // namenode requested a registration - at start or if NN lost contact
    LOG.info("DatanodeCommand action: DNA_REGISTER");
    if (shouldRun) {
      register();
      firstBlockReportSent = false;
    }
    break;
  case DatanodeProtocol.DNA_FINALIZE:
     storage.finalizedUpgrade(namespaceId);
    break;
  case UpgradeCommand.UC_ACTION_START_UPGRADE:
    // start distributed upgrade here
    processDistributedUpgradeCommand((UpgradeCommand)cmd);
    break;
  case DatanodeProtocol.DNA_RECOVERBLOCK:
    recoverBlocks(namespaceId, bcmd.getBlocks(), bcmd.getTargets());
    break;
  default:
    LOG.warn("Unknown DatanodeCommand action: " + cmd.getAction());
  }
  long endTime = System.currentTimeMillis();
  if (endTime - startTime > 1000) {
    LOG.info("processCommand() took " + (endTime - startTime)
        + " msec to process command " + cmd.getAction() + " from " + nnAddr);
  } else if (LOG.isDebugEnabled()) {
    LOG.debug("processCommand() took " + (endTime - startTime)
        + " msec to process command " + cmd.getAction() + " from " + nnAddr);
  }
  return retValue;
}
Example 13
Source File: DataNode.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up 
/**
   * 
   * @param cmd
   * @return true if further processing may be required or false otherwise. 
   * @throws IOException
   */
private boolean processCommand(DatanodeCommand cmd) throws IOException {
  if (cmd == null)
    return true;
  final BlockCommand bcmd = cmd instanceof BlockCommand? (BlockCommand)cmd: null;

  switch(cmd.getAction()) {
  case DatanodeProtocol.DNA_TRANSFER:
    // Send a copy of a block to another datanode
    transferBlocks(bcmd.getBlocks(), bcmd.getTargets());
    myMetrics.blocksReplicated.inc(bcmd.getBlocks().length);
    break;
  case DatanodeProtocol.DNA_INVALIDATE:
    //
    // Some local block(s) are obsolete and can be 
    // safely garbage-collected.
    //
    Block toDelete[] = bcmd.getBlocks();
    try {
      if (blockScanner != null) {
        blockScanner.deleteBlocks(toDelete);
      }
      data.invalidate(toDelete);
    } catch(IOException e) {
      checkDiskError();
      throw e;
    }
    myMetrics.blocksRemoved.inc(toDelete.length);
    break;
  case DatanodeProtocol.DNA_SHUTDOWN:
    // shut down the data node
    this.shutdown();
    return false;
  case DatanodeProtocol.DNA_REGISTER:
    // namenode requested a registration - at start or if NN lost contact
    LOG.info("DatanodeCommand action: DNA_REGISTER");
    if (shouldRun) {
      register();
    }
    break;
  case DatanodeProtocol.DNA_FINALIZE:
    storage.finalizeUpgrade();
    break;
  case UpgradeCommand.UC_ACTION_START_UPGRADE:
    // start distributed upgrade here
    processDistributedUpgradeCommand((UpgradeCommand)cmd);
    break;
  case DatanodeProtocol.DNA_RECOVERBLOCK:
    recoverBlocks(bcmd.getBlocks(), bcmd.getTargets());
    break;
  default:
    LOG.warn("Unknown DatanodeCommand action: " + cmd.getAction());
  }
  return true;
}