org.apache.hadoop.hdfs.server.namenode.CheckpointSignature Java Examples

/**
 * Copy the new fsimage into the NameNode
 */
private void putFSImage(CheckpointSignature sig) throws IOException {
  String fileid = "putimage=1&port=" + infoPort +
    "&machine=" +
    machineName +
    "&token=" + sig.toString();
  LOG.info("Standby: Posted URL " + fsName + fileid);
  TransferFsImage.getFileClient(fsName, fileid, (File[])null, false);
}
 

/**
 * Quiesces the ingest for the given file typ
 * 
 * @param type ingest to quiesce
 * @param sig signature for quiescing (checkpointing)
 */
private void quiesceIngest(IngestFile type, CheckpointSignature sig) 
    throws IOException {  
  File edits; 
  InjectionHandler.processEvent(InjectionEvent.STANDBY_QUIESCE_INGEST);
  synchronized (ingestStateLock) {
    if (type == IngestFile.EDITS) {
      assertState(StandbyIngestState.INGESTING_EDITS,
          StandbyIngestState.QUIESCING_EDITS);
    } else {
      assertState(StandbyIngestState.INGESTING_EDITS_NEW,
          StandbyIngestState.QUIESCING_EDITS_NEW);
    }
    edits = getIngestFile(type);
    currentIngestState = (type == IngestFile.EDITS)
        ? StandbyIngestState.QUIESCING_EDITS
        : StandbyIngestState.QUIESCING_EDITS_NEW;
    ingest.quiesce(sig);       
  } 
  try {
    ingestThread.join();
    currentIngestState = StandbyIngestState.NOT_INGESTING;
    LOG.info("Standby: Quiesce - Ingest thread for " 
        + edits.getName() + " exited.");
  } catch (InterruptedException e) {
    LOG.info("Standby: Quiesce - Ingest thread interrupted.");
    throw new IOException(e.getMessage());
  }
}
 

public static CheckpointSignatureProto convert(CheckpointSignature s) {
  return CheckpointSignatureProto.newBuilder()
      .setBlockPoolId(s.getBlockpoolID())
      .setCurSegmentTxId(s.getCurSegmentTxId())
      .setMostRecentCheckpointTxId(s.getMostRecentCheckpointTxId())
      .setStorageInfo(PBHelper.convert((StorageInfo) s)).build();
}
 

@Test
public void testConvertCheckpointSignature() {
  CheckpointSignature s = new CheckpointSignature(
      getStorageInfo(NodeType.NAME_NODE), "bpid", 100, 1);
  CheckpointSignatureProto sProto = PBHelper.convert(s);
  CheckpointSignature s1 = PBHelper.convert(sProto);
  assertEquals(s.getBlockpoolID(), s1.getBlockpoolID());
  assertEquals(s.getClusterID(), s1.getClusterID());
  assertEquals(s.getCTime(), s1.getCTime());
  assertEquals(s.getCurSegmentTxId(), s1.getCurSegmentTxId());
  assertEquals(s.getLayoutVersion(), s1.getLayoutVersion());
  assertEquals(s.getMostRecentCheckpointTxId(),
      s1.getMostRecentCheckpointTxId());
  assertEquals(s.getNamespaceID(), s1.getNamespaceID());
}
 

@Override
public RollEditLogResponseProto rollEditLog(RpcController unused,
    RollEditLogRequestProto request) throws ServiceException {
  CheckpointSignature signature;
  try {
    signature = impl.rollEditLog();
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  return RollEditLogResponseProto.newBuilder()
      .setSignature(PBHelper.convert(signature)).build();
}
 

@Override
public CheckpointSignature rollEditLog() throws IOException {
  try {
    return PBHelper.convert(rpcProxy.rollEditLog(NULL_CONTROLLER,
        VOID_ROLL_EDITLOG_REQUEST).getSignature());
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

@Override
public void endCheckpoint(NamenodeRegistration registration,
    CheckpointSignature sig) throws IOException {
  EndCheckpointRequestProto req = EndCheckpointRequestProto.newBuilder()
      .setRegistration(PBHelper.convert(registration))
      .setSignature(PBHelper.convert(sig)).build();
  try {
    rpcProxy.endCheckpoint(NULL_CONTROLLER, req);
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

/**
 * Test for the case where the shared edits dir doesn't have
 * all of the recent edit logs.
 */
@Test
public void testSharedEditsMissingLogs() throws Exception {
  removeStandbyNameDirs();
  
  CheckpointSignature sig = nn0.getRpcServer().rollEditLog();
  assertEquals(3, sig.getCurSegmentTxId());
  
  // Should have created edits_1-2 in shared edits dir
  URI editsUri = cluster.getSharedEditsDir(0, 1);
  File editsDir = new File(editsUri);
  File editsSegment = new File(new File(editsDir, "current"),
      NNStorage.getFinalizedEditsFileName(1, 2));
  GenericTestUtils.assertExists(editsSegment);

  // Delete the segment.
  assertTrue(editsSegment.delete());
  
  // Trying to bootstrap standby should now fail since the edit
  // logs aren't available in the shared dir.
  LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(
      LogFactory.getLog(BootstrapStandby.class));
  try {
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(BootstrapStandby.ERR_CODE_LOGS_UNAVAILABLE, rc);
  } finally {
    logs.stopCapturing();
  }
  GenericTestUtils.assertMatches(logs.getOutput(),
      "FATAL.*Unable to read transaction ids 1-3 from the configured shared");
}
 

@Test
public void testConvertCheckpointSignature() {
  CheckpointSignature s = new CheckpointSignature(
      getStorageInfo(NodeType.NAME_NODE), "bpid", 100, 1);
  CheckpointSignatureProto sProto = PBHelper.convert(s);
  CheckpointSignature s1 = PBHelper.convert(sProto);
  assertEquals(s.getBlockpoolID(), s1.getBlockpoolID());
  assertEquals(s.getClusterID(), s1.getClusterID());
  assertEquals(s.getCTime(), s1.getCTime());
  assertEquals(s.getCurSegmentTxId(), s1.getCurSegmentTxId());
  assertEquals(s.getLayoutVersion(), s1.getLayoutVersion());
  assertEquals(s.getMostRecentCheckpointTxId(),
      s1.getMostRecentCheckpointTxId());
  assertEquals(s.getNamespaceID(), s1.getNamespaceID());
}
 

/**
 * Test for the case where the shared edits dir doesn't have
 * all of the recent edit logs.
 */
@Test
public void testSharedEditsMissingLogs() throws Exception {
  removeStandbyNameDirs();
  
  CheckpointSignature sig = nn0.getRpcServer().rollEditLog();
  assertEquals(3, sig.getCurSegmentTxId());
  
  // Should have created edits_1-2 in shared edits dir
  URI editsUri = cluster.getSharedEditsDir(0, 1);
  File editsDir = new File(editsUri);
  File editsSegment = new File(new File(editsDir, "current"),
      NNStorage.getFinalizedEditsFileName(1, 2));
  GenericTestUtils.assertExists(editsSegment);

  // Delete the segment.
  assertTrue(editsSegment.delete());
  
  // Trying to bootstrap standby should now fail since the edit
  // logs aren't available in the shared dir.
  LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(
      LogFactory.getLog(BootstrapStandby.class));
  try {
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(BootstrapStandby.ERR_CODE_LOGS_UNAVAILABLE, rc);
  } finally {
    logs.stopCapturing();
  }
  GenericTestUtils.assertMatches(logs.getOutput(),
      "FATAL.*Unable to read transaction ids 1-3 from the configured shared");
}
 

@Override
public void endCheckpoint(NamenodeRegistration registration,
    CheckpointSignature sig) throws IOException {
  EndCheckpointRequestProto req = EndCheckpointRequestProto.newBuilder()
      .setRegistration(PBHelper.convert(registration))
      .setSignature(PBHelper.convert(sig)).build();
  try {
    rpcProxy.endCheckpoint(NULL_CONTROLLER, req);
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

@Override
public CheckpointSignature rollEditLog() throws IOException {
  try {
    return PBHelper.convert(rpcProxy.rollEditLog(NULL_CONTROLLER,
        VOID_ROLL_EDITLOG_REQUEST).getSignature());
  } catch (ServiceException e) {
    throw ProtobufHelper.getRemoteException(e);
  }
}
 

@Override
public RollEditLogResponseProto rollEditLog(RpcController unused,
    RollEditLogRequestProto request) throws ServiceException {
  CheckpointSignature signature;
  try {
    signature = impl.rollEditLog();
  } catch (IOException e) {
    throw new ServiceException(e);
  }
  return RollEditLogResponseProto.newBuilder()
      .setSignature(PBHelper.convert(signature)).build();
}
 

public static CheckpointSignatureProto convert(CheckpointSignature s) {
  return CheckpointSignatureProto.newBuilder()
      .setBlockPoolId(s.getBlockpoolID())
      .setCurSegmentTxId(s.getCurSegmentTxId())
      .setMostRecentCheckpointTxId(s.getMostRecentCheckpointTxId())
      .setStorageInfo(PBHelper.convert((StorageInfo) s)).build();
}
 

public CheckpointCommand(CheckpointSignature sig,
                         boolean needToReturnImg) {
  super(NamenodeProtocol.ACT_CHECKPOINT);
  this.cSig = sig;
  this.needToReturnImage = needToReturnImg;
}
 

private void finalizeCheckpoint(CheckpointSignature sig) 
    throws IOException{

  File[] imageFiles = fsImage.getImageFiles();
  if (imageFiles.length == 0) {
    throw new IOException("No good image is left");
  }
  File imageFile = imageFiles[0];
  InjectionHandler.processEvent(InjectionEvent.STANDBY_BEFORE_PUT_IMAGE,
      imageFile);
  
  // start a thread to validate image while uploading the image to primary
  ImageValidator imageValidator = new ImageValidator(imageFile);
  imageValidator.run();
  
  // copy image to primary namenode
  LOG.info("Standby: Checkpointing - Upload fsimage to remote namenode.");
  checkpointStatus("Image upload started");
  putFSImage(sig);

  // check if the image is valid
  try {
    imageValidator.join();
  } catch (InterruptedException ie) {
    throw (IOException)new InterruptedIOException().initCause(ie);
  }
  if (!imageValidator.succeeded) {
    throw new IOException("Image file validation failed", imageValidator.error);
  }
  
  // make transaction to primary namenode to switch edit logs
  LOG.info("Standby: Checkpointing - Roll fsimage on primary namenode.");
  InjectionHandler.processEventIO(InjectionEvent.STANDBY_BEFORE_ROLL_IMAGE);
    
  assertState(
      StandbyIngestState.NOT_INGESTING,
      StandbyIngestState.INGESTING_EDITS_NEW);
  
  // we might concurrently reopen ingested file because of 
  // checksum error
  synchronized (ingestStateLock) {
    boolean editsNewExisted = editsNewExists();
    try {
      primaryNamenode.rollFsImage(new CheckpointSignature(fsImage));
    } catch (IOException e) {
      if (editsNewExisted && !editsNewExists()
          && currentIngestState == StandbyIngestState.INGESTING_EDITS_NEW) {
        // we were ingesting edits.new
        // the roll did not succeed but edits.new does not exist anymore          
        // assume that the roll succeeded   
        LOG.warn("Roll did not succeed but edits.new does not exist!!! - assuming roll succeeded", e);
      } else {
        throw e;
      }
    }
    // after successful roll edits.new is rolled to edits
    // and we should be consuming it
    setCurrentIngestFile(editsFile);
    if (currentIngestState == StandbyIngestState.INGESTING_EDITS_NEW) {
      // 1) We currently consume edits.new - do the swap
      currentIngestState = StandbyIngestState.INGESTING_EDITS;
    } // 2) otherwise we don't consume anything - do not change the state
  }
  setLastRollSignature(null);
  lastFinalizeCheckpointFailed = false;
  
  LOG.info("Standby: Checkpointing - Checkpoint done. New Image Size: "
      + fsImage.getFsImageName().length());
  checkpointStatus("Completed");
}
 

public static CheckpointSignature convert(CheckpointSignatureProto s) {
  StorageInfo si = PBHelper.convert(s.getStorageInfo(), NodeType.NAME_NODE);
  return new CheckpointSignature(si, s.getBlockPoolId(),
      s.getMostRecentCheckpointTxId(), s.getCurSegmentTxId());
}
 

public void setLastRollSignature(CheckpointSignature sig) {
  this.sig = sig;
}
 

public CheckpointSignature getLastRollSignature() {
  return this.sig;
}
 

public CheckpointCommand(CheckpointSignature sig,
                         boolean needToReturnImg) {
  super(NamenodeProtocol.ACT_CHECKPOINT);
  this.cSig = sig;
  this.needToReturnImage = needToReturnImg;
}
 

public static CheckpointSignature convert(CheckpointSignatureProto s) {
  StorageInfo si = PBHelper.convert(s.getStorageInfo(), NodeType.NAME_NODE);
  return new CheckpointSignature(si, s.getBlockPoolId(),
      s.getMostRecentCheckpointTxId(), s.getCurSegmentTxId());
}
 

/**
 * Closes the current edit log and opens a new one. The 
 * call fails if the file system is in SafeMode.
 * @throws IOException
 * @return a unique token to identify this transaction.
 */
public CheckpointSignature rollEditLog() throws IOException;
 

/**
 * Rolls the fsImage log. It removes the old fsImage, copies the
 * new image to fsImage, removes the old edits and renames edits.new 
 * to edits. The call fails if any of the four files are missing.
 * 
 * @param newImageSignature the signature of the new fsimage
 * @throws IOException
 */
public void rollFsImage(CheckpointSignature newImageSignature)
throws IOException;
 

/**
 * Gets the CheckpointSignature at the time the call was made
 * @return the CheckpointSignature
 */
public CheckpointSignature getCheckpointSignature();
 

/**
 * Closes the current edit log and opens a new one. The 
 * call fails if the file system is in SafeMode.
 * @throws IOException
 * @return a unique token to identify this transaction.
 */
public CheckpointSignature rollEditLog() throws IOException;
 

/**
 * A request to the active name-node to finalize
 * previously started checkpoint.
 * 
 * @param registration the requesting node
 * @param sig {@code CheckpointSignature} which identifies the checkpoint.
 * @throws IOException
 */
@AtMostOnce
public void endCheckpoint(NamenodeRegistration registration,
                          CheckpointSignature sig) throws IOException;
 

/**
 * Closes the current edit log and opens a new one. The 
 * call fails if the file system is in SafeMode.
 * @throws IOException
 * @return a unique token to identify this transaction.
 */
@Idempotent
public CheckpointSignature rollEditLog() throws IOException;
 

/**
 * Checkpoint signature is used to ensure 
 * that nodes are talking about the same checkpoint.
 */
public CheckpointSignature getSignature() {
  return cSig;
}
 

/**
 * A request to the active name-node to finalize
 * previously started checkpoint.
 * 
 * @param registration the requesting node
 * @param sig {@code CheckpointSignature} which identifies the checkpoint.
 * @throws IOException
 */
@AtMostOnce
public void endCheckpoint(NamenodeRegistration registration,
                          CheckpointSignature sig) throws IOException;
 

/**
 * Closes the current edit log and opens a new one. The 
 * call fails if the file system is in SafeMode.
 * @throws IOException
 * @return a unique token to identify this transaction.
 */
@Idempotent
public CheckpointSignature rollEditLog() throws IOException;