org.apache.hadoop.hdfs.server.namenode.FSImage Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.namenode.FSImage. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BootstrapStandby.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private boolean checkLogsAvailableForRead(FSImage image, long imageTxId,
    long curTxIdOnOtherNode) {

  if (imageTxId == curTxIdOnOtherNode) {
    // The other node hasn't written any logs since the last checkpoint.
    // This can be the case if the NN was freshly formatted as HA, and
    // then started in standby mode, so it has no edit logs at all.
    return true;
  }
  long firstTxIdInLogs = imageTxId + 1;
  
  assert curTxIdOnOtherNode >= firstTxIdInLogs :
    "first=" + firstTxIdInLogs + " onOtherNode=" + curTxIdOnOtherNode;
  
  try {
    Collection<EditLogInputStream> streams =
      image.getEditLog().selectInputStreams(
        firstTxIdInLogs, curTxIdOnOtherNode, null, true);
    for (EditLogInputStream stream : streams) {
      IOUtils.closeStream(stream);
    }
    return true;
  } catch (IOException e) {
    String msg = "Unable to read transaction ids " +
        firstTxIdInLogs + "-" + curTxIdOnOtherNode +
        " from the configured shared edits storage " +
        Joiner.on(",").join(sharedEditsUris) + ". " +
        "Please copy these logs into the shared edits storage " + 
        "or call saveNamespace on the active node.\n" +
        "Error: " + e.getLocalizedMessage();
    if (LOG.isDebugEnabled()) {
      LOG.fatal(msg, e);
    } else {
      LOG.fatal(msg);
    }
    return false;
  }
}
Example #2
Source File: BootstrapStandby.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private boolean checkLogsAvailableForRead(FSImage image, long imageTxId,
    long curTxIdOnOtherNode) {

  if (imageTxId == curTxIdOnOtherNode) {
    // The other node hasn't written any logs since the last checkpoint.
    // This can be the case if the NN was freshly formatted as HA, and
    // then started in standby mode, so it has no edit logs at all.
    return true;
  }
  long firstTxIdInLogs = imageTxId + 1;
  
  assert curTxIdOnOtherNode >= firstTxIdInLogs :
    "first=" + firstTxIdInLogs + " onOtherNode=" + curTxIdOnOtherNode;
  
  try {
    Collection<EditLogInputStream> streams =
      image.getEditLog().selectInputStreams(
        firstTxIdInLogs, curTxIdOnOtherNode, null, true);
    for (EditLogInputStream stream : streams) {
      IOUtils.closeStream(stream);
    }
    return true;
  } catch (IOException e) {
    String msg = "Unable to read transaction ids " +
        firstTxIdInLogs + "-" + curTxIdOnOtherNode +
        " from the configured shared edits storage " +
        Joiner.on(",").join(sharedEditsUris) + ". " +
        "Please copy these logs into the shared edits storage " + 
        "or call saveNamespace on the active node.\n" +
        "Error: " + e.getLocalizedMessage();
    if (LOG.isDebugEnabled()) {
      LOG.fatal(msg, e);
    } else {
      LOG.fatal(msg);
    }
    return false;
  }
}
Example #3
Source File: EditLogTailer.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
void doTailEdits() throws IOException, InterruptedException {
  // Write lock needs to be interruptible here because the 
  // transitionToActive RPC takes the write lock before calling
  // tailer.stop() -- so if we're not interruptible, it will
  // deadlock.
  namesystem.writeLockInterruptibly();
  try {
    FSImage image = namesystem.getFSImage();

    long lastTxnId = image.getLastAppliedTxId();
    
    if (LOG.isDebugEnabled()) {
      LOG.debug("lastTxnId: " + lastTxnId);
    }
    Collection<EditLogInputStream> streams;
    try {
      streams = editLog.selectInputStreams(lastTxnId + 1, 0, null, false);
    } catch (IOException ioe) {
      // This is acceptable. If we try to tail edits in the middle of an edits
      // log roll, i.e. the last one has been finalized but the new inprogress
      // edits file hasn't been started yet.
      LOG.warn("Edits tailer failed to find any streams. Will try again " +
          "later.", ioe);
      return;
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("edit streams to load from: " + streams.size());
    }
    
    // Once we have streams to load, errors encountered are legitimate cause
    // for concern, so we don't catch them here. Simple errors reading from
    // disk are ignored.
    long editsLoaded = 0;
    try {
      editsLoaded = image.loadEdits(streams, namesystem);
    } catch (EditLogInputException elie) {
      editsLoaded = elie.getNumEditsLoaded();
      throw elie;
    } finally {
      if (editsLoaded > 0 || LOG.isDebugEnabled()) {
        LOG.info(String.format("Loaded %d edits starting from txid %d ",
            editsLoaded, lastTxnId));
      }
    }

    if (editsLoaded > 0) {
      lastLoadTimeMs = monotonicNow();
    }
    lastLoadedTxnId = image.getLastAppliedTxId();
  } finally {
    namesystem.writeUnlock();
  }
}
Example #4
Source File: StandbyCheckpointer.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private long countUncheckpointedTxns() {
  FSImage img = namesystem.getFSImage();
  return img.getLastAppliedOrWrittenTxId() -
    img.getStorage().getMostRecentCheckpointTxId();
}
Example #5
Source File: TestRollingUpgrade.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 300000)
public void testFinalize() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    FSImage fsimage = dfsCluster.getNamesystem(0).getFSImage();

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());
    dfs.mkdirs(bar);

    queryForPreparation(dfs);

    // The NN should have a copy of the fsimage in case of rollbacks.
    Assert.assertTrue(fsimage.hasRollbackFSImage());

    info = dfs.rollingUpgrade(RollingUpgradeAction.FINALIZE);
    Assert.assertTrue(info.isFinalized());
    Assert.assertTrue(dfs.exists(foo));

    // Once finalized, there should be no more fsimage for rollbacks.
    Assert.assertFalse(fsimage.hasRollbackFSImage());

    // Should have no problem in restart and replaying edits that include
    // the FINALIZE op.
    dfsCluster.restartNameNode(0);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
Example #6
Source File: TestBootstrapStandbyWithQJM.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void testUpgrade(UpgradeState state) throws Exception {
  cluster.transitionToActive(0);
  final Configuration confNN1 = cluster.getConfiguration(1);

  final File current = cluster.getNameNode(1).getFSImage().getStorage()
      .getStorageDir(0).getCurrentDir();
  final File tmp = cluster.getNameNode(1).getFSImage().getStorage()
      .getStorageDir(0).getPreviousTmp();
  // shut down nn1
  cluster.shutdownNameNode(1);

  // make NN0 in upgrade state
  FSImage fsImage0 = cluster.getNameNode(0).getNamesystem().getFSImage();
  Whitebox.setInternalState(fsImage0, "isUpgradeFinalized", false);

  switch (state) {
    case RECOVER:
      // rename the current directory to previous.tmp in nn1
      NNStorage.rename(current, tmp);
      break;
    case FORMAT:
      // rename the current directory to a random name so it's not formatted
      final File wrongPath = new File(current.getParentFile(), "wrong");
      NNStorage.rename(current, wrongPath);
      break;
    default:
      break;
  }

  int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
  assertEquals(0, rc);

  // Should have copied over the namespace from the standby
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);

  // make sure the NN1 is in upgrade state, i.e., the previous directory has
  // been successfully created
  cluster.restartNameNode(1);
  assertFalse(cluster.getNameNode(1).getNamesystem().isUpgradeFinalized());
}
Example #7
Source File: EditLogTailer.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
void doTailEdits() throws IOException, InterruptedException {
  // Write lock needs to be interruptible here because the 
  // transitionToActive RPC takes the write lock before calling
  // tailer.stop() -- so if we're not interruptible, it will
  // deadlock.
  namesystem.writeLockInterruptibly();
  try {
    FSImage image = namesystem.getFSImage();

    long lastTxnId = image.getLastAppliedTxId();
    
    if (LOG.isDebugEnabled()) {
      LOG.debug("lastTxnId: " + lastTxnId);
    }
    Collection<EditLogInputStream> streams;
    try {
      streams = editLog.selectInputStreams(lastTxnId + 1, 0, null, false);
    } catch (IOException ioe) {
      // This is acceptable. If we try to tail edits in the middle of an edits
      // log roll, i.e. the last one has been finalized but the new inprogress
      // edits file hasn't been started yet.
      LOG.warn("Edits tailer failed to find any streams. Will try again " +
          "later.", ioe);
      return;
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("edit streams to load from: " + streams.size());
    }
    
    // Once we have streams to load, errors encountered are legitimate cause
    // for concern, so we don't catch them here. Simple errors reading from
    // disk are ignored.
    long editsLoaded = 0;
    try {
      editsLoaded = image.loadEdits(streams, namesystem);
    } catch (EditLogInputException elie) {
      editsLoaded = elie.getNumEditsLoaded();
      throw elie;
    } finally {
      if (editsLoaded > 0 || LOG.isDebugEnabled()) {
        LOG.info(String.format("Loaded %d edits starting from txid %d ",
            editsLoaded, lastTxnId));
      }
    }

    if (editsLoaded > 0) {
      lastLoadTimeMs = monotonicNow();
    }
    lastLoadedTxnId = image.getLastAppliedTxId();
  } finally {
    namesystem.writeUnlock();
  }
}
Example #8
Source File: StandbyCheckpointer.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private long countUncheckpointedTxns() {
  FSImage img = namesystem.getFSImage();
  return img.getLastAppliedOrWrittenTxId() -
    img.getStorage().getMostRecentCheckpointTxId();
}
Example #9
Source File: TestRollingUpgrade.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 300000)
public void testFinalize() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    FSImage fsimage = dfsCluster.getNamesystem(0).getFSImage();

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());
    dfs.mkdirs(bar);

    queryForPreparation(dfs);

    // The NN should have a copy of the fsimage in case of rollbacks.
    Assert.assertTrue(fsimage.hasRollbackFSImage());

    info = dfs.rollingUpgrade(RollingUpgradeAction.FINALIZE);
    Assert.assertTrue(info.isFinalized());
    Assert.assertTrue(dfs.exists(foo));

    // Once finalized, there should be no more fsimage for rollbacks.
    Assert.assertFalse(fsimage.hasRollbackFSImage());

    // Should have no problem in restart and replaying edits that include
    // the FINALIZE op.
    dfsCluster.restartNameNode(0);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
Example #10
Source File: TestBootstrapStandbyWithQJM.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void testUpgrade(UpgradeState state) throws Exception {
  cluster.transitionToActive(0);
  final Configuration confNN1 = cluster.getConfiguration(1);

  final File current = cluster.getNameNode(1).getFSImage().getStorage()
      .getStorageDir(0).getCurrentDir();
  final File tmp = cluster.getNameNode(1).getFSImage().getStorage()
      .getStorageDir(0).getPreviousTmp();
  // shut down nn1
  cluster.shutdownNameNode(1);

  // make NN0 in upgrade state
  FSImage fsImage0 = cluster.getNameNode(0).getNamesystem().getFSImage();
  Whitebox.setInternalState(fsImage0, "isUpgradeFinalized", false);

  switch (state) {
    case RECOVER:
      // rename the current directory to previous.tmp in nn1
      NNStorage.rename(current, tmp);
      break;
    case FORMAT:
      // rename the current directory to a random name so it's not formatted
      final File wrongPath = new File(current.getParentFile(), "wrong");
      NNStorage.rename(current, wrongPath);
      break;
    default:
      break;
  }

  int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
  assertEquals(0, rc);

  // Should have copied over the namespace from the standby
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);

  // make sure the NN1 is in upgrade state, i.e., the previous directory has
  // been successfully created
  cluster.restartNameNode(1);
  assertFalse(cluster.getNameNode(1).getNamesystem().isUpgradeFinalized());
}