Java Code Examples for org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster#shutdown()

@Test(timeout = 120000)
public void testNNFailover() throws IOException, URISyntaxException,
    MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    DFSClient client = ((DistributedFileSystem) HATestUtil.configureFailoverFs
        (cluster.getDfsCluster(), conf)).dfs;
    DFSInotifyEventInputStream eis = client.getInotifyEventStream();
    for (int i = 0; i < 10; i++) {
      client.mkdirs("/dir" + i, null, false);
    }
    cluster.getDfsCluster().shutdownNameNode(0);
    cluster.getDfsCluster().transitionToActive(1);
    EventBatch batch = null;
    // we can read all of the edits logged by the old active from the new
    // active
    for (int i = 0; i < 10; i++) {
      batch = waitForNextEvents(eis);
      Assert.assertEquals(1, batch.getEvents().length);
      Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
      Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" +
          i));
    }
    Assert.assertTrue(eis.poll() == null);
  } finally {
    cluster.shutdown();
  }
}
 

@Test(timeout = 120000)
public void testReadEventsWithTimeout() throws IOException,
    InterruptedException, MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    final DFSClient client = new DFSClient(cluster.getDfsCluster()
        .getNameNode(0).getNameNodeAddress(), conf);
    DFSInotifyEventInputStream eis = client.getInotifyEventStream();
    ScheduledExecutorService ex = Executors
        .newSingleThreadScheduledExecutor();
    ex.schedule(new Runnable() {
      @Override
      public void run() {
        try {
          client.mkdirs("/dir", null, false);
        } catch (IOException e) {
          // test will fail
          LOG.error("Unable to create /dir", e);
        }
      }
    }, 1, TimeUnit.SECONDS);
    // a very generous wait period -- the edit will definitely have been
    // processed by the time this is up
    EventBatch batch = eis.poll(5, TimeUnit.SECONDS);
    Assert.assertNotNull(batch);
    Assert.assertEquals(1, batch.getEvents().length);
    Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
    Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath());
  } finally {
    cluster.shutdown();
  }
}
 

@Test (timeout = 300000)
public void testQuery() throws Exception {
  final Configuration conf = new Configuration();
  MiniQJMHACluster cluster = null;
  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);

    dfsCluster.shutdownNameNode(1);

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());

    info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY);
    Assert.assertFalse(info.createdRollbackImages());

    dfsCluster.restartNameNode(1);

    queryForPreparation(dfs);

    // The NN should have a copy of the fsimage in case of rollbacks.
    Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage()
        .hasRollbackFSImage());
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test (timeout = 300000)
public void testQuery() throws Exception {
  final Configuration conf = new Configuration();
  MiniQJMHACluster cluster = null;
  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);

    dfsCluster.shutdownNameNode(1);

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());

    info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY);
    Assert.assertFalse(info.createdRollbackImages());

    dfsCluster.restartNameNode(1);

    queryForPreparation(dfs);

    // The NN should have a copy of the fsimage in case of rollbacks.
    Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage()
        .hasRollbackFSImage());
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test(timeout = 120000)
public void testReadEventsWithTimeout() throws IOException,
    InterruptedException, MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    final DFSClient client = new DFSClient(cluster.getDfsCluster()
        .getNameNode(0).getNameNodeAddress(), conf);
    DFSInotifyEventInputStream eis = client.getInotifyEventStream();
    ScheduledExecutorService ex = Executors
        .newSingleThreadScheduledExecutor();
    ex.schedule(new Runnable() {
      @Override
      public void run() {
        try {
          client.mkdirs("/dir", null, false);
        } catch (IOException e) {
          // test will fail
          LOG.error("Unable to create /dir", e);
        }
      }
    }, 1, TimeUnit.SECONDS);
    // a very generous wait period -- the edit will definitely have been
    // processed by the time this is up
    EventBatch batch = eis.poll(5, TimeUnit.SECONDS);
    Assert.assertNotNull(batch);
    Assert.assertEquals(1, batch.getEvents().length);
    Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
    Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath());
  } finally {
    cluster.shutdown();
  }
}
 

@Test(timeout = 120000)
public void testNNFailover() throws IOException, URISyntaxException,
    MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    DFSClient client = ((DistributedFileSystem) HATestUtil.configureFailoverFs
        (cluster.getDfsCluster(), conf)).dfs;
    DFSInotifyEventInputStream eis = client.getInotifyEventStream();
    for (int i = 0; i < 10; i++) {
      client.mkdirs("/dir" + i, null, false);
    }
    cluster.getDfsCluster().shutdownNameNode(0);
    cluster.getDfsCluster().transitionToActive(1);
    EventBatch batch = null;
    // we can read all of the edits logged by the old active from the new
    // active
    for (int i = 0; i < 10; i++) {
      batch = waitForNextEvents(eis);
      Assert.assertEquals(1, batch.getEvents().length);
      Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
      Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" +
          i));
    }
    Assert.assertTrue(eis.poll() == null);
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * Make sure that an HA NN can successfully upgrade when configured using
 * JournalNodes.
 */
@Test
public void testUpgradeWithJournalNodes() throws IOException,
    URISyntaxException {
  MiniQJMHACluster qjCluster = null;
  FileSystem fs = null;
  try {
    Builder builder = new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder()
        .numDataNodes(0);
    qjCluster = builder.build();

    MiniDFSCluster cluster = qjCluster.getDfsCluster();
    
    // No upgrade is in progress at the moment.
    checkJnPreviousDirExistence(qjCluster, false);
    checkClusterPreviousDirExistence(cluster, false);
    assertCTimesEqual(cluster);
    
    // Transition NN0 to active and do some FS ops.
    cluster.transitionToActive(0);
    fs = HATestUtil.configureFailoverFs(cluster, conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));

    // get the value of the committedTxnId in journal nodes
    final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster);

    // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
    // flag.
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0, false);
    
    checkNnPreviousDirExistence(cluster, 0, true);
    checkNnPreviousDirExistence(cluster, 1, false);
    checkJnPreviousDirExistence(qjCluster, true);

    assertTrue(cidBeforeUpgrade <= getCommittedTxnIdValue(qjCluster));
    
    // NN0 should come up in the active state when given the -upgrade option,
    // so no need to transition it to active.
    assertTrue(fs.mkdirs(new Path("/foo2")));
    
    // Restart NN0 without the -upgrade flag, to make sure that works.
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
    cluster.restartNameNode(0, false);
    
    // Make sure we can still do FS ops after upgrading.
    cluster.transitionToActive(0);
    assertTrue(fs.mkdirs(new Path("/foo3")));

    assertTrue(getCommittedTxnIdValue(qjCluster) > cidBeforeUpgrade);
    
    // Now bootstrap the standby with the upgraded info.
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(0, rc);
    
    // Now restart NN1 and make sure that we can do ops against that as well.
    cluster.restartNameNode(1);
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    assertTrue(fs.mkdirs(new Path("/foo4")));
    
    assertCTimesEqual(cluster);
  } finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}
 

@Test(timeout = 300000)
public void testDowngrade() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());
    dfs.mkdirs(bar);

    TestRollingUpgrade.queryForPreparation(dfs);
    dfs.close();

    dfsCluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade");
    // Once downgraded, there should be no more fsimage for rollbacks.
    Assert.assertFalse(dfsCluster.getNamesystem(0).getFSImage()
        .hasRollbackFSImage());
    // shutdown NN1
    dfsCluster.shutdownNameNode(1);
    dfsCluster.transitionToActive(0);

    dfs = dfsCluster.getFileSystem(0);
    Assert.assertTrue(dfs.exists(foo));
    Assert.assertTrue(dfs.exists(bar));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test(timeout = 300000)
public void testCheckpoint() throws IOException, InterruptedException {
  final Configuration conf = new Configuration();
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);

  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());

    queryForPreparation(dfs);

    dfs.mkdirs(foo);
    long txid = dfs.rollEdits();
    Assert.assertTrue(txid > 0);

    int retries = 0;
    while (++retries < 5) {
      NNStorage storage = dfsCluster.getNamesystem(1).getFSImage()
          .getStorage();
      if (storage.getFsImageName(txid - 1) != null) {
        return;
      }
      Thread.sleep(1000);
    }
    Assert.fail("new checkpoint does not exist");

  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test (timeout = 300000)
public void testFinalize() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    FSImage fsimage = dfsCluster.getNamesystem(0).getFSImage();

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());
    dfs.mkdirs(bar);

    queryForPreparation(dfs);

    // The NN should have a copy of the fsimage in case of rollbacks.
    Assert.assertTrue(fsimage.hasRollbackFSImage());

    info = dfs.rollingUpgrade(RollingUpgradeAction.FINALIZE);
    Assert.assertTrue(info.isFinalized());
    Assert.assertTrue(dfs.exists(foo));

    // Once finalized, there should be no more fsimage for rollbacks.
    Assert.assertFalse(fsimage.hasRollbackFSImage());

    // Should have no problem in restart and replaying edits that include
    // the FINALIZE op.
    dfsCluster.restartNameNode(0);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test(timeout = 120000)
public void testTwoActiveNNs() throws IOException, MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    DFSClient client0 = new DFSClient(cluster.getDfsCluster().getNameNode(0)
        .getNameNodeAddress(), conf);
    DFSClient client1 = new DFSClient(cluster.getDfsCluster().getNameNode(1)
        .getNameNodeAddress(), conf);
    DFSInotifyEventInputStream eis = client0.getInotifyEventStream();
    for (int i = 0; i < 10; i++) {
      client0.mkdirs("/dir" + i, null, false);
    }

    cluster.getDfsCluster().transitionToActive(1);
    for (int i = 10; i < 20; i++) {
      client1.mkdirs("/dir" + i, null, false);
    }

    // make sure that the old active can't read any further than the edits
    // it logged itself (it has no idea whether the in-progress edits from
    // the other writer have actually been committed)
    EventBatch batch = null;
    for (int i = 0; i < 10; i++) {
      batch = waitForNextEvents(eis);
      Assert.assertEquals(1, batch.getEvents().length);
      Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
      Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" +
          i));
    }
    Assert.assertTrue(eis.poll() == null);
  } finally {
    try {
      cluster.shutdown();
    } catch (ExitUtil.ExitException e) {
      // expected because the old active will be unable to flush the
      // end-of-segment op since it is fenced
    }
  }
}
 

@Test(timeout = 300000)
public void testDowngrade() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());
    dfs.mkdirs(bar);

    TestRollingUpgrade.queryForPreparation(dfs);
    dfs.close();

    dfsCluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade");
    // Once downgraded, there should be no more fsimage for rollbacks.
    Assert.assertFalse(dfsCluster.getNamesystem(0).getFSImage()
        .hasRollbackFSImage());
    // shutdown NN1
    dfsCluster.shutdownNameNode(1);
    dfsCluster.transitionToActive(0);

    dfs = dfsCluster.getFileSystem(0);
    Assert.assertTrue(dfs.exists(foo));
    Assert.assertTrue(dfs.exists(bar));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/**
 * Test rollback scenarios where StandbyNameNode does checkpoints during
 * rolling upgrade.
 */
@Test
public void testRollbackWithHAQJM() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    // start rolling upgrade
    RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());

    // create new directory
    dfs.mkdirs(bar);
    dfs.close();

    TestRollingUpgrade.queryForPreparation(dfs);

    // If the query returns true, both active and the standby NN should have
    // rollback fsimage ready.
    Assert.assertTrue(dfsCluster.getNameNode(0).getFSImage()
        .hasRollbackFSImage());
    Assert.assertTrue(dfsCluster.getNameNode(1).getFSImage()
        .hasRollbackFSImage());
    
    // rollback NN0
    dfsCluster.restartNameNode(0, true, "-rollingUpgrade",
        "rollback");
    // shutdown NN1
    dfsCluster.shutdownNameNode(1);
    dfsCluster.transitionToActive(0);

    // make sure /foo is still there, but /bar is not
    dfs = dfsCluster.getFileSystem(0);
    Assert.assertTrue(dfs.exists(foo));
    Assert.assertFalse(dfs.exists(bar));

    // check the details of NNStorage
    NNStorage storage = dfsCluster.getNamesystem(0).getFSImage()
        .getStorage();
    // segments:(startSegment, mkdir, start upgrade endSegment), 
    // (startSegment, mkdir, endSegment)
    checkNNStorage(storage, 4, 7);

    // check storage in JNs
    for (int i = 0; i < NUM_JOURNAL_NODES; i++) {
      File dir = cluster.getJournalCluster().getCurrentDir(0,
          MiniQJMHACluster.NAMESERVICE);
      checkJNStorage(dir, 5, 7);
    }

    // restart NN0 again to make sure we can start using the new fsimage and
    // the corresponding md5 checksum
    dfsCluster.restartNameNode(0);
    // start the rolling upgrade again to make sure we do not load upgrade
    // status after the rollback
    dfsCluster.transitionToActive(0);
    dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test
public void testRollbackWithJournalNodes() throws IOException,
    URISyntaxException {
  MiniQJMHACluster qjCluster = null;
  FileSystem fs = null;
  try {
    Builder builder = new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder()
        .numDataNodes(0);
    qjCluster = builder.build();

    MiniDFSCluster cluster = qjCluster.getDfsCluster();
    
    // No upgrade is in progress at the moment.
    checkClusterPreviousDirExistence(cluster, false);
    assertCTimesEqual(cluster);
    checkJnPreviousDirExistence(qjCluster, false);
    
    // Transition NN0 to active and do some FS ops.
    cluster.transitionToActive(0);
    fs = HATestUtil.configureFailoverFs(cluster, conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));

    final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster);

    // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
    // flag.
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0, false);
    
    checkNnPreviousDirExistence(cluster, 0, true);
    checkNnPreviousDirExistence(cluster, 1, false);
    checkJnPreviousDirExistence(qjCluster, true);
    
    // NN0 should come up in the active state when given the -upgrade option,
    // so no need to transition it to active.
    assertTrue(fs.mkdirs(new Path("/foo2")));

    final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster);
    assertTrue(cidDuringUpgrade > cidBeforeUpgrade);

    // Now bootstrap the standby with the upgraded info.
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(0, rc);
    
    cluster.restartNameNode(1);
    
    checkNnPreviousDirExistence(cluster, 0, true);
    checkNnPreviousDirExistence(cluster, 1, true);
    checkJnPreviousDirExistence(qjCluster, true);
    assertCTimesEqual(cluster);
    
    // Shut down the NNs, but deliberately leave the JNs up and running.
    Collection<URI> nn1NameDirs = cluster.getNameDirs(0);
    cluster.shutdown();

    conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs));
    NameNode.doRollback(conf, false);

    final long cidAfterRollback = getCommittedTxnIdValue(qjCluster);
    assertTrue(cidBeforeUpgrade < cidAfterRollback);
    // make sure the committedTxnId has been reset correctly after rollback
    assertTrue(cidDuringUpgrade > cidAfterRollback);

    // The rollback operation should have rolled back the first NN's local
    // dirs, and the shared dir, but not the other NN's dirs. Those have to be
    // done by bootstrapping the standby.
    checkNnPreviousDirExistence(cluster, 0, false);
    checkJnPreviousDirExistence(qjCluster, false);
  } finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}
 

@Test
public void testFinalizeWithJournalNodes() throws IOException,
    URISyntaxException {
  MiniQJMHACluster qjCluster = null;
  FileSystem fs = null;
  try {
    Builder builder = new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder()
        .numDataNodes(0);
    qjCluster = builder.build();

    MiniDFSCluster cluster = qjCluster.getDfsCluster();
    
    // No upgrade is in progress at the moment.
    checkJnPreviousDirExistence(qjCluster, false);
    checkClusterPreviousDirExistence(cluster, false);
    assertCTimesEqual(cluster);
    
    // Transition NN0 to active and do some FS ops.
    cluster.transitionToActive(0);
    fs = HATestUtil.configureFailoverFs(cluster, conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));

    final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster);
    
    // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
    // flag.
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0, false);
    assertTrue(cidBeforeUpgrade <= getCommittedTxnIdValue(qjCluster));
    
    assertTrue(fs.mkdirs(new Path("/foo2")));

    checkNnPreviousDirExistence(cluster, 0, true);
    checkNnPreviousDirExistence(cluster, 1, false);
    checkJnPreviousDirExistence(qjCluster, true);
    
    // Now bootstrap the standby with the upgraded info.
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(0, rc);
    
    cluster.restartNameNode(1);

    final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster);
    assertTrue(cidDuringUpgrade > cidBeforeUpgrade);

    runFinalizeCommand(cluster);

    assertEquals(cidDuringUpgrade, getCommittedTxnIdValue(qjCluster));
    checkClusterPreviousDirExistence(cluster, false);
    checkJnPreviousDirExistence(qjCluster, false);
    assertCTimesEqual(cluster);
  } finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}
 

@Test
public void testFinalizeWithJournalNodes() throws IOException,
    URISyntaxException {
  MiniQJMHACluster qjCluster = null;
  FileSystem fs = null;
  try {
    Builder builder = new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder()
        .numDataNodes(0);
    qjCluster = builder.build();

    MiniDFSCluster cluster = qjCluster.getDfsCluster();
    
    // No upgrade is in progress at the moment.
    checkJnPreviousDirExistence(qjCluster, false);
    checkClusterPreviousDirExistence(cluster, false);
    assertCTimesEqual(cluster);
    
    // Transition NN0 to active and do some FS ops.
    cluster.transitionToActive(0);
    fs = HATestUtil.configureFailoverFs(cluster, conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));

    final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster);
    
    // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
    // flag.
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0, false);
    assertTrue(cidBeforeUpgrade <= getCommittedTxnIdValue(qjCluster));
    
    assertTrue(fs.mkdirs(new Path("/foo2")));

    checkNnPreviousDirExistence(cluster, 0, true);
    checkNnPreviousDirExistence(cluster, 1, false);
    checkJnPreviousDirExistence(qjCluster, true);
    
    // Now bootstrap the standby with the upgraded info.
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(0, rc);
    
    cluster.restartNameNode(1);

    final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster);
    assertTrue(cidDuringUpgrade > cidBeforeUpgrade);

    runFinalizeCommand(cluster);

    assertEquals(cidDuringUpgrade, getCommittedTxnIdValue(qjCluster));
    checkClusterPreviousDirExistence(cluster, false);
    checkJnPreviousDirExistence(qjCluster, false);
    assertCTimesEqual(cluster);
  } finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}
 

/**
 * Make sure that even if the NN which initiated the upgrade is in the standby
 * state that we're allowed to finalize.
 */
@Test
public void testFinalizeFromSecondNameNodeWithJournalNodes()
    throws IOException, URISyntaxException {
  MiniQJMHACluster qjCluster = null;
  FileSystem fs = null;
  try {
    Builder builder = new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder()
        .numDataNodes(0);
    qjCluster = builder.build();

    MiniDFSCluster cluster = qjCluster.getDfsCluster();
    
    // No upgrade is in progress at the moment.
    checkJnPreviousDirExistence(qjCluster, false);
    checkClusterPreviousDirExistence(cluster, false);
    assertCTimesEqual(cluster);
    
    // Transition NN0 to active and do some FS ops.
    cluster.transitionToActive(0);
    fs = HATestUtil.configureFailoverFs(cluster, conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    
    // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
    // flag.
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0, false);
    
    checkNnPreviousDirExistence(cluster, 0, true);
    checkNnPreviousDirExistence(cluster, 1, false);
    checkJnPreviousDirExistence(qjCluster, true);
    
    // Now bootstrap the standby with the upgraded info.
    int rc = BootstrapStandby.run(
        new String[]{"-force"},
        cluster.getConfiguration(1));
    assertEquals(0, rc);
    
    cluster.restartNameNode(1);
    
    // Make the second NN (not the one that initiated the upgrade) active when
    // the finalize command is run.
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    
    runFinalizeCommand(cluster);
    
    checkClusterPreviousDirExistence(cluster, false);
    checkJnPreviousDirExistence(qjCluster, false);
    assertCTimesEqual(cluster);
  } finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}
 

/**
 * Test rollback scenarios where StandbyNameNode does checkpoints during
 * rolling upgrade.
 */
@Test
public void testRollbackWithHAQJM() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    // start rolling upgrade
    RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());

    // create new directory
    dfs.mkdirs(bar);
    dfs.close();

    TestRollingUpgrade.queryForPreparation(dfs);

    // If the query returns true, both active and the standby NN should have
    // rollback fsimage ready.
    Assert.assertTrue(dfsCluster.getNameNode(0).getFSImage()
        .hasRollbackFSImage());
    Assert.assertTrue(dfsCluster.getNameNode(1).getFSImage()
        .hasRollbackFSImage());
    
    // rollback NN0
    dfsCluster.restartNameNode(0, true, "-rollingUpgrade",
        "rollback");
    // shutdown NN1
    dfsCluster.shutdownNameNode(1);
    dfsCluster.transitionToActive(0);

    // make sure /foo is still there, but /bar is not
    dfs = dfsCluster.getFileSystem(0);
    Assert.assertTrue(dfs.exists(foo));
    Assert.assertFalse(dfs.exists(bar));

    // check the details of NNStorage
    NNStorage storage = dfsCluster.getNamesystem(0).getFSImage()
        .getStorage();
    // segments:(startSegment, mkdir, start upgrade endSegment), 
    // (startSegment, mkdir, endSegment)
    checkNNStorage(storage, 4, 7);

    // check storage in JNs
    for (int i = 0; i < NUM_JOURNAL_NODES; i++) {
      File dir = cluster.getJournalCluster().getCurrentDir(0,
          MiniQJMHACluster.NAMESERVICE);
      checkJNStorage(dir, 5, 7);
    }

    // restart NN0 again to make sure we can start using the new fsimage and
    // the corresponding md5 checksum
    dfsCluster.restartNameNode(0);
    // start the rolling upgrade again to make sure we do not load upgrade
    // status after the rollback
    dfsCluster.transitionToActive(0);
    dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test (timeout = 300000)
public void testFinalize() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    cluster = new MiniQJMHACluster.Builder(conf).build();
    MiniDFSCluster dfsCluster = cluster.getDfsCluster();
    dfsCluster.waitActive();

    // let NN1 tail editlog every 1s
    dfsCluster.getConfiguration(1).setInt(
        DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    dfsCluster.restartNameNode(1);

    dfsCluster.transitionToActive(0);
    DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
    dfs.mkdirs(foo);

    FSImage fsimage = dfsCluster.getNamesystem(0).getFSImage();

    // start rolling upgrade
    RollingUpgradeInfo info = dfs
        .rollingUpgrade(RollingUpgradeAction.PREPARE);
    Assert.assertTrue(info.isStarted());
    dfs.mkdirs(bar);

    queryForPreparation(dfs);

    // The NN should have a copy of the fsimage in case of rollbacks.
    Assert.assertTrue(fsimage.hasRollbackFSImage());

    info = dfs.rollingUpgrade(RollingUpgradeAction.FINALIZE);
    Assert.assertTrue(info.isFinalized());
    Assert.assertTrue(dfs.exists(foo));

    // Once finalized, there should be no more fsimage for rollbacks.
    Assert.assertFalse(fsimage.hasRollbackFSImage());

    // Should have no problem in restart and replaying edits that include
    // the FINALIZE op.
    dfsCluster.restartNameNode(0);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test(timeout = 120000)
public void testTwoActiveNNs() throws IOException, MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    DFSClient client0 = new DFSClient(cluster.getDfsCluster().getNameNode(0)
        .getNameNodeAddress(), conf);
    DFSClient client1 = new DFSClient(cluster.getDfsCluster().getNameNode(1)
        .getNameNodeAddress(), conf);
    DFSInotifyEventInputStream eis = client0.getInotifyEventStream();
    for (int i = 0; i < 10; i++) {
      client0.mkdirs("/dir" + i, null, false);
    }

    cluster.getDfsCluster().transitionToActive(1);
    for (int i = 10; i < 20; i++) {
      client1.mkdirs("/dir" + i, null, false);
    }

    // make sure that the old active can't read any further than the edits
    // it logged itself (it has no idea whether the in-progress edits from
    // the other writer have actually been committed)
    EventBatch batch = null;
    for (int i = 0; i < 10; i++) {
      batch = waitForNextEvents(eis);
      Assert.assertEquals(1, batch.getEvents().length);
      Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
      Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" +
          i));
    }
    Assert.assertTrue(eis.poll() == null);
  } finally {
    try {
      cluster.shutdown();
    } catch (ExitUtil.ExitException e) {
      // expected because the old active will be unable to flush the
      // end-of-segment op since it is fenced
    }
  }
}