org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster Java Examples
The following examples show how to use
org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDFSAdminWithHA.java From big-c with Apache License 2.0 | 6 votes |
private void setUpHaCluster(boolean security) throws Exception { conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, security); cluster = new MiniQJMHACluster.Builder(conf).build(); setHAConf(conf, cluster.getDfsCluster().getNameNode(0).getHostAndPort(), cluster.getDfsCluster().getNameNode(1).getHostAndPort()); cluster.getDfsCluster().getNameNode(0).getHostAndPort(); admin = new DFSAdmin(); admin.setConf(conf); assertTrue(HAUtil.isHAEnabled(conf, "ns1")); originOut = System.out; originErr = System.err; System.setOut(new PrintStream(out)); System.setErr(new PrintStream(err)); }
Example #2
Source File: TestDFSAdminWithHA.java From hadoop with Apache License 2.0 | 6 votes |
private void setUpHaCluster(boolean security) throws Exception { conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, security); cluster = new MiniQJMHACluster.Builder(conf).build(); setHAConf(conf, cluster.getDfsCluster().getNameNode(0).getHostAndPort(), cluster.getDfsCluster().getNameNode(1).getHostAndPort()); cluster.getDfsCluster().getNameNode(0).getHostAndPort(); admin = new DFSAdmin(); admin.setConf(conf); assertTrue(HAUtil.isHAEnabled(conf, "ns1")); originOut = System.out; originErr = System.err; System.setOut(new PrintStream(out)); System.setErr(new PrintStream(err)); }
Example #3
Source File: TestBootstrapStandbyWithQJM.java From hadoop with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { Configuration conf = new Configuration(); // Turn off IPC client caching, so that the suite can handle // the restart of the daemons between test cases. conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).build(); cluster = miniQjmHaCluster.getDfsCluster(); jCluster = miniQjmHaCluster.getJournalCluster(); // make nn0 active cluster.transitionToActive(0); // do sth to generate in-progress edit log data DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil.configureFailoverFs(cluster, conf); dfs.mkdirs(new Path("/test2")); dfs.close(); }
Example #4
Source File: TestDFSUpgradeWithHA.java From big-c with Apache License 2.0 | 6 votes |
private static void assertEpochFilesCopied(MiniQJMHACluster jnCluster) throws IOException { for (int i = 0; i < 3; i++) { File journalDir = jnCluster.getJournalCluster().getJournalDir(i, "ns1"); File currDir = new File(journalDir, "current"); File prevDir = new File(journalDir, "previous"); for (String fileName : new String[]{ Journal.LAST_PROMISED_FILENAME, Journal.LAST_WRITER_EPOCH }) { File prevFile = new File(prevDir, fileName); // Possible the prev file doesn't exist, e.g. if there has never been a // writer before the upgrade. if (prevFile.exists()) { PersistentLongFile prevLongFile = new PersistentLongFile(prevFile, -10); PersistentLongFile currLongFile = new PersistentLongFile(new File(currDir, fileName), -11); assertTrue("Value in " + fileName + " has decreased on upgrade in " + journalDir, prevLongFile.get() <= currLongFile.get()); } } } }
Example #5
Source File: TestDFSUpgradeWithHA.java From hadoop with Apache License 2.0 | 6 votes |
private static void assertEpochFilesCopied(MiniQJMHACluster jnCluster) throws IOException { for (int i = 0; i < 3; i++) { File journalDir = jnCluster.getJournalCluster().getJournalDir(i, "ns1"); File currDir = new File(journalDir, "current"); File prevDir = new File(journalDir, "previous"); for (String fileName : new String[]{ Journal.LAST_PROMISED_FILENAME, Journal.LAST_WRITER_EPOCH }) { File prevFile = new File(prevDir, fileName); // Possible the prev file doesn't exist, e.g. if there has never been a // writer before the upgrade. if (prevFile.exists()) { PersistentLongFile prevLongFile = new PersistentLongFile(prevFile, -10); PersistentLongFile currLongFile = new PersistentLongFile(new File(currDir, fileName), -11); assertTrue("Value in " + fileName + " has decreased on upgrade in " + journalDir, prevLongFile.get() <= currLongFile.get()); } } } }
Example #6
Source File: TestBootstrapStandbyWithQJM.java From big-c with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { Configuration conf = new Configuration(); // Turn off IPC client caching, so that the suite can handle // the restart of the daemons between test cases. conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).build(); cluster = miniQjmHaCluster.getDfsCluster(); jCluster = miniQjmHaCluster.getJournalCluster(); // make nn0 active cluster.transitionToActive(0); // do sth to generate in-progress edit log data DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil.configureFailoverFs(cluster, conf); dfs.mkdirs(new Path("/test2")); dfs.close(); }
Example #7
Source File: TestDFSUpgradeWithHA.java From big-c with Apache License 2.0 | 5 votes |
private long getCommittedTxnIdValue(MiniQJMHACluster qjCluster) throws IOException { Journal journal1 = qjCluster.getJournalCluster().getJournalNode(0) .getOrCreateJournal(MiniQJMHACluster.NAMESERVICE); BestEffortLongFile committedTxnId = (BestEffortLongFile) Whitebox .getInternalState(journal1, "committedTxnId"); return committedTxnId != null ? committedTxnId.get() : HdfsConstants.INVALID_TXID; }
Example #8
Source File: TestDFSInotifyEventInputStream.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = 120000) public void testReadEventsWithTimeout() throws IOException, InterruptedException, MissingEventsException { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build(); try { cluster.getDfsCluster().waitActive(); cluster.getDfsCluster().transitionToActive(0); final DFSClient client = new DFSClient(cluster.getDfsCluster() .getNameNode(0).getNameNodeAddress(), conf); DFSInotifyEventInputStream eis = client.getInotifyEventStream(); ScheduledExecutorService ex = Executors .newSingleThreadScheduledExecutor(); ex.schedule(new Runnable() { @Override public void run() { try { client.mkdirs("/dir", null, false); } catch (IOException e) { // test will fail LOG.error("Unable to create /dir", e); } } }, 1, TimeUnit.SECONDS); // a very generous wait period -- the edit will definitely have been // processed by the time this is up EventBatch batch = eis.poll(5, TimeUnit.SECONDS); Assert.assertNotNull(batch); Assert.assertEquals(1, batch.getEvents().length); Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath()); } finally { cluster.shutdown(); } }
Example #9
Source File: TestRollingUpgrade.java From big-c with Apache License 2.0 | 5 votes |
@Test (timeout = 300000) public void testQuery() throws Exception { final Configuration conf = new Configuration(); MiniQJMHACluster cluster = null; try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); dfsCluster.shutdownNameNode(1); // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY); Assert.assertFalse(info.createdRollbackImages()); dfsCluster.restartNameNode(1); queryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage() .hasRollbackFSImage()); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #10
Source File: TestDFSInotifyEventInputStream.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = 120000) public void testNNFailover() throws IOException, URISyntaxException, MissingEventsException { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build(); try { cluster.getDfsCluster().waitActive(); cluster.getDfsCluster().transitionToActive(0); DFSClient client = ((DistributedFileSystem) HATestUtil.configureFailoverFs (cluster.getDfsCluster(), conf)).dfs; DFSInotifyEventInputStream eis = client.getInotifyEventStream(); for (int i = 0; i < 10; i++) { client.mkdirs("/dir" + i, null, false); } cluster.getDfsCluster().shutdownNameNode(0); cluster.getDfsCluster().transitionToActive(1); EventBatch batch = null; // we can read all of the edits logged by the old active from the new // active for (int i = 0; i < 10; i++) { batch = waitForNextEvents(eis); Assert.assertEquals(1, batch.getEvents().length); Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" + i)); } Assert.assertTrue(eis.poll() == null); } finally { cluster.shutdown(); } }
Example #11
Source File: TestFailureToReadEdits.java From big-c with Apache License 2.0 | 5 votes |
@Before public void setUpCluster() throws Exception { conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); HAUtil.setAllowStandbyReads(conf, true); if (clusterType == TestType.SHARED_DIR_HA) { MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) .numDataNodes(0) .checkExitOnShutdown(false) .build(); } else { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder().numDataNodes(0).checkExitOnShutdown(false); miniQjmHaCluster = builder.build(); cluster = miniQjmHaCluster.getDfsCluster(); } cluster.waitActive(); nn0 = cluster.getNameNode(0); nn1 = cluster.getNameNode(1); cluster.transitionToActive(0); fs = HATestUtil.configureFailoverFs(cluster, conf); }
Example #12
Source File: TestDFSUpgradeWithHA.java From big-c with Apache License 2.0 | 5 votes |
private static void checkJnPreviousDirExistence(MiniQJMHACluster jnCluster, boolean shouldExist) throws IOException { for (int i = 0; i < 3; i++) { checkPreviousDirExistence( jnCluster.getJournalCluster().getJournalDir(i, "ns1"), shouldExist); } if (shouldExist) { assertEpochFilesCopied(jnCluster); } }
Example #13
Source File: TestDFSUpgradeWithHA.java From hadoop with Apache License 2.0 | 5 votes |
private long getCommittedTxnIdValue(MiniQJMHACluster qjCluster) throws IOException { Journal journal1 = qjCluster.getJournalCluster().getJournalNode(0) .getOrCreateJournal(MiniQJMHACluster.NAMESERVICE); BestEffortLongFile committedTxnId = (BestEffortLongFile) Whitebox .getInternalState(journal1, "committedTxnId"); return committedTxnId != null ? committedTxnId.get() : HdfsConstants.INVALID_TXID; }
Example #14
Source File: TestDFSInotifyEventInputStream.java From hadoop with Apache License 2.0 | 5 votes |
@Test(timeout = 120000) public void testNNFailover() throws IOException, URISyntaxException, MissingEventsException { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build(); try { cluster.getDfsCluster().waitActive(); cluster.getDfsCluster().transitionToActive(0); DFSClient client = ((DistributedFileSystem) HATestUtil.configureFailoverFs (cluster.getDfsCluster(), conf)).dfs; DFSInotifyEventInputStream eis = client.getInotifyEventStream(); for (int i = 0; i < 10; i++) { client.mkdirs("/dir" + i, null, false); } cluster.getDfsCluster().shutdownNameNode(0); cluster.getDfsCluster().transitionToActive(1); EventBatch batch = null; // we can read all of the edits logged by the old active from the new // active for (int i = 0; i < 10; i++) { batch = waitForNextEvents(eis); Assert.assertEquals(1, batch.getEvents().length); Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" + i)); } Assert.assertTrue(eis.poll() == null); } finally { cluster.shutdown(); } }
Example #15
Source File: TestDFSUpgradeWithHA.java From hadoop with Apache License 2.0 | 5 votes |
private static void checkJnPreviousDirExistence(MiniQJMHACluster jnCluster, boolean shouldExist) throws IOException { for (int i = 0; i < 3; i++) { checkPreviousDirExistence( jnCluster.getJournalCluster().getJournalDir(i, "ns1"), shouldExist); } if (shouldExist) { assertEpochFilesCopied(jnCluster); } }
Example #16
Source File: TestFailureToReadEdits.java From hadoop with Apache License 2.0 | 5 votes |
@Before public void setUpCluster() throws Exception { conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); HAUtil.setAllowStandbyReads(conf, true); if (clusterType == TestType.SHARED_DIR_HA) { MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) .numDataNodes(0) .checkExitOnShutdown(false) .build(); } else { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder().numDataNodes(0).checkExitOnShutdown(false); miniQjmHaCluster = builder.build(); cluster = miniQjmHaCluster.getDfsCluster(); } cluster.waitActive(); nn0 = cluster.getNameNode(0); nn1 = cluster.getNameNode(1); cluster.transitionToActive(0); fs = HATestUtil.configureFailoverFs(cluster, conf); }
Example #17
Source File: TestRollingUpgrade.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout = 300000) public void testQuery() throws Exception { final Configuration conf = new Configuration(); MiniQJMHACluster cluster = null; try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); dfsCluster.shutdownNameNode(1); // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY); Assert.assertFalse(info.createdRollbackImages()); dfsCluster.restartNameNode(1); queryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage() .hasRollbackFSImage()); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #18
Source File: TestDFSInotifyEventInputStream.java From hadoop with Apache License 2.0 | 5 votes |
@Test(timeout = 120000) public void testReadEventsWithTimeout() throws IOException, InterruptedException, MissingEventsException { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build(); try { cluster.getDfsCluster().waitActive(); cluster.getDfsCluster().transitionToActive(0); final DFSClient client = new DFSClient(cluster.getDfsCluster() .getNameNode(0).getNameNodeAddress(), conf); DFSInotifyEventInputStream eis = client.getInotifyEventStream(); ScheduledExecutorService ex = Executors .newSingleThreadScheduledExecutor(); ex.schedule(new Runnable() { @Override public void run() { try { client.mkdirs("/dir", null, false); } catch (IOException e) { // test will fail LOG.error("Unable to create /dir", e); } } }, 1, TimeUnit.SECONDS); // a very generous wait period -- the edit will definitely have been // processed by the time this is up EventBatch batch = eis.poll(5, TimeUnit.SECONDS); Assert.assertNotNull(batch); Assert.assertEquals(1, batch.getEvents().length); Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath()); } finally { cluster.shutdown(); } }
Example #19
Source File: TestDFSUpgradeWithHA.java From big-c with Apache License 2.0 | 4 votes |
/** * Make sure that an HA NN can successfully upgrade when configured using * JournalNodes. */ @Test public void testUpgradeWithJournalNodes() throws IOException, URISyntaxException { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder() .numDataNodes(0); qjCluster = builder.build(); MiniDFSCluster cluster = qjCluster.getDfsCluster(); // No upgrade is in progress at the moment. checkJnPreviousDirExistence(qjCluster, false); checkClusterPreviousDirExistence(cluster, false); assertCTimesEqual(cluster); // Transition NN0 to active and do some FS ops. cluster.transitionToActive(0); fs = HATestUtil.configureFailoverFs(cluster, conf); assertTrue(fs.mkdirs(new Path("/foo1"))); // get the value of the committedTxnId in journal nodes final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.shutdownNameNode(1); cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); cluster.restartNameNode(0, false); checkNnPreviousDirExistence(cluster, 0, true); checkNnPreviousDirExistence(cluster, 1, false); checkJnPreviousDirExistence(qjCluster, true); assertTrue(cidBeforeUpgrade <= getCommittedTxnIdValue(qjCluster)); // NN0 should come up in the active state when given the -upgrade option, // so no need to transition it to active. assertTrue(fs.mkdirs(new Path("/foo2"))); // Restart NN0 without the -upgrade flag, to make sure that works. cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR); cluster.restartNameNode(0, false); // Make sure we can still do FS ops after upgrading. cluster.transitionToActive(0); assertTrue(fs.mkdirs(new Path("/foo3"))); assertTrue(getCommittedTxnIdValue(qjCluster) > cidBeforeUpgrade); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.run( new String[]{"-force"}, cluster.getConfiguration(1)); assertEquals(0, rc); // Now restart NN1 and make sure that we can do ops against that as well. cluster.restartNameNode(1); cluster.transitionToStandby(0); cluster.transitionToActive(1); assertTrue(fs.mkdirs(new Path("/foo4"))); assertCTimesEqual(cluster); } finally { if (fs != null) { fs.close(); } if (qjCluster != null) { qjCluster.shutdown(); } } }
Example #20
Source File: TestDFSUpgradeWithHA.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testRollbackWithJournalNodes() throws IOException, URISyntaxException { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder() .numDataNodes(0); qjCluster = builder.build(); MiniDFSCluster cluster = qjCluster.getDfsCluster(); // No upgrade is in progress at the moment. checkClusterPreviousDirExistence(cluster, false); assertCTimesEqual(cluster); checkJnPreviousDirExistence(qjCluster, false); // Transition NN0 to active and do some FS ops. cluster.transitionToActive(0); fs = HATestUtil.configureFailoverFs(cluster, conf); assertTrue(fs.mkdirs(new Path("/foo1"))); final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.shutdownNameNode(1); cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); cluster.restartNameNode(0, false); checkNnPreviousDirExistence(cluster, 0, true); checkNnPreviousDirExistence(cluster, 1, false); checkJnPreviousDirExistence(qjCluster, true); // NN0 should come up in the active state when given the -upgrade option, // so no need to transition it to active. assertTrue(fs.mkdirs(new Path("/foo2"))); final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster); assertTrue(cidDuringUpgrade > cidBeforeUpgrade); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.run( new String[]{"-force"}, cluster.getConfiguration(1)); assertEquals(0, rc); cluster.restartNameNode(1); checkNnPreviousDirExistence(cluster, 0, true); checkNnPreviousDirExistence(cluster, 1, true); checkJnPreviousDirExistence(qjCluster, true); assertCTimesEqual(cluster); // Shut down the NNs, but deliberately leave the JNs up and running. Collection<URI> nn1NameDirs = cluster.getNameDirs(0); cluster.shutdown(); conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs)); NameNode.doRollback(conf, false); final long cidAfterRollback = getCommittedTxnIdValue(qjCluster); assertTrue(cidBeforeUpgrade < cidAfterRollback); // make sure the committedTxnId has been reset correctly after rollback assertTrue(cidDuringUpgrade > cidAfterRollback); // The rollback operation should have rolled back the first NN's local // dirs, and the shared dir, but not the other NN's dirs. Those have to be // done by bootstrapping the standby. checkNnPreviousDirExistence(cluster, 0, false); checkJnPreviousDirExistence(qjCluster, false); } finally { if (fs != null) { fs.close(); } if (qjCluster != null) { qjCluster.shutdown(); } } }
Example #21
Source File: TestRollingUpgradeRollback.java From big-c with Apache License 2.0 | 4 votes |
/** * Test rollback scenarios where StandbyNameNode does checkpoints during * rolling upgrade. */ @Test public void testRollbackWithHAQJM() throws Exception { final Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); final Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); // let NN1 tail editlog every 1s dfsCluster.getConfiguration(1).setInt( DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); dfsCluster.restartNameNode(1); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); dfs.mkdirs(foo); // start rolling upgrade RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); // create new directory dfs.mkdirs(bar); dfs.close(); TestRollingUpgrade.queryForPreparation(dfs); // If the query returns true, both active and the standby NN should have // rollback fsimage ready. Assert.assertTrue(dfsCluster.getNameNode(0).getFSImage() .hasRollbackFSImage()); Assert.assertTrue(dfsCluster.getNameNode(1).getFSImage() .hasRollbackFSImage()); // rollback NN0 dfsCluster.restartNameNode(0, true, "-rollingUpgrade", "rollback"); // shutdown NN1 dfsCluster.shutdownNameNode(1); dfsCluster.transitionToActive(0); // make sure /foo is still there, but /bar is not dfs = dfsCluster.getFileSystem(0); Assert.assertTrue(dfs.exists(foo)); Assert.assertFalse(dfs.exists(bar)); // check the details of NNStorage NNStorage storage = dfsCluster.getNamesystem(0).getFSImage() .getStorage(); // segments:(startSegment, mkdir, start upgrade endSegment), // (startSegment, mkdir, endSegment) checkNNStorage(storage, 4, 7); // check storage in JNs for (int i = 0; i < NUM_JOURNAL_NODES; i++) { File dir = cluster.getJournalCluster().getCurrentDir(0, MiniQJMHACluster.NAMESERVICE); checkJNStorage(dir, 5, 7); } // restart NN0 again to make sure we can start using the new fsimage and // the corresponding md5 checksum dfsCluster.restartNameNode(0); // start the rolling upgrade again to make sure we do not load upgrade // status after the rollback dfsCluster.transitionToActive(0); dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #22
Source File: TestDFSUpgradeWithHA.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testFinalizeWithJournalNodes() throws IOException, URISyntaxException { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder() .numDataNodes(0); qjCluster = builder.build(); MiniDFSCluster cluster = qjCluster.getDfsCluster(); // No upgrade is in progress at the moment. checkJnPreviousDirExistence(qjCluster, false); checkClusterPreviousDirExistence(cluster, false); assertCTimesEqual(cluster); // Transition NN0 to active and do some FS ops. cluster.transitionToActive(0); fs = HATestUtil.configureFailoverFs(cluster, conf); assertTrue(fs.mkdirs(new Path("/foo1"))); final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.shutdownNameNode(1); cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); cluster.restartNameNode(0, false); assertTrue(cidBeforeUpgrade <= getCommittedTxnIdValue(qjCluster)); assertTrue(fs.mkdirs(new Path("/foo2"))); checkNnPreviousDirExistence(cluster, 0, true); checkNnPreviousDirExistence(cluster, 1, false); checkJnPreviousDirExistence(qjCluster, true); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.run( new String[]{"-force"}, cluster.getConfiguration(1)); assertEquals(0, rc); cluster.restartNameNode(1); final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster); assertTrue(cidDuringUpgrade > cidBeforeUpgrade); runFinalizeCommand(cluster); assertEquals(cidDuringUpgrade, getCommittedTxnIdValue(qjCluster)); checkClusterPreviousDirExistence(cluster, false); checkJnPreviousDirExistence(qjCluster, false); assertCTimesEqual(cluster); } finally { if (fs != null) { fs.close(); } if (qjCluster != null) { qjCluster.shutdown(); } } }
Example #23
Source File: TestRollingUpgradeRollback.java From hadoop with Apache License 2.0 | 4 votes |
/** * Test rollback scenarios where StandbyNameNode does checkpoints during * rolling upgrade. */ @Test public void testRollbackWithHAQJM() throws Exception { final Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); final Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); // let NN1 tail editlog every 1s dfsCluster.getConfiguration(1).setInt( DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); dfsCluster.restartNameNode(1); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); dfs.mkdirs(foo); // start rolling upgrade RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); // create new directory dfs.mkdirs(bar); dfs.close(); TestRollingUpgrade.queryForPreparation(dfs); // If the query returns true, both active and the standby NN should have // rollback fsimage ready. Assert.assertTrue(dfsCluster.getNameNode(0).getFSImage() .hasRollbackFSImage()); Assert.assertTrue(dfsCluster.getNameNode(1).getFSImage() .hasRollbackFSImage()); // rollback NN0 dfsCluster.restartNameNode(0, true, "-rollingUpgrade", "rollback"); // shutdown NN1 dfsCluster.shutdownNameNode(1); dfsCluster.transitionToActive(0); // make sure /foo is still there, but /bar is not dfs = dfsCluster.getFileSystem(0); Assert.assertTrue(dfs.exists(foo)); Assert.assertFalse(dfs.exists(bar)); // check the details of NNStorage NNStorage storage = dfsCluster.getNamesystem(0).getFSImage() .getStorage(); // segments:(startSegment, mkdir, start upgrade endSegment), // (startSegment, mkdir, endSegment) checkNNStorage(storage, 4, 7); // check storage in JNs for (int i = 0; i < NUM_JOURNAL_NODES; i++) { File dir = cluster.getJournalCluster().getCurrentDir(0, MiniQJMHACluster.NAMESERVICE); checkJNStorage(dir, 5, 7); } // restart NN0 again to make sure we can start using the new fsimage and // the corresponding md5 checksum dfsCluster.restartNameNode(0); // start the rolling upgrade again to make sure we do not load upgrade // status after the rollback dfsCluster.transitionToActive(0); dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #24
Source File: TestRollingUpgrade.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout = 300000) public void testCheckpoint() throws IOException, InterruptedException { final Configuration conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); queryForPreparation(dfs); dfs.mkdirs(foo); long txid = dfs.rollEdits(); Assert.assertTrue(txid > 0); int retries = 0; while (++retries < 5) { NNStorage storage = dfsCluster.getNamesystem(1).getFSImage() .getStorage(); if (storage.getFsImageName(txid - 1) != null) { return; } Thread.sleep(1000); } Assert.fail("new checkpoint does not exist"); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #25
Source File: TestDFSUpgradeWithHA.java From big-c with Apache License 2.0 | 4 votes |
/** * Make sure that even if the NN which initiated the upgrade is in the standby * state that we're allowed to finalize. */ @Test public void testFinalizeFromSecondNameNodeWithJournalNodes() throws IOException, URISyntaxException { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder() .numDataNodes(0); qjCluster = builder.build(); MiniDFSCluster cluster = qjCluster.getDfsCluster(); // No upgrade is in progress at the moment. checkJnPreviousDirExistence(qjCluster, false); checkClusterPreviousDirExistence(cluster, false); assertCTimesEqual(cluster); // Transition NN0 to active and do some FS ops. cluster.transitionToActive(0); fs = HATestUtil.configureFailoverFs(cluster, conf); assertTrue(fs.mkdirs(new Path("/foo1"))); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.shutdownNameNode(1); cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); cluster.restartNameNode(0, false); checkNnPreviousDirExistence(cluster, 0, true); checkNnPreviousDirExistence(cluster, 1, false); checkJnPreviousDirExistence(qjCluster, true); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.run( new String[]{"-force"}, cluster.getConfiguration(1)); assertEquals(0, rc); cluster.restartNameNode(1); // Make the second NN (not the one that initiated the upgrade) active when // the finalize command is run. cluster.transitionToStandby(0); cluster.transitionToActive(1); runFinalizeCommand(cluster); checkClusterPreviousDirExistence(cluster, false); checkJnPreviousDirExistence(qjCluster, false); assertCTimesEqual(cluster); } finally { if (fs != null) { fs.close(); } if (qjCluster != null) { qjCluster.shutdown(); } } }
Example #26
Source File: TestRollingUpgrade.java From big-c with Apache License 2.0 | 4 votes |
@Test (timeout = 300000) public void testFinalize() throws Exception { final Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); final Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); // let NN1 tail editlog every 1s dfsCluster.getConfiguration(1).setInt( DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); dfsCluster.restartNameNode(1); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); dfs.mkdirs(foo); FSImage fsimage = dfsCluster.getNamesystem(0).getFSImage(); // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); dfs.mkdirs(bar); queryForPreparation(dfs); // The NN should have a copy of the fsimage in case of rollbacks. Assert.assertTrue(fsimage.hasRollbackFSImage()); info = dfs.rollingUpgrade(RollingUpgradeAction.FINALIZE); Assert.assertTrue(info.isFinalized()); Assert.assertTrue(dfs.exists(foo)); // Once finalized, there should be no more fsimage for rollbacks. Assert.assertFalse(fsimage.hasRollbackFSImage()); // Should have no problem in restart and replaying edits that include // the FINALIZE op. dfsCluster.restartNameNode(0); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #27
Source File: TestDFSInotifyEventInputStream.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout = 120000) public void testTwoActiveNNs() throws IOException, MissingEventsException { Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build(); try { cluster.getDfsCluster().waitActive(); cluster.getDfsCluster().transitionToActive(0); DFSClient client0 = new DFSClient(cluster.getDfsCluster().getNameNode(0) .getNameNodeAddress(), conf); DFSClient client1 = new DFSClient(cluster.getDfsCluster().getNameNode(1) .getNameNodeAddress(), conf); DFSInotifyEventInputStream eis = client0.getInotifyEventStream(); for (int i = 0; i < 10; i++) { client0.mkdirs("/dir" + i, null, false); } cluster.getDfsCluster().transitionToActive(1); for (int i = 10; i < 20; i++) { client1.mkdirs("/dir" + i, null, false); } // make sure that the old active can't read any further than the edits // it logged itself (it has no idea whether the in-progress edits from // the other writer have actually been committed) EventBatch batch = null; for (int i = 0; i < 10; i++) { batch = waitForNextEvents(eis); Assert.assertEquals(1, batch.getEvents().length); Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" + i)); } Assert.assertTrue(eis.poll() == null); } finally { try { cluster.shutdown(); } catch (ExitUtil.ExitException e) { // expected because the old active will be unable to flush the // end-of-segment op since it is fenced } } }
Example #28
Source File: TestRollingUpgradeDowngrade.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout = 300000) public void testDowngrade() throws Exception { final Configuration conf = new HdfsConfiguration(); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); final Path bar = new Path("/bar"); try { cluster = new MiniQJMHACluster.Builder(conf).build(); MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); // let NN1 tail editlog every 1s dfsCluster.getConfiguration(1).setInt( DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); dfsCluster.restartNameNode(1); dfsCluster.transitionToActive(0); DistributedFileSystem dfs = dfsCluster.getFileSystem(0); dfs.mkdirs(foo); // start rolling upgrade RollingUpgradeInfo info = dfs .rollingUpgrade(RollingUpgradeAction.PREPARE); Assert.assertTrue(info.isStarted()); dfs.mkdirs(bar); TestRollingUpgrade.queryForPreparation(dfs); dfs.close(); dfsCluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade"); // Once downgraded, there should be no more fsimage for rollbacks. Assert.assertFalse(dfsCluster.getNamesystem(0).getFSImage() .hasRollbackFSImage()); // shutdown NN1 dfsCluster.shutdownNameNode(1); dfsCluster.transitionToActive(0); dfs = dfsCluster.getFileSystem(0); Assert.assertTrue(dfs.exists(foo)); Assert.assertTrue(dfs.exists(bar)); } finally { if (cluster != null) { cluster.shutdown(); } } }
Example #29
Source File: TestMiniClusterHadoopNNAWithStreamEngine.java From NNAnalytics with Apache License 2.0 | 4 votes |
@BeforeClass public static void beforeClass() throws Exception { RANDOM.nextBytes(TINY_FILE_BYTES); RANDOM.nextBytes(SMALL_FILE_BYTES); RANDOM.nextBytes(MEDIUM_FILE_BYTES); // Speed up editlog tailing. CONF.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); CONF.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1); CONF.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); CONF.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10); CONF.setBoolean("fs.hdfs.impl.disable.cache", true); CONF.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NAMESERVICE); MiniQJMHACluster.Builder qjmBuilder = new MiniQJMHACluster.Builder(CONF); qjmBuilder.getDfsBuilder().numDataNodes(NUMDATANODES); cluster = qjmBuilder.build(); cluster.getDfsCluster().waitActive(); cluster.getDfsCluster().transitionToActive(0); HATestUtil.setFailoverConfigurations(cluster.getDfsCluster(), CONF, NAMESERVICE, 0); CONF.set("dfs.nameservice.id", NAMESERVICE); nna = new HadoopWebServerMain(); nnaConf = new ApplicationConfiguration(); nnaConf.set("nna.support.bootstrap.overrides", "true"); nnaConf.set("ldap.enable", "false"); nnaConf.set("authorization.enable", "false"); nnaConf.set("nna.historical", "true"); nnaConf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory()); nnaConf.set("nna.web.base.dir", "src/main/resources/webapps/nna"); nnaConf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName()); nna.init(nnaConf, null, CONF); hostPort = new HttpHost("localhost", 4567); client = new DefaultHttpClient(); // Fetch NNA Namespace. HttpGet fetch = new HttpGet("http://localhost:4567/fetchNamespace"); HttpResponse fetchRes = client.execute(hostPort, fetch); assertThat(fetchRes.getStatusLine().getStatusCode(), is(200)); IOUtils.readLines(fetchRes.getEntity().getContent()); // Reload NNA Namespace. HttpGet reload = new HttpGet("http://localhost:4567/reloadNamespace"); HttpResponse reloadRes = client.execute(hostPort, reload); assertThat(reloadRes.getStatusLine().getStatusCode(), is(200)); IOUtils.readLines(reloadRes.getEntity().getContent()); }
Example #30
Source File: TestDFSUpgradeWithHA.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testRollbackWithJournalNodes() throws IOException, URISyntaxException { MiniQJMHACluster qjCluster = null; FileSystem fs = null; try { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder() .numDataNodes(0); qjCluster = builder.build(); MiniDFSCluster cluster = qjCluster.getDfsCluster(); // No upgrade is in progress at the moment. checkClusterPreviousDirExistence(cluster, false); assertCTimesEqual(cluster); checkJnPreviousDirExistence(qjCluster, false); // Transition NN0 to active and do some FS ops. cluster.transitionToActive(0); fs = HATestUtil.configureFailoverFs(cluster, conf); assertTrue(fs.mkdirs(new Path("/foo1"))); final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster); // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade // flag. cluster.shutdownNameNode(1); cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE); cluster.restartNameNode(0, false); checkNnPreviousDirExistence(cluster, 0, true); checkNnPreviousDirExistence(cluster, 1, false); checkJnPreviousDirExistence(qjCluster, true); // NN0 should come up in the active state when given the -upgrade option, // so no need to transition it to active. assertTrue(fs.mkdirs(new Path("/foo2"))); final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster); assertTrue(cidDuringUpgrade > cidBeforeUpgrade); // Now bootstrap the standby with the upgraded info. int rc = BootstrapStandby.run( new String[]{"-force"}, cluster.getConfiguration(1)); assertEquals(0, rc); cluster.restartNameNode(1); checkNnPreviousDirExistence(cluster, 0, true); checkNnPreviousDirExistence(cluster, 1, true); checkJnPreviousDirExistence(qjCluster, true); assertCTimesEqual(cluster); // Shut down the NNs, but deliberately leave the JNs up and running. Collection<URI> nn1NameDirs = cluster.getNameDirs(0); cluster.shutdown(); conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs)); NameNode.doRollback(conf, false); final long cidAfterRollback = getCommittedTxnIdValue(qjCluster); assertTrue(cidBeforeUpgrade < cidAfterRollback); // make sure the committedTxnId has been reset correctly after rollback assertTrue(cidDuringUpgrade > cidAfterRollback); // The rollback operation should have rolled back the first NN's local // dirs, and the shared dir, but not the other NN's dirs. Those have to be // done by bootstrapping the standby. checkNnPreviousDirExistence(cluster, 0, false); checkJnPreviousDirExistence(qjCluster, false); } finally { if (fs != null) { fs.close(); } if (qjCluster != null) { qjCluster.shutdown(); } } }