Java Code Examples for org.apache.hadoop.hdfs.MiniDFSCluster#getFileSystem()

/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-basedir",
                                       "/basedir",
                                       namenode+"/basedir/middle/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat/middle/srcdat", files));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/basedir");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
 

public TestFSRMStateStoreTester(MiniDFSCluster cluster, boolean adminCheckEnable) throws Exception {
  Path workingDirPath = new Path("/yarn/Test");
  this.adminCheckEnable = adminCheckEnable;
  this.cluster = cluster;
  FileSystem fs = cluster.getFileSystem();
  fs.mkdirs(workingDirPath);
  Path clusterURI = new Path(cluster.getURI());
  workingDirPathURI = new Path(clusterURI, workingDirPath);
  fs.close();
}
 

/**
 * This function ensures that writing causes TotalWritetime to increment
 * and reading causes totalReadTime to move.
 * @throws Exception
 */
@Test
public void testDataNodeTimeSpend() throws Exception {
  Configuration conf = new HdfsConfiguration();
  SimulatedFSDataset.setFactory(conf);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  try {
    FileSystem fs = cluster.getFileSystem();
    List<DataNode> datanodes = cluster.getDataNodes();
    assertEquals(datanodes.size(), 1);
    DataNode datanode = datanodes.get(0);
    MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
    final long LONG_FILE_LEN = 1024 * 1024 * 10;

    long startWriteValue = getLongCounter("TotalWriteTime", rb);
    long startReadValue = getLongCounter("TotalReadTime", rb);

    for (int x =0; x < 50; x++) {
      DFSTestUtil.createFile(fs, new Path("/time.txt."+ x),
              LONG_FILE_LEN, (short) 1, Time.monotonicNow());
    }

    for (int x =0; x < 50; x++) {
      String s = DFSTestUtil.readFile(fs, new Path("/time.txt." + x));
    }

    MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
    long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
    long endReadValue = getLongCounter("TotalReadTime", rbNew);

    assertTrue(endReadValue > startReadValue);
    assertTrue(endWriteValue > startWriteValue);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/** Test that syncBlock is correctly performed at replicas */
@Test
public void testHSyncWithReplication() throws Exception {
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  final FileSystem fs = cluster.getFileSystem();

  final Path p = new Path("/testHSyncWithReplication/foo");
  final int len = 1 << 16;
  FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
      4096, (short) 3, len, null);
  out.write(1);
  out.hflush();
  checkSyncMetric(cluster, 0, 0);
  checkSyncMetric(cluster, 1, 0);
  checkSyncMetric(cluster, 2, 0);
  out.hsync();
  checkSyncMetric(cluster, 0, 1);
  checkSyncMetric(cluster, 1, 1);
  checkSyncMetric(cluster, 2, 1);
  out.hsync();
  checkSyncMetric(cluster, 0, 2);
  checkSyncMetric(cluster, 1, 2);
  checkSyncMetric(cluster, 2, 2);
  cluster.shutdown();
}
 

@Override
protected void setUp() throws Exception {
  cluster = new MiniDFSCluster(CONF, DATANODE_COUNT, true, null);
  cluster.waitActive();
  namesystem = cluster.getNameNode().getNamesystem();
  fs = (DistributedFileSystem) cluster.getFileSystem();
  metrics = namesystem.getFSNamesystemMetrics();
}
 

private void setUpCluster(JobConf conf) throws IOException {
  miniDFSCluster = new MiniDFSCluster(conf, 1, true, null);
  FileSystem fileSys = miniDFSCluster.getFileSystem();
  String namenode = fileSys.getUri().toString();
  miniMRCluster = new MiniMRCluster(1, namenode, 3, 
                    null, null, conf);
}
 

/** copy empty directory on dfs file system */
public void testEmptyDir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      
      FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
      fs.mkdirs(new Path("/empty"));

      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/empty",
                                       namenode+"/dest"});
      fs = FileSystem.get(URI.create(namenode+"/destdat"), conf);
      assertTrue("Destination directory does not exist.",
                 fs.exists(new Path(namenode+"/dest")));
      deldir(hdfs, "/dest");
      deldir(hdfs, "/empty");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
 

/** copy files from local file system to dfs file system */
public void testCopyFromLocalToDfs() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = hdfs.getUri().toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       "file:///"+TEST_ROOT_DIR+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(cluster.getFileSystem(), "/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/logs");
      deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
 

/**
 * Test that block IDs are generated sequentially.
 *
 * @throws IOException
 */
@Test
public void testBlockIdGeneration() throws IOException {
  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
  MiniDFSCluster cluster =
      new MiniDFSCluster.Builder(conf).numDataNodes(1).build();

  try {
    cluster.waitActive();
    FileSystem fs = cluster.getFileSystem();

    // Create a file that is 10 blocks long.
    Path path = new Path("testBlockIdGeneration.dat");
    DFSTestUtil.createFile(
        fs, path, IO_SIZE, BLOCK_SIZE * 10, BLOCK_SIZE, REPLICATION, SEED);
    List<LocatedBlock> blocks = DFSTestUtil.getAllBlocks(fs, path);
    LOG.info("Block0 id is " + blocks.get(0).getBlock().getBlockId());
    long nextBlockExpectedId = blocks.get(0).getBlock().getBlockId() + 1;

    // Ensure that the block IDs are sequentially increasing.
    for (int i = 1; i < blocks.size(); ++i) {
      long nextBlockId = blocks.get(i).getBlock().getBlockId();
      LOG.info("Block" + i + " id is " + nextBlockId);
      assertThat(nextBlockId, is(nextBlockExpectedId));
      ++nextBlockExpectedId;
    }
  } finally {
    cluster.shutdown();
  }
}
 

/** This creates a slow writer and check to see
  * if pipeline heartbeats work fine
  */
 public void testStuckDataNode() throws Exception {
   final int DATANODE_NUM = 3;
   Configuration conf = new Configuration();
   final int timeout = 8000;
   conf.setInt("dfs.socket.timeout",timeout);

   final Path p = new Path("/pipelineHeartbeat/foo");
   System.out.println("p=" + p);

   MiniDFSCluster cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null);
   DistributedFileSystem fs = (DistributedFileSystem)cluster.getFileSystem();

DataNodeMetrics metrics = cluster.getDataNodes().get(0).myMetrics;
MetricsTimeVaryingLong spyBytesWritten = spy(metrics.bytesWritten);
DelayAnswer delayAnswer = new DelayAnswer(); 
doAnswer(delayAnswer).when(spyBytesWritten).inc(anyInt());
metrics.bytesWritten = spyBytesWritten;

try {
   	// create a new file.
   	FSDataOutputStream stm = fs.create(p);
   	stm.write(1);
   	stm.sync();
   	stm.write(2);
   	stm.close();

   	// verify that entire file is good
   	FSDataInputStream in = fs.open(p);
   	assertEquals(1, in.read());
   	assertEquals(2, in.read());
   	in.close();
   } finally {
     fs.close();
     cluster.shutdown();
   }
 }
 

protected void setUp() throws Exception {
  super.setUp();
  dfscluster = new MiniDFSCluster(new JobConf(), 2, true, null);
  fs = dfscluster.getFileSystem();
  mapred = new MiniMRCluster(2, fs.getUri().toString(), 1);
  inputPath = new Path(fs.getHomeDirectory(), "test"); 
  filea = new Path(inputPath,"a");
  fileb = new Path(inputPath,"b");
  filec = new Path(inputPath,"c");
  archivePath = new Path(fs.getHomeDirectory(), "tmp");
}
 

@Test
public void testSafemodeBringsDownMaster() throws Exception {
  final TableName tableName = TableName.valueOf("testSafemodeBringsDownMaster");
  final byte[][] splitKeys = new byte[][] {
    Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
  MiniDFSCluster dfsCluster = UTIL.getDFSCluster();
  DistributedFileSystem dfs = (DistributedFileSystem) dfsCluster.getFileSystem();
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
  final long timeOut = 180000;
  long startTime = System.currentTimeMillis();
  int index = -1;
  do {
    index = UTIL.getMiniHBaseCluster().getServerWithMeta();
  } while (index == -1 &&
    startTime + timeOut < System.currentTimeMillis());

  if (index != -1){
    UTIL.getMiniHBaseCluster().abortRegionServer(index);
    UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
  }
  UTIL.waitFor(timeOut, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      List<JVMClusterUtil.MasterThread> threads = UTIL.getMiniHBaseCluster().getLiveMasterThreads();
      return threads == null || threads.isEmpty();
    }
  });
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
}
 

/**
 * Test that two namenodes can't continue as primary
 */
@Test
public void testMultiplePrimariesStarted() throws Exception {
  Path p1 = new Path("/testBKJMMultiplePrimary");

  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
             BKJMUtil.createJournalURI("/hotfailoverMultiple").toString());
    BKJMUtil.addJournalManagerDefinition(conf);

    cluster = new MiniDFSCluster.Builder(conf)
      .nnTopology(MiniDFSNNTopology.simpleHATopology())
      .numDataNodes(0)
      .manageNameDfsSharedDirs(false)
      .checkExitOnShutdown(false)
      .build();
    NameNode nn1 = cluster.getNameNode(0);
    NameNode nn2 = cluster.getNameNode(1);
    cluster.waitActive();
    cluster.transitionToActive(0);

    FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
    fs.mkdirs(p1);
    nn1.getRpcServer().rollEditLog();
    cluster.transitionToActive(1);
    fs = cluster.getFileSystem(0); // get the older active server.

    try {
      fs.delete(p1, true);
      fail("Log update on older active should cause it to exit");
    } catch (RemoteException re) {
      assertTrue(re.getClassName().contains("ExitException"));
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/**
 * testing that WRITE operation can handle token expiration when
 * re-establishing pipeline is needed
 */
@Test
public void testWrite() throws Exception {
  MiniDFSCluster cluster = null;
  int numDataNodes = 2;
  Configuration conf = getConf(numDataNodes);

  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    assertEquals(numDataNodes, cluster.getDataNodes().size());

    final NameNode nn = cluster.getNameNode();
    final BlockManager bm = nn.getNamesystem().getBlockManager();
    final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();

    // set a short token lifetime (1 second)
    SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
    Path fileToWrite = new Path(FILE_TO_WRITE);
    FileSystem fs = cluster.getFileSystem();

    FSDataOutputStream stm = writeFile(fs, fileToWrite, (short) numDataNodes,
        BLOCK_SIZE);
    // write a partial block
    int mid = rawData.length - 1;
    stm.write(rawData, 0, mid);
    stm.hflush();

    /*
     * wait till token used in stm expires
     */
    Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(stm);
    while (!SecurityTestUtil.isBlockTokenExpired(token)) {
      try {
        Thread.sleep(10);
      } catch (InterruptedException ignored) {
      }
    }

    // remove a datanode to force re-establishing pipeline
    cluster.stopDataNode(0);
    // write the rest of the file
    stm.write(rawData, mid, rawData.length - mid);
    stm.close();
    // check if write is successful
    FSDataInputStream in4 = fs.open(fileToWrite);
    assertTrue(checkFile1(in4));
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

/** Test processOverReplicatedBlock can handle corrupt replicas fine.
 * It make sure that it won't treat corrupt replicas as valid ones 
 * thus prevents NN deleting valid replicas but keeping
 * corrupt ones.
 */
public void testProcesOverReplicateBlock() throws IOException {
  Configuration conf = new Configuration();
  conf.setLong("dfs.blockreport.intervalMsec", 1000L);
  conf.set("dfs.replication.pending.timeout.sec", Integer.toString(2));
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 3, true, null);
  FileSystem fs = cluster.getFileSystem();

  try {
    int namespaceId = cluster.getNameNode().getNamespaceID();
    final Path fileName = new Path("/foo1");
    DFSTestUtil.createFile(fs, fileName, 2, (short)3, 0L);
    DFSTestUtil.waitReplication(fs, fileName, (short)3);
    
    // corrupt the block on datanode 0
    Block block = DFSTestUtil.getFirstBlock(fs, fileName);
    TestDatanodeBlockScanner.corruptReplica(block.getBlockName(), 0, cluster);
    DataNodeProperties dnProps = cluster.stopDataNode(0);
    // remove block scanner log to trigger block scanning
    File scanLog = new File(cluster.getBlockDirectory("data1").getParent(), "dncp_block_verification.log.curr");
    //wait for one minute for deletion to succeed;
    scanLog.delete();
    
    // restart the datanode so the corrupt replica will be detected
    cluster.restartDataNode(dnProps);
    DFSTestUtil.waitReplication(fs, fileName, (short)2);
    
    final DatanodeID corruptDataNode = 
      cluster.getDataNodes().get(2).getDNRegistrationForNS(namespaceId);
    final FSNamesystem namesystem = cluster.getNameNode().getNamesystem();
    synchronized (namesystem.heartbeats) {
      // set live datanode's remaining space to be 0 
      // so they will be chosen to be deleted when over-replication occurs
      for (DatanodeDescriptor datanode : namesystem.heartbeats) {
        if (!corruptDataNode.equals(datanode)) {
          datanode.updateHeartbeat(100L, 100L, 0L, 100L, 0);
        }
      }
    }
      
    // decrease the replication factor to 1; 
    namesystem.setReplication(fileName.toString(), (short)1);
    waitReplication(namesystem, block, (short)1);
    
    // corrupt one won't be chosen to be excess one
    // without 4910 the number of live replicas would be 0: block gets lost
    assertEquals(1, namesystem.countNodes(block).liveReplicas());

    // Test the case when multiple calls to setReplication still succeeds.
    System.out.println("Starting next test with file foo2.");
    final Path fileName2 = new Path("/foo1");
    DFSTestUtil.createFile(fs, fileName2, 2, (short)3, 0L);
    DFSTestUtil.waitReplication(fs, fileName2, (short)3);
    LocatedBlocks lbs = namesystem.getBlockLocations(
               fileName2.toString(), 0, 10);
    Block firstBlock = lbs.get(0).getBlock();
    namesystem.setReplication(fileName2.toString(), (short)2);
    namesystem.setReplication(fileName2.toString(), (short)1);
    
    // wait upto one minute for excess replicas to get deleted. It is not
    // immediate because excess replicas are being handled asyncronously.
    waitReplication(namesystem, firstBlock, (short)1);
    assertEquals(1, namesystem.countNodes(firstBlock).liveReplicas());
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * Test that CFIF can handle missing blocks.
 */
@Test
public void testMissingBlocks() throws Exception {
  String namenode = null;
  MiniDFSCluster dfs = null;
  FileSystem fileSys = null;
  String testName = "testMissingBlocks";
  try {
    Configuration conf = new Configuration();
    conf.set("fs.hdfs.impl", MissingBlockFileSystem.class.getName());
    conf.setBoolean("dfs.replication.considerLoad", false);
    dfs = new MiniDFSCluster.Builder(conf).racks(rack1).hosts(hosts1)
        .build();
    dfs.waitActive();

    namenode = (dfs.getFileSystem()).getUri().getHost() + ":" +
               (dfs.getFileSystem()).getUri().getPort();

    fileSys = dfs.getFileSystem();
    if (!fileSys.mkdirs(inDir)) {
      throw new IOException("Mkdirs failed to create " + inDir.toString());
    }

    Path file1 = new Path(dir1 + "/file1");
    writeFile(conf, file1, (short)1, 1);
    // create another file on the same datanode
    Path file5 = new Path(dir5 + "/file5");
    writeFile(conf, file5, (short)1, 1);

    ((MissingBlockFileSystem)fileSys).setFileWithMissingBlocks(file1.toUri().getPath());
    // split it using a CombinedFile input format
    DummyInputFormat inFormat = new DummyInputFormat();
    Job job = Job.getInstance(conf);
    FileInputFormat.setInputPaths(job, dir1 + "," + dir5);
    List<InputSplit> splits = inFormat.getSplits(job);
    System.out.println("Made splits(Test0): " + splits.size());
    for (InputSplit split : splits) {
      System.out.println("File split(Test0): " + split);
    }
    assertEquals(splits.size(), 1);
    CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0);
    assertEquals(2, fileSplit.getNumPaths());
    assertEquals(1, fileSplit.getLocations().length);
    assertEquals(file1.getName(), fileSplit.getPath(0).getName());
    assertEquals(0, fileSplit.getOffset(0));
    assertEquals(BLOCKSIZE, fileSplit.getLength(0));
    assertEquals(file5.getName(), fileSplit.getPath(1).getName());
    assertEquals(0, fileSplit.getOffset(1));
    assertEquals(BLOCKSIZE, fileSplit.getLength(1));
    assertEquals(hosts1[0], fileSplit.getLocations()[0]);

  } finally {
    if (dfs != null) {
      dfs.shutdown();
    }
  }
}
 

/**
 * Tests that the # of misreplaced replicas is correct
 * @throws IOException
 */
@Test
public void testFsckMisPlacedReplicas() throws IOException {
  // Desired replication factor
  final short REPL_FACTOR = 2;
  // Number of replicas to actually start
  short NUM_DN = 2;
  // Number of blocks to write
  final short NUM_BLOCKS = 3;
  // Set a small-ish blocksize
  final long blockSize = 512;
  
  String [] racks = {"/rack1", "/rack1"};
  String [] hosts = {"host1", "host2"};
  
  Configuration conf = new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
  
  MiniDFSCluster cluster = null;
  DistributedFileSystem dfs = null;
  
  try {
    // Startup a minicluster
    cluster = 
        new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts)
        .racks(racks).build();
    assertNotNull("Failed Cluster Creation", cluster);
    cluster.waitClusterUp();
    dfs = cluster.getFileSystem();
    assertNotNull("Failed to get FileSystem", dfs);
    
    // Create a file that will be intentionally under-replicated
    final String pathString = new String("/testfile");
    final Path path = new Path(pathString);
    long fileLen = blockSize * NUM_BLOCKS;
    DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1);
    
    // Create an under-replicated file
    NameNode namenode = cluster.getNameNode();
    NetworkTopology nettop = cluster.getNamesystem().getBlockManager()
        .getDatanodeManager().getNetworkTopology();
    // Add a new node on different rack, so previous blocks' replicas 
    // are considered to be misplaced
    nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3"));
    NUM_DN++;
    
    Map<String,String[]> pmap = new HashMap<String, String[]>();
    Writer result = new StringWriter();
    PrintWriter out = new PrintWriter(result, true);
    InetAddress remoteAddress = InetAddress.getLocalHost();
    NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, 
        NUM_DN, remoteAddress);
    
    // Run the fsck and check the Result
    final HdfsFileStatus file = 
        namenode.getRpcServer().getFileInfo(pathString);
    assertNotNull(file);
    Result res = new Result(conf);
    fsck.check(pathString, file, res);
    // check misReplicatedBlock number.
    assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS);
  } finally {
    if(dfs != null) {
      dfs.close();
    }
    if(cluster != null) {
      cluster.shutdown();
    }
  }
}
 

public void testRecoveryWithLostTracker(MiniDFSCluster dfs,
                                        MiniMRCluster mr) 
throws IOException {
  FileSystem fileSys = dfs.getFileSystem();
  JobConf jobConf = mr.createJobConf();
  int numMaps = 50;
  int numReds = 1;
  String mapSignalFile = UtilsForTests.getMapSignalFile(shareDir);
  String redSignalFile = UtilsForTests.getReduceSignalFile(shareDir);
  
  // Configure the jobs
  JobConf job = configureJob(jobConf, numMaps, numReds, 
                             mapSignalFile, redSignalFile);
    
  fileSys.delete(shareDir, true);
  
  // Submit a master job   
  JobClient jobClient = new JobClient(job);
  RunningJob rJob = jobClient.submitJob(job);
  JobID id = rJob.getID();
  
  // wait for the job to be inited
  mr.initializeJob(id);
  
  // Make sure that the master job is 50% completed
  while (UtilsForTests.getJobStatus(jobClient, id).mapProgress() 
         < 0.5f) {
    UtilsForTests.waitFor(100);
  }

  // Kill the jobtracker
  mr.stopJobTracker();

  // Signal the maps to complete
  UtilsForTests.signalTasks(dfs, fileSys, true, mapSignalFile, redSignalFile);
  
  // Enable recovery on restart
  mr.getJobTrackerConf().setBoolean("mapred.jobtracker.restart.recover", 
                                    true);
  
  // Kill the 2nd tasktracker
  mr.stopTaskTracker(1);
  
  // Wait for a minute before submitting a job
  UtilsForTests.waitFor(60 * 1000);
  
  // Restart the jobtracker
  mr.startJobTracker();

  // Check if the jobs are still running
  
  // Wait for the JT to be ready
  UtilsForTests.waitForJobTracker(jobClient);

  // Signal the reducers to complete
  UtilsForTests.signalTasks(dfs, fileSys, false, mapSignalFile, 
                            redSignalFile);
  
  UtilsForTests.waitTillDone(jobClient);

  // Check if the tasks on the lost tracker got re-executed
  assertEquals("Tracker killed while the jobtracker was down did not get lost "
               + "upon restart", 
               jobClient.getClusterStatus().getTaskTrackers(), 1);

  // validate the history file
  TestJobHistory.validateJobHistoryFileFormat(id, job, "SUCCESS", true);
  TestJobHistory.validateJobHistoryFileContent(mr, rJob, job);
}
 

/**
 * This is a very basic test that writes one event to HDFS and reads it back.
 */
@Test
public void simpleHDFSTest() throws EventDeliveryException, IOException {
  cluster = new MiniDFSCluster(new Configuration(), 1, true, null);
  cluster.waitActive();

  String outputDir = "/flume/simpleHDFSTest";
  Path outputDirPath = new Path(outputDir);

  logger.info("Running test with output dir: {}", outputDir);

  FileSystem fs = cluster.getFileSystem();
  // ensure output directory is empty
  if (fs.exists(outputDirPath)) {
    fs.delete(outputDirPath, true);
  }

  String nnURL = getNameNodeURL(cluster);
  logger.info("Namenode address: {}", nnURL);

  Context chanCtx = new Context();
  MemoryChannel channel = new MemoryChannel();
  channel.setName("simpleHDFSTest-mem-chan");
  channel.configure(chanCtx);
  channel.start();

  Context sinkCtx = new Context();
  sinkCtx.put("hdfs.path", nnURL + outputDir);
  sinkCtx.put("hdfs.fileType", HDFSWriterFactory.DataStreamType);
  sinkCtx.put("hdfs.batchSize", Integer.toString(1));

  HDFSEventSink sink = new HDFSEventSink();
  sink.setName("simpleHDFSTest-hdfs-sink");
  sink.configure(sinkCtx);
  sink.setChannel(channel);
  sink.start();

  // create an event
  String EVENT_BODY = "yarg!";
  channel.getTransaction().begin();
  try {
    channel.put(EventBuilder.withBody(EVENT_BODY, Charsets.UTF_8));
    channel.getTransaction().commit();
  } finally {
    channel.getTransaction().close();
  }

  // store event to HDFS
  sink.process();

  // shut down flume
  sink.stop();
  channel.stop();

  // verify that it's in HDFS and that its content is what we say it should be
  FileStatus[] statuses = fs.listStatus(outputDirPath);
  Assert.assertNotNull("No files found written to HDFS", statuses);
  Assert.assertEquals("Only one file expected", 1, statuses.length);

  for (FileStatus status : statuses) {
    Path filePath = status.getPath();
    logger.info("Found file on DFS: {}", filePath);
    FSDataInputStream stream = fs.open(filePath);
    BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
    String line = reader.readLine();
    logger.info("First line in file {}: {}", filePath, line);
    Assert.assertEquals(EVENT_BODY, line);
  }

  if (!KEEP_DATA) {
    fs.delete(outputDirPath, true);
  }

  cluster.shutdown();
  cluster = null;
}
 

/** Test fsck with permission set on inodes */
@Test
public void testFsckPermission() throws Exception {
  final DFSTestUtil util = new DFSTestUtil.Builder().
      setName(getClass().getSimpleName()).setNumFiles(20).build();
  final Configuration conf = new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);

  MiniDFSCluster cluster = null;
  try {
    // Create a cluster with the current user, write some files
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    final MiniDFSCluster c2 = cluster;
    final String dir = "/dfsck";
    final Path dirpath = new Path(dir);
    final FileSystem fs = c2.getFileSystem();

    util.createFiles(fs, dir);
    util.waitReplication(fs, dir, (short) 3);
    fs.setPermission(dirpath, new FsPermission((short) 0700));

    // run DFSck as another user, should fail with permission issue
    UserGroupInformation fakeUGI = UserGroupInformation.createUserForTesting(
        "ProbablyNotARealUserName", new String[] { "ShangriLa" });
    fakeUGI.doAs(new PrivilegedExceptionAction<Object>() {
      @Override
      public Object run() throws Exception {
        System.out.println(runFsck(conf, -1, true, dir));
        return null;
      }
    });
    
    // set permission and try DFSck again as the fake user, should succeed
    fs.setPermission(dirpath, new FsPermission((short) 0777));
    fakeUGI.doAs(new PrivilegedExceptionAction<Object>() {
      @Override
      public Object run() throws Exception {
        final String outStr = runFsck(conf, 0, true, dir);
        System.out.println(outStr);
        assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
        return null;
      }
    });

    util.cleanup(fs, dir);
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}