Java Code Examples for org.apache.hadoop.fs.FSDataOutputStream#sync()

/**
 * Scenario: Read an under construction file using hftp.
 * 
 * Expected: Hftp should be able to read the latest byte after the file
 * has been hdfsSynced (but not yet closed).
 * 
 * @throws IOException
 */
public void testConcurrentRead() throws IOException {
  // Write a test file.
  FSDataOutputStream out = hdfs.create(TEST_FILE, true);
  out.writeBytes("123");
  out.sync();  // sync but not close
  
  // Try read using hftp.
  FSDataInputStream in = hftpFs.open(TEST_FILE);
  assertEquals('1', in.read());
  assertEquals('2', in.read());
  assertEquals('3', in.read());
  in.close();
  
  // Try seek and read.
  in = hftpFs.open(TEST_FILE);
  in.seek(2);
  assertEquals('3', in.read());
  in.close();
  
  out.close();
}
 

@Test
public void testAvatarSetQuota() throws Exception {
  String test = "/testAvatarSetQuota";
  DFSTestUtil util = new DFSTestUtil(test, 10, 10, 1024);
  util.createFiles(fs, test);

  FSDataOutputStream out = fs.create(new Path(test + "/abc"));
  byte[] buffer = new byte[10 * 1024];
  Random r = new Random();
  r.nextBytes(buffer);
  out.write(buffer);
  out.sync();
  ((DistributedFileSystem) fs).setQuota(new Path(test), 5, -1);
  out.close();

  cluster.getStandbyAvatar(0).avatar.quiesceStandby(-1);
}
 

private Path createFile(DistributedFileSystem dfs, int size,
    boolean triggerSoftLease) throws IOException, InterruptedException {
  // create a random file name
  String filestr = "/foo" + AppendTestUtil.nextInt();
  System.out.println("filestr=" + filestr);
  Path filepath = new Path(filestr);
  FSDataOutputStream stm = dfs.create(filepath, true,
      bufferSize, REPLICATION_NUM, BLOCK_SIZE);
  assertTrue(dfs.dfs.exists(filestr));

  // write random number of bytes into it.
  System.out.println("size=" + size);
  stm.write(buffer, 0, size);

  // sync file
  AppendTestUtil.LOG.info("sync");
  stm.sync();
  if (triggerSoftLease) {
    AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
    dfs.dfs.leasechecker.interruptAndJoin();
  }
  return filepath;
}
 

private Path createFile(DistributedFileSystem dfs, int size
    ) throws IOException, InterruptedException {
  // create a random file name
  String filestr = "/foo" + AppendTestUtil.nextInt();
  System.out.println("filestr=" + filestr);
  Path filepath = new Path(filestr);
  FSDataOutputStream stm = dfs.create(filepath, true,
      bufferSize, REPLICATION_NUM, BLOCK_SIZE);
  assertTrue(dfs.dfs.exists(filestr));

  // write random number of bytes into it.
  System.out.println("size=" + size);
  stm.write(buffer, 0, size);

  // sync file
  AppendTestUtil.LOG.info("sync");
  stm.sync();

  // write another piece of data to file. This piece of data
  // is not yet synced
  stm.write(buffer, 0, size);
  return filepath;
}
 

public void run() {
  FSDataOutputStream out = null;
  int i = 0;
  try {
    out = fs.create(filepath);
    for(; running; i++) {
      System.out.println(getName() + " writes " + i);
      out.write(i);
      out.sync();
      sleep(100);
    }
  }
  catch(Exception e) {
    System.out.println(getName() + " dies: e=" + e);
  }
  finally {
    System.out.println(getName() + ": i=" + i);
    IOUtils.closeStream(out);
  }
}
 

/** This creates a slow writer and check to see
  * if pipeline heartbeats work fine
  */
 public void testStuckDataNode() throws Exception {
   final int DATANODE_NUM = 3;
   Configuration conf = new Configuration();
   final int timeout = 8000;
   conf.setInt("dfs.socket.timeout",timeout);

   final Path p = new Path("/pipelineHeartbeat/foo");
   System.out.println("p=" + p);

   MiniDFSCluster cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null);
   DistributedFileSystem fs = (DistributedFileSystem)cluster.getFileSystem();

DataNodeMetrics metrics = cluster.getDataNodes().get(0).myMetrics;
MetricsTimeVaryingLong spyBytesWritten = spy(metrics.bytesWritten);
DelayAnswer delayAnswer = new DelayAnswer(); 
doAnswer(delayAnswer).when(spyBytesWritten).inc(anyInt());
metrics.bytesWritten = spyBytesWritten;

try {
   	// create a new file.
   	FSDataOutputStream stm = fs.create(p);
   	stm.write(1);
   	stm.sync();
   	stm.write(2);
   	stm.close();

   	// verify that entire file is good
   	FSDataInputStream in = fs.open(p);
   	assertEquals(1, in.read());
   	assertEquals(2, in.read());
   	in.close();
   } finally {
     fs.close();
     cluster.shutdown();
   }
 }
 

@Test
public void testLeaseAfterFailover() throws Exception {
  String fileName = "/testLeaseAfterFailover";
  FSDataOutputStream out = fs.create(new Path(fileName));
  byte[] buffer = new byte[1024];
  random.nextBytes(buffer);
  out.write(buffer);
  out.sync();

  FSNamesystem primary = cluster.getPrimaryAvatar(0).avatar.namesystem;

  // Prevents lease recovery to work.
  cluster.shutDownDataNodes();

  // Expire the lease.
  primary.leaseManager.setLeasePeriod(0, 0);
  primary.leaseManager.checkLeases();
  cluster.killPrimary();
  cluster.restartDataNodes(false);

  AvatarNode standbyAvatar = cluster.getStandbyAvatar(0).avatar;
  standbyAvatar.setAvatar(Avatar.ACTIVE);

  String lease = standbyAvatar.namesystem.leaseManager
      .getLeaseByPath(fileName).getHolder();
  assertEquals(HdfsConstants.NN_RECOVERY_LEASEHOLDER, lease);
}
 

@Test
public void testLeaseExpiry() throws Exception {
  setUp(true);
  h.setIgnoreDatanodes(false);
  LeaseManager leaseManager = cluster.getStandbyAvatar(0).avatar.namesystem.leaseManager;
  // Set low lease periods.
  leaseManager.setLeasePeriod(LEASE_PERIOD, LEASE_PERIOD);
  String src = "/testLeaseExpiry";

  // Create some data.
  FSDataOutputStream out = fs.create(new Path(src));
  byte[] buffer = new byte[BLOCK_SIZE * 2];
  random.nextBytes(buffer);
  out.write(buffer);
  out.sync();

  // Wait for the hard lease time to expire.
  Thread.sleep(LEASE_PERIOD * 2);

  cluster.failOver();
  LOG.info("Failover done");

  // Renew lease.
  ((DistributedFileSystem)fs).getClient().leasechecker.renew();
  LOG.info("Lease renewal done");

  // Wait to see whether lease expires.
  long start = System.currentTimeMillis();
  while (System.currentTimeMillis() - start < MAX_WAIT_TIME
      && leaseManager.getLeaseByPath(src) != null) {
    Thread.sleep(1000);
  }
  LOG.info("Wait for lease done");

  // Now try to write to the file.
  out.write(buffer);
  out.sync();
}
 

private static void createOpenFiles(int nFiles, String prefix) throws Exception {
  for (int i = 0; i < nFiles; i++) {
    FSDataOutputStream out = fs.create(new Path("/" + prefix + random.nextInt()));
    byte[] buffer = new byte[random.nextInt(MAX_FILE_SIZE)];
    random.nextBytes(buffer);
    out.write(buffer);
    out.sync();
  }
}
 

@Test
public void testFileUnderConstruction() throws Exception {
  String fileName = "/testFileUnderConstruction";
  FSDataOutputStream out = fs.create(new Path(fileName));
  byte [] buffer = new byte[BLOCK_SIZE * 5 + 256];
  random.nextBytes(buffer);
  out.write(buffer);
  out.sync();

  NameNode nn = cluster.getNameNode();
  List<FileStatusExtended> stats = nn.getRandomFiles(1);
  assertEquals(1, stats.size());
  assertEquals(((DistributedFileSystem) fs).getClient().clientName, stats
      .get(0).getHolder());
}
 

public void testLeaseRecoveryOnTrashedFile() throws Exception {
  Configuration conf = new Configuration();
  
  conf.setLong("dfs.block.size", blockSize);
  
  init(conf);
  
  String src = "/file-1";
  String dst = "/file-2";
  Path srcPath = new Path(src);
  Path dstPath = new Path(dst);
  FSDataOutputStream fos = fs.create(srcPath);

  AppendTestUtil.write(fos, 0, writeSize);
  fos.sync();
  
  // renaming a file out from under a client will cause close to fail
  // and result in the lease remaining while the blocks are finalized on
  // the DNs
  fs.rename(srcPath, dstPath);

  try {
    fos.close();
    fail("expected IOException");
  } catch (IOException e) {
    //expected
  }

  FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf);
  AppendTestUtil.recoverFile(cluster, fs2, dstPath);
  AppendTestUtil.check(fs2, dstPath, writeSize);
}
 

private void createFile(FileSystem fs, FSDataOutputStream out,
    String fileName, int fileLen) throws IOException {
  Random random = new Random(fileName.hashCode());
  byte buffer[] = new byte[fileLen];
  random.nextBytes(buffer);
  out.write(buffer);
  out.sync();
  ((DFSOutputStream) out.getWrappedStream()).abortForTests();
}
 

private void createFile(FileSystem fs, FSDataOutputStream out,
    String fileName, int fileLen) throws IOException {
  Random random = new Random(fileName.hashCode());
  byte buffer[] = new byte[fileLen];
  random.nextBytes(buffer);
  out.write(buffer);
  out.sync();
  ((DFSOutputStream) out.getWrappedStream()).abortForTests();
}
 

/** This creates a slow writer and check to see
 * if pipeline heartbeats work fine
 */
public void testPipelineHeartbeat() throws Exception {
  final int DATANODE_NUM = 2;
  final int fileLen = 6;
  Configuration conf = new Configuration();
  final int timeout = 2000;
  conf.setInt("dfs.socket.timeout",timeout);

  final Path p = new Path("/pipelineHeartbeat/foo");
  System.out.println("p=" + p);

  MiniDFSCluster cluster = new MiniDFSCluster(conf, DATANODE_NUM, true, null);
  DistributedFileSystem fs = (DistributedFileSystem)cluster.getFileSystem();

  initBuffer(fileLen);

  try {
    // create a new file.
    FSDataOutputStream stm = createFile(fs, p, DATANODE_NUM);

    stm.write(fileContents, 0, 1);
    Thread.sleep(timeout);
    stm.sync();
    System.out.println("Wrote 1 byte and hflush " + p);

    // write another byte
    Thread.sleep(timeout);
    stm.write(fileContents, 1, 1);
    stm.sync();

    stm.write(fileContents, 2, 1);
    Thread.sleep(timeout);
    stm.sync();

    stm.write(fileContents, 3, 1);
    Thread.sleep(timeout);
    stm.write(fileContents, 4, 1);
    stm.sync();

    stm.write(fileContents, 5, 1);
    Thread.sleep(timeout);
    stm.close();

    // verify that entire file is good
    checkFullFile(fs, p);
  } finally {
    fs.close();
    cluster.shutdown();
  }
}
 

public void testSecondLastBlockNotReceived() throws Exception {
  String fileName = "/testSecondLastBlockNotReceived";
  Path growingFile = new Path(fileName); 
  FSDataInputStream fis = null;
  FSDataOutputStream fos = fs.create(growingFile, false, 1024, (short)1, 1024);
  try {
    int fileLength = 2096;
    AppendTestUtil.write(fos, 0, fileLength);
    fos.sync();

    fis = fs.open(growingFile);
    for (int i = 0; i < fileLength; i++) {
      fis.read();
    }
    fis.close();

    FSNamesystem fsns = cluster.getNameNode().namesystem;
    INode[] inodes = fsns.dir.getExistingPathINodes(fileName);
    BlockInfo[] bis = ((INodeFile) (inodes[inodes.length - 1])).getBlocks();
    bis[bis.length - 2].setNumBytes(1);

    try {
      fis = fs.open(growingFile);
      TestCase.fail();
    } catch (IOException e) {
    }
    bis[bis.length - 2].setNumBytes(1024);

    bis[bis.length - 1].setNumBytes(1);
    fis = fs.open(growingFile);
    for (int i = 0; i < fileLength; i++) {
      fis.read();
    }
  } finally {
    if (fos != null) {
      fos.close();
    }
    if (fis != null) {
      fis.close();
    }
  }
}
 

public void testBlockSynchronization() throws Exception {
  final long softLease = 1000;
  final long hardLease = 60 * 60 *1000;
  final long packetSize =  BLOCK_SIZE/2;
  conf.setLong("dfs.block.size", BLOCK_SIZE);
  conf.setInt("dfs.heartbeat.interval", 1);
  conf.setInt("dfs.write.packet.size", (int) packetSize);

  MiniDFSCluster cluster = null;
  byte[] actual = new byte[FILE_SIZE];

  FSDataOutputStream stm =null;
  try {
    cluster = new MiniDFSCluster(conf, 5, true, null);
    cluster.waitActive();

    //create a file, write 2 blocks, then do a sync and then write
    //another block. The last block is not synced.
    DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
    int fileSize = 2 * (int)BLOCK_SIZE;
    String filestr = "/foo.dhruba";

    System.out.println("Creating file " + filestr);
    Path filepath = new Path(filestr);
    stm = dfs.create(filepath, true,
      bufferSize, REPLICATION_NUM, BLOCK_SIZE);
    assertTrue(dfs.dfs.exists(filestr));

    // write 2 blocks worth of data
    System.out.println("Writing data " + fileSize);
    stm.write(buffer, 0, fileSize);

    stm.sync();
    AppendTestUtil.LOG.info("sync done");

    // write another piece of data to file. This piece of data
    // is not yet synced
    System.out.println("Writing data again " + (packetSize + 1));
    stm.write(buffer, 0, (int)(packetSize + 1));

    // invoke recoverLease from a different client. This should truncate the
    // file to 2 blocks.
    recoverLease(filepath, null);

    // verify that file is 2 blocks long.
    verifyFile(null, filepath, actual, fileSize);
  }
  finally {
    try {
      if (stm != null) {
        stm.close();
      }
      if (cluster != null) {cluster.getFileSystem().close(); cluster.shutdown();}
    } catch (Exception e) {
      // ignore
    }
  }
}
 

@Test
public void testRBW() throws Exception {
  String fileName = "/testRBW";
  FSDataOutputStream out = fs.create(new Path(fileName));
  // Create RBW.
  byte[] buffer = new byte[1024 * 10 + 100];
  Random r = new Random();
  r.nextBytes(buffer);
  out.write(buffer);
  out.sync();

  cluster.restartNameNode(0, new String[] {}, false);
  ((DFSOutputStream) out.getWrappedStream()).abortForTests();

  // Send multiple RBW reports.
  waitForBlocks();
  cluster.restartDataNodes();
  Thread.sleep(10000);

  System.out.println("Restarts done");
  FSNamesystem namesystem = cluster.getNameNode().namesystem;

  long totalBlocks = namesystem.getBlocksTotal();
  long safeBlocks = namesystem.getSafeBlocks();
  long startTime = System.currentTimeMillis();
  while (totalBlocks != safeBlocks
      && (System.currentTimeMillis() - startTime < 15000)) {
    Thread.sleep(1000);
    System.out.println("Waiting for blocks, Total : " + totalBlocks
        + " Safe : " + safeBlocks);
    totalBlocks = namesystem.getBlocksTotal();
    safeBlocks = namesystem.getSafeBlocks();
  }

  assertEquals(11, totalBlocks);
  assertEquals(totalBlocks, safeBlocks);
  for (DataNode dn : cluster.getDataNodes()) {
    assertEquals(1, dn.data.getBlocksBeingWrittenReport(cluster.getNameNode()
        .getNamespaceID()).length);
  }
}
 

public void run() {
  try {
    fs.mkdirs(outputPath); 
    long endTime = System.currentTimeMillis() + rtc.max_time;
    long lastRollTime = System.currentTimeMillis();
    long lastSyncTime = System.currentTimeMillis();
    long currentId = 0;
    FSDataOutputStream out = null;
    while (System.currentTimeMillis() < endTime) {
      Path fileName = new Path(outputPath, "part" + currentId);
      try { 
        out = fs.create(fileName, (short)3);
        dc.openFile();

        long size = 0;
        while (true) {
          rb.nextBytes(buffer);
          dc.getFileChecksum().update(buffer, 0, rtc.buffer_size);
          tb.getTokens(rtc.buffer_size);
          out.write(buffer, 0, rtc.buffer_size);
          
          size += rtc.buffer_size;
          if (rtc.sync_interval > 0 &&
              System.currentTimeMillis() - lastSyncTime > rtc.sync_interval) {
            // Sync the file
            out.sync();
            LOG.info("file " + fileName + " is synced");
            lastSyncTime = System.currentTimeMillis() + 
                rb.nextInt((int)rtc.sync_interval);
          }
          if (System.currentTimeMillis() > endTime ||
              rtc.roll_interval > 0 &&
              System.currentTimeMillis() - lastRollTime > rtc.roll_interval) {
            // Roll the file
            out.close();
            out = null;
            currentId++;
            files_processed++;
            processed_size += size;
            LOG.info("file " + fileName + " is closed with " + size + " bytes");
            lastRollTime = System.currentTimeMillis() + 
                rb.nextInt((int)rtc.roll_interval);
            break;
          }
        }
      } catch (Exception e) {
        LOG.error("Error in writing file: " + fileName, e);
        this.errors.add(e);
      } finally {
        IOUtils.closeStream(out);
        dc.closeFile();
      }
    }
    LOG.info("Checksum of files under dir " + outputPath + " is " + dc.getDirectoryChecksum());
    LOG.info("Thread " + name + "_" + id + " is done.");
  } catch (Exception ioe) {
    LOG.error("Error:", ioe);
    this.errors.add(ioe);
  }
}
 

private void createRBWFile(String fileName) throws IOException {
  FSDataOutputStream out = dafs.create(new Path(fileName));
  random.nextBytes(buffer);
  out.write(buffer);
  out.sync();
}
 

private void writeFileAndSync(FSDataOutputStream stm, int size)
  throws IOException {
  byte[] buffer = DFSTestUtil.generateSequentialBytes(0, size);
  stm.write(buffer, 0, size);
  stm.sync();
}