Java Code Examples for org.apache.hadoop.fs.FileSystem#append()

HadoopRecoverableFsDataOutputStream(
		FileSystem fs,
		HadoopFsRecoverable recoverable) throws IOException {

	ensureTruncateInitialized();

	this.fs = checkNotNull(fs);
	this.targetFile = checkNotNull(recoverable.targetFile());
	this.tempFile = checkNotNull(recoverable.tempFile());

	safelyTruncateFile(fs, tempFile, recoverable);

	out = fs.append(tempFile);

	// sanity check
	long pos = out.getPos();
	if (pos != recoverable.offset()) {
		IOUtils.closeQuietly(out);
		throw new IOException("Truncate failed: " + tempFile +
				" (requested=" + recoverable.offset() + " ,size=" + pos + ')');
	}
}
 

HadoopRecoverableFsDataOutputStream(
		FileSystem fs,
		HadoopFsRecoverable recoverable) throws IOException {

	ensureTruncateInitialized();

	this.fs = checkNotNull(fs);
	this.targetFile = checkNotNull(recoverable.targetFile());
	this.tempFile = checkNotNull(recoverable.tempFile());

	safelyTruncateFile(fs, tempFile, recoverable);

	out = fs.append(tempFile);

	// sanity check
	long pos = out.getPos();
	if (pos != recoverable.offset()) {
		IOUtils.closeQuietly(out);
		throw new IOException("Truncate failed: " + tempFile +
				" (requested=" + recoverable.offset() + " ,size=" + pos + ')');
	}
}
 

@Test
public void append_shouldFail_whenFileDoesNotExist() throws Exception {
  URI path = gcsFsIHelper.getUniqueObjectUri("append_shouldFail_whenFileDoesNotExist");
  Path hadoopPath = new Path(path);

  FileSystem ghfs = GoogleHadoopFileSystemIntegrationHelper.createGhfs(path, getTestConfig());

  // Test appending three 9-character strings to existing object using 20 bytes buffer size
  FSDataOutputStream fsos = ghfs.append(hadoopPath, 20, /* progress= */ () -> {});
  fsos.write("_append-1".getBytes(UTF_8));

  assertThrows(GoogleJsonResponseException.class, fsos::hsync);

  assertThrows(NullPointerException.class, fsos::close);

  // Validate that file wasn't created
  assertThat(ghfs.exists(hadoopPath)).isFalse();
}
 

/** {@inheritDoc} */
@Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
    @Nullable Map<String, String> props) {
    try {
        Path hadoopPath = convert(path);

        FileSystem fs = fileSystemForUser();

        if (create && !fs.exists(hadoopPath))
            return fs.create(hadoopPath, false, bufSize);
        else
            return fs.append(convert(path), bufSize);
    }
    catch (IOException e) {
        throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]");
    }
}
 

@Override
public Void run() throws Exception {
    String effectiveDirPath = "/user/" + (serviceAsNamespace ? "" : (hdfsUser + "/")) + filePath;
    FileSystem fileSystem = fsGetter.get();
    Path path = new Path(effectiveDirPath);
    FSDataOutputStream out = fileSystem.append(path);

    if (out == null) {
        fileSystem.close();
        throw new CygnusPersistenceError("The /user/" + (serviceAsNamespace ? "" : (hdfsUser + "/"))
                + filePath + " file could not be created in HDFS");
    } // if

    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out));
    writer.append(data + "\n");
    writer.close();
    fileSystem.close();
    return null;
}
 

protected void open(Path dstPath, CompressionCodec codeC,
    CompressionType compType, Configuration conf, FileSystem hdfs)
        throws IOException {
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
          (dstPath)) {
    outStream = hdfs.append(dstPath);
  } else {
    outStream = hdfs.create(dstPath);
  }
  writer = SequenceFile.createWriter(conf, outStream,
      serializer.getKeyClass(), serializer.getValueClass(), compType, codeC);

  registerCurrentStream(outStream, hdfs, dstPath);
}
 

private long copyToFile(Path targetPath, FileSystem targetFS,
    FileStatus sourceFileStatus, long sourceOffset, Mapper.Context context,
    EnumSet<FileAttribute> fileAttributes, final FileChecksum sourceChecksum)
    throws IOException {
  FsPermission permission = FsPermission.getFileDefault().applyUMask(
      FsPermission.getUMask(targetFS.getConf()));
  final OutputStream outStream;
  if (action == FileAction.OVERWRITE) {
    final short repl = getReplicationFactor(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    final long blockSize = getBlockSize(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    FSDataOutputStream out = targetFS.create(targetPath, permission,
        EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
        BUFFER_SIZE, repl, blockSize, context,
        getChecksumOpt(fileAttributes, sourceChecksum));
    outStream = new BufferedOutputStream(out);
  } else {
    outStream = new BufferedOutputStream(targetFS.append(targetPath,
        BUFFER_SIZE));
  }
  return copyBytes(sourceFileStatus, sourceOffset, outStream, BUFFER_SIZE,
      context);
}
 

protected FSDataOutputStream getHdfsStream() throws IllegalArgumentException, IOException {
    if (hdfsStream == null) {
        Path path = getPath();
        FileSystem fs = getFileSystemAsUser(getConfiguration(), getUserGroupInformation());
        hdfsStream = fs.exists(path) ? fs.append(path) : fs.create(path);
    }
    return hdfsStream;
}
 

/**
 * Test which randomly alternates between appending with
 * CRC32 and with CRC32C, crossing several block boundaries.
 * Then, checks that all of the data can be read back correct.
 */
@Test(timeout=RANDOM_TEST_RUNTIME*2)
public void testAlgoSwitchRandomized() throws IOException {
  FileSystem fsWithCrc32 = createFsWithChecksum("CRC32", 512);
  FileSystem fsWithCrc32C = createFsWithChecksum("CRC32C", 512);

  Path p = new Path("/testAlgoSwitchRandomized");
  long seed = Time.now();
  System.out.println("seed: " + seed);
  Random r = new Random(seed);
  
  // Create empty to start
  IOUtils.closeStream(fsWithCrc32.create(p));
  
  long st = Time.now();
  int len = 0;
  while (Time.now() - st < RANDOM_TEST_RUNTIME) {
    int thisLen = r.nextInt(500);
    FileSystem fs = (r.nextBoolean() ? fsWithCrc32 : fsWithCrc32C);
    FSDataOutputStream stm = fs.append(p);
    try {
      AppendTestUtil.write(stm, len, thisLen);
    } finally {
      stm.close();
    }
    len += thisLen;
  }
  
  AppendTestUtil.check(fsWithCrc32, p, len);
  AppendTestUtil.check(fsWithCrc32C, p, len);
}
 

private static void writeTestDataToFile(FileSystem fs) throws IOException {
  OutputStream out = null;
  if (!fs.exists(TEST_PATH)) {
    out = fs.create(TEST_PATH);
  } else {
    out = fs.append(TEST_PATH);
  }
  out.write(PLAIN_TEXT.getBytes());
  out.close();
}
 

protected void doOpen(Configuration conf,
  Path dstPath, FileSystem hdfs) throws
  IOException {
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }

  boolean appending = false;
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
          (dstPath)) {
    outStream = hdfs.append(dstPath);
    appending = true;
  } else {
    outStream = hdfs.create(dstPath);
  }

  serializer = EventSerializerFactory.getInstance(
      serializerType, serializerContext, outStream);
  if (appending && !serializer.supportsReopen()) {
    outStream.close();
    serializer = null;
    throw new IOException("serializer (" + serializerType +
        ") does not support append");
  }

  // must call superclass to check for replication issues
  registerCurrentStream(outStream, hdfs, dstPath);

  if (appending) {
    serializer.afterReopen();
  } else {
    serializer.afterCreate();
  }
}
 

/**
 * Test which randomly alternates between appending with
 * CRC32 and with CRC32C, crossing several block boundaries.
 * Then, checks that all of the data can be read back correct.
 */
@Test(timeout=RANDOM_TEST_RUNTIME*2)
public void testAlgoSwitchRandomized() throws IOException {
  FileSystem fsWithCrc32 = createFsWithChecksum("CRC32", 512);
  FileSystem fsWithCrc32C = createFsWithChecksum("CRC32C", 512);

  Path p = new Path("/testAlgoSwitchRandomized");
  long seed = Time.now();
  System.out.println("seed: " + seed);
  Random r = new Random(seed);
  
  // Create empty to start
  IOUtils.closeStream(fsWithCrc32.create(p));
  
  long st = Time.now();
  int len = 0;
  while (Time.now() - st < RANDOM_TEST_RUNTIME) {
    int thisLen = r.nextInt(500);
    FileSystem fs = (r.nextBoolean() ? fsWithCrc32 : fsWithCrc32C);
    FSDataOutputStream stm = fs.append(p);
    try {
      AppendTestUtil.write(stm, len, thisLen);
    } finally {
      stm.close();
    }
    len += thisLen;
  }
  
  AppendTestUtil.check(fsWithCrc32, p, len);
  AppendTestUtil.check(fsWithCrc32C, p, len);
}
 

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    
//    conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));
//    conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
//    conf.addResource(new Path("/etc/hadoop/conf/yarn-site.xml"));
//    conf.addResource(new Path("/etc/hadoop/conf/mapred-site.xml"));
    String hostname="";
    try {
      hostname = InetAddress.getLocalHost().getHostName();
    } catch (UnknownHostException e) {
      e.printStackTrace();
    }
    
    String uri = "hdfs://"+hostname+":8020";
    System.out.println(uri);
    
    
    System.out.println("STARTING");
    String filepath = uri+"/user/yarn/"+UUID.randomUUID().toString();
    System.out.println(filepath);
    Path path = new Path(filepath);
    
    System.out.println("FIRST WRITE");
    FileSystem fs = FileSystem.get(URI.create(uri), conf);
    FSDataOutputStream os = fs.create(path);
    os.write("hellothere\n".getBytes());
    os.close();
    
    System.out.println("APPEND");
    FileSystem fs2 = FileSystem.get(URI.create(uri), conf);
    System.out.println("GETTING OUTPUT STREAM");
    FSDataOutputStream os2 = fs2.append(path);
    System.out.println("DOING THE SECOND WRITE");
    os2.write("Hope this works\n".getBytes());
    System.out.println("CLOSING SECOND WRITE");
    os2.close();
    System.out.println("DONE");
    
  }
 

/**
 * Test that a file is not considered complete when it only has in-progress
 * blocks. This ensures that when a block is appended to, it is converted
 * back into the right kind of "in progress" state.
 */
public void testNotPrematurelyComplete() throws Exception {
  LOG.info("START");
  cluster = new MiniDFSCluster(conf, 3, true, null);
  FileSystem fs1 = cluster.getFileSystem();
  try {
    int halfBlock = (int)BLOCK_SIZE/2;
    short rep = 3; // replication
    assertTrue(BLOCK_SIZE%4 == 0);

    file1 = new Path("/delayedReceiveBlock");

    // write 1/2 block & close
    stm = fs1.create(file1, true, (int)BLOCK_SIZE*2, rep, BLOCK_SIZE);
    AppendTestUtil.write(stm, 0, halfBlock);
    stm.close();

    NameNode nn = cluster.getNameNode();
    LOG.info("======== Appending");
    stm = fs1.append(file1);
    LOG.info("======== Writing");
    AppendTestUtil.write(stm, 0, halfBlock/2);
    LOG.info("======== Checking progress");
    assertFalse(NameNodeAdapter.checkFileProgress(nn.namesystem, "/delayedReceiveBlock", true));
    LOG.info("======== Closing");
    stm.close();

  } catch (Throwable e) {
    e.printStackTrace();
    throw new IOException(e);
  } finally {
    LOG.info("======== Cleaning up");
    fs1.close();
    cluster.shutdown();
  }
}
 

private static void writeTestDataToFile(FileSystem fs) throws IOException {
  OutputStream out = null;
  if (!fs.exists(TEST_PATH)) {
    out = fs.create(TEST_PATH);
  } else {
    out = fs.append(TEST_PATH);
  }
  out.write(PLAIN_TEXT.getBytes());
  out.close();
}
 

@Override
public void open(String filePath, CompressionCodec codec,
    CompressionType cType) throws IOException {
  Configuration conf = new Configuration();
  Path dstPath = new Path(filePath);
  FileSystem hdfs = dstPath.getFileSystem(conf);
  if(useRawLocalFileSystem) {
    if(hdfs instanceof LocalFileSystem) {
      hdfs = ((LocalFileSystem)hdfs).getRaw();
    } else {
      logger.warn("useRawLocalFileSystem is set to true but file system " +
          "is not of type LocalFileSystem: " + hdfs.getClass().getName());
    }
  }

  boolean appending = false;
  if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
  (dstPath)) {
    fsOut = hdfs.append(dstPath);
    appending = true;
  } else {
    fsOut = hdfs.create(dstPath);
  }
  cmpOut = codec.createOutputStream(fsOut);
  serializer = EventSerializerFactory.getInstance(serializerType,
      serializerContext, cmpOut);
  if (appending && !serializer.supportsReopen()) {
    cmpOut.close();
    serializer = null;
    throw new IOException("serializer (" + serializerType
        + ") does not support append");
  }

  registerCurrentStream(fsOut, hdfs, dstPath);

  if (appending) {
    serializer.afterReopen();
  } else {
    serializer.afterCreate();
  }
  isFinished = false;
}
 

private void appendOneByte(FileSystem fs, Path p) throws IOException {
  final FSDataOutputStream out = fs.append(p);
  out.write((byte) 0x123);
  out.close();
}
 

/**
 * Test to verify the processing of PendingDataNodeMessageQueue in case of
 * append. One block will marked as corrupt if the OP_ADD, OP_UPDATE_BLOCKS
 * comes in one edit log segment and OP_CLOSE edit comes in next log segment
 * which is loaded during failover. Regression test for HDFS-3605.
 */
@Test
public void testMultipleAppendsDuringCatchupTailing() throws Exception {
  Configuration conf = new Configuration();
  
  // Set a length edits tailing period, and explicit rolling, so we can
  // control the ingest of edits by the standby for this test.
  conf.set(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, "5000");
  conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, -1);

  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .nnTopology(MiniDFSNNTopology.simpleHATopology())
      .numDataNodes(3).build();
  FileSystem fs = null;
  try {
    cluster.transitionToActive(0);
    fs = HATestUtil.configureFailoverFs(cluster, conf);

    Path fileToAppend = new Path("/FileToAppend");
    Path fileToTruncate = new Path("/FileToTruncate");
    
    final byte[] data = new byte[1 << 16];
    DFSUtil.getRandom().nextBytes(data);
    final int[] appendPos = AppendTestUtil.randomFilePartition(
        data.length, COUNT);
    final int[] truncatePos = AppendTestUtil.randomFilePartition(
        data.length, 1);

    // Create file, write some data, and hflush so that the first
    // block is in the edit log prior to roll.
    FSDataOutputStream out = createAndHflush(
        fs, fileToAppend, data, appendPos[0]);

    FSDataOutputStream out4Truncate = createAndHflush(
        fs, fileToTruncate, data, data.length);
    
    // Let the StandbyNode catch the creation of the file. 
    cluster.getNameNode(0).getRpcServer().rollEditLog();
    cluster.getNameNode(1).getNamesystem().getEditLogTailer().doTailEdits();
    out.close();
    out4Truncate.close();

    // Append and re-close a few time, so that many block entries are queued.
    for (int i = 0; i < COUNT; i++) {
      int end = i < COUNT - 1? appendPos[i + 1]: data.length;
      out = fs.append(fileToAppend);
      out.write(data, appendPos[i], end - appendPos[i]);
      out.close();
    }
    boolean isTruncateReady = fs.truncate(fileToTruncate, truncatePos[0]);

    // Ensure that blocks have been reported to the SBN ahead of the edits
    // arriving.
    cluster.triggerBlockReports();

    // Failover the current standby to active.
    cluster.shutdownNameNode(0);
    cluster.transitionToActive(1);
    
    // Check the FSCK doesn't detect any bad blocks on the SBN.
    int rc = ToolRunner.run(new DFSck(cluster.getConfiguration(1)),
        new String[] { "/", "-files", "-blocks" });
    assertEquals(0, rc);
    
    assertEquals("CorruptBlocks should be empty.", 0, cluster.getNameNode(1)
        .getNamesystem().getCorruptReplicaBlocks());

    AppendTestUtil.checkFullFile(fs, fileToAppend, data.length, data,
        fileToAppend.toString());

    if (!isTruncateReady) {
      TestFileTruncate.checkBlockRecovery(fileToTruncate,
          cluster.getFileSystem(1));
    }
    AppendTestUtil.checkFullFile(fs, fileToTruncate, truncatePos[0], data,
        fileToTruncate.toString());
  } finally {
    if (null != cluster) {
      cluster.shutdown();
    }
    if (null != fs) {
      fs.close();
    }
  }
}
 

/**
 * Test for a race in appendFile where the file might get removed in between
 * the two synchronized sections.
 */
public void testAppendFileRace() throws Throwable {
  LOG.info("START");
  cluster = new MiniDFSCluster(conf, 1, true, null);
  final FileSystem fs1 = cluster.getFileSystem();;

  try {
    createFile(fs1, "/testAppendFileRace", 1, BBW_SIZE);
    stm.close();

    NameNode nn = cluster.getNameNode();
    FSEditLog editLogSpy = FSImageAdapter.injectEditLogSpy(nn.getNamesystem());
    DelayAnswer  delayer = new DelayAnswer();
    doAnswer(delayer).when(editLogSpy).logSync();

    final AtomicReference<Throwable> err = new AtomicReference<Throwable>();
    Thread appender = new Thread() {
        public void run() {
          try {
            stm = fs1.append(file1);
          } catch (Throwable t) {
            err.set(t);
          }
        }
      };
    LOG.info("Triggering append in other thread");
    appender.start();

    LOG.info("Waiting for logsync");
    delayer.waitForCall();

    LOG.info("Resetting spy");
    reset(editLogSpy);

    LOG.info("Deleting file");
    fs1.delete(file1, true);

    LOG.info("Allowing append to proceed");
    delayer.proceed();

    LOG.info("Waiting for append to finish");

    appender.join();

    if (err.get() != null) {
      if (err.get().getMessage().contains(
            "File does not exist.")) {
        LOG.info("Got expected exception", err.get());
      } else {
        throw err.get();
      }
    }
    LOG.info("Closing stream");
    stm.close();
  } finally {
    fs1.close();
    cluster.shutdown();
  }
}
 

@Test(timeout=120000)
public void testDecommissionWithOpenfile() throws IOException, InterruptedException {
  LOG.info("Starting test testDecommissionWithOpenfile");
  
  //At most 4 nodes will be decommissioned
  startCluster(1, 7, conf);
      
  FileSystem fileSys = cluster.getFileSystem(0);
  FSNamesystem ns = cluster.getNamesystem(0);
  
  String openFile = "/testDecommissionWithOpenfile.dat";
         
  writeFile(fileSys, new Path(openFile), (short)3);   
  // make sure the file was open for write
  FSDataOutputStream fdos =  fileSys.append(new Path(openFile)); 
  
  LocatedBlocks lbs = NameNodeAdapter.getBlockLocations(cluster.getNameNode(0), openFile, 0, fileSize);
            
  DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations();
  DatanodeInfo[] dnInfos4FirstBlock = lbs.get(0).getLocations();
  
  ArrayList<String> nodes = new ArrayList<String>();
  ArrayList<DatanodeInfo> dnInfos = new ArrayList<DatanodeInfo>();

  DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
  for (DatanodeInfo datanodeInfo : dnInfos4FirstBlock) {
    DatanodeInfo found = datanodeInfo;
    for (DatanodeInfo dif: dnInfos4LastBlock) {
      if (datanodeInfo.equals(dif)) {
       found = null;
      }
    }
    if (found != null) {
      nodes.add(found.getXferAddr());
      dnInfos.add(dm.getDatanode(found));
    }
  }
  //decommission one of the 3 nodes which have last block
  nodes.add(dnInfos4LastBlock[0].getXferAddr());
  dnInfos.add(dm.getDatanode(dnInfos4LastBlock[0]));
  
  writeConfigFile(excludeFile, nodes);
  refreshNodes(ns, conf);  
  for (DatanodeInfo dn : dnInfos) {
    waitNodeState(dn, AdminStates.DECOMMISSIONED);
  }           

  fdos.close();
}