Java Code Examples for org.apache.hadoop.io.IOUtils#copyBytes()

/**
 * Copy a file from a srcFs {@link FileSystem} to a dstFs {@link FileSystem}. The src {@link Path} must be a file,
 * that is {@link FileSystem#isFile(Path)} must return true for src.
 *
 * <p>
 *   If overwrite is specified to true, this method may delete the dst directory even if the copy from src to dst fails.
 * </p>
 *
 * @param srcFs the src {@link FileSystem} to copy the file from
 * @param src the src {@link Path} to copy
 * @param dstFs the destination {@link FileSystem} to write to
 * @param dst the destination {@link Path} to write to
 * @param overwrite true if the dst {@link Path} should be overwritten, false otherwise
 */
public static void copyFile(FileSystem srcFs, Path src, FileSystem dstFs, Path dst, boolean overwrite,
    Configuration conf) throws IOException {

  Preconditions.checkArgument(srcFs.isFile(src),
      String.format("Cannot copy from %s to %s because src is not a file", src, dst));
  Preconditions.checkArgument(overwrite || !dstFs.exists(dst),
      String.format("Cannot copy from %s to %s because dst exists", src, dst));

  try (InputStream in = srcFs.open(src); OutputStream out = dstFs.create(dst, overwrite)) {
    IOUtils.copyBytes(in, out, conf, false);
  } catch (Throwable t1) {
    try {
      deleteIfExists(dstFs, dst, true);
    } catch (Throwable t2) {
      // Do nothing
    }
    throw t1;
  }
}
 

public static String read(String path) {
	String content = null;
	Path dst = new Path(path);
	try {
		FSDataInputStream in = hdfs.open(dst);
		OutputStream os = new ByteArrayOutputStream();
		IOUtils.copyBytes(in, os, 4096, true);
		content = os.toString();
		os.close();
		in.close();
		log.info("read hdfs content from " + path + " successed. ");
	} catch (Exception e) {
		log.error("read hdfs content from " + path + " failed. ", e);
	}
	return content;
}
 

@Test
public void testSchemaFromHdfs() throws IOException {
  MiniDFSTest.setupFS();
  FileSystem fs = MiniDFSTest.getDFS();

  // copy a schema to HDFS
  Path schemaPath = fs.makeQualified(new Path("schema.avsc"));
  FSDataOutputStream out = fs.create(schemaPath);
  IOUtils.copyBytes(DatasetTestUtilities.USER_SCHEMA_URL.toURL().openStream(),
      out, fs.getConf());
  out.close();

  // build a schema using the HDFS path and check it's the same
  Schema schema = new DatasetDescriptor.Builder().schemaUri(schemaPath.toUri()).build()
      .getSchema();

  Assert.assertEquals(DatasetTestUtilities.USER_SCHEMA, schema);
  MiniDFSTest.teardownFS();
}
 

@Override
protected void processArguments(LinkedList<PathData> items)
throws IOException {
  super.processArguments(items);
  if (exitCode != 0) { // check for error collecting paths
    return;
  }
  FSDataOutputStream out = dst.fs.create(dst.path);
  try {
    for (PathData src : srcs) {
      FSDataInputStream in = src.fs.open(src.path);
      try {
        IOUtils.copyBytes(in, out, getConf(), false);
        if (delimiter != null) {
          out.write(delimiter.getBytes("UTF-8"));
        }
      } finally {
        in.close();
      }
    }
  } finally {
    out.close();
  }      
}
 

/** Copy FileSystem files to local files. */
private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
                            File dst, boolean deleteSource,
                            Configuration conf) throws IOException {
  Path src = srcStatus.getPath();
  if (srcStatus.isDirectory()) {
    if (!dst.mkdirs()) {
      return false;
    }
    FileStatus contents[] = srcFS.listStatus(src);
    for (int i = 0; i < contents.length; i++) {
      copy(srcFS, contents[i],
           new File(dst, contents[i].getPath().getName()),
           deleteSource, conf);
    }
  } else {
    InputStream in = srcFS.open(src);
    IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
  }
  if (deleteSource) {
    return srcFS.delete(src, true);
  } else {
    return true;
  }
}
 

@Override
public void write(OutputStream os) throws IOException {
  IOUtils.skipFully(is, offset);
  if (len == -1) {
    IOUtils.copyBytes(is, os, 4096, true);
  } else {
    IOUtils.copyBytes(is, os, len, true);
  }
}
 

void writeStreamToFile(InputStream in, PathData target,
                       boolean lazyPersist) throws IOException {
  FSDataOutputStream out = null;
  try {
    out = create(target, lazyPersist);
    IOUtils.copyBytes(in, out, getConf(), true);
  } finally {
    IOUtils.closeStream(out); // just in case copyBytes didn't
  }
}
 

/**
 * Unpack matching files from a jar. Entries inside the jar that do
 * not match the given pattern will be skipped.
 *
 * @param jarFile the .jar file to unpack
 * @param toDir the destination directory into which to unpack the jar
 * @param unpackRegex the pattern to match jar entries against
 */
public static void unJar(File jarFile, File toDir, Pattern unpackRegex)
  throws IOException {
  JarFile jar = new JarFile(jarFile);
  try {
    Enumeration<JarEntry> entries = jar.entries();
    while (entries.hasMoreElements()) {
      final JarEntry entry = entries.nextElement();
      if (!entry.isDirectory() &&
          unpackRegex.matcher(entry.getName()).matches()) {
        InputStream in = jar.getInputStream(entry);
        try {
          File file = new File(toDir, entry.getName());
          ensureDirectory(file.getParentFile());
          OutputStream out = new FileOutputStream(file);
          try {
            IOUtils.copyBytes(in, out, 8192);
          } finally {
            out.close();
          }
        } finally {
          in.close();
        }
      }
    }
  } finally {
    jar.close();
  }
}
 

/**
 * Compress a file.
 */
private int compress(String argv[], Configuration conf) throws IOException {
  int i = 0;
  String cmd = argv[i++];
  String srcf = argv[i++];
  String dstf = argv[i++];

  Path srcPath = new Path(srcf);
  FileSystem srcFs = srcPath.getFileSystem(getConf());
  Path dstPath = new Path(dstf);
  FileSystem dstFs = dstPath.getFileSystem(getConf());

  // Create codec
  CompressionCodecFactory factory = new CompressionCodecFactory(conf);
  CompressionCodec codec = factory.getCodec(dstPath);
  if (codec == null) {
    System.err.println(cmd.substring(1) + ": cannot find compression codec for "
        + dstf);
    return 1;
  }

  // open input stream
  InputStream in = srcFs.open(srcPath);

  // Create compression stream
  OutputStream out = dstFs.create(dstPath);
  out = codec.createOutputStream(out);

  IOUtils.copyBytes(in, out, conf, true);

  return 0;
}
 

void writeStreamToFile(InputStream in, PathData target,
                       boolean lazyPersist) throws IOException {
  FSDataOutputStream out = null;
  try {
    out = create(target, lazyPersist);
    IOUtils.copyBytes(in, out, getConf(), true);
  } finally {
    IOUtils.closeStream(out); // just in case copyBytes didn't
  }
}
 

/**
 * Cleanup the error stream if any, for keepAlive connections
 *
 * @param errorStream
 */
private void readErrorStream(InputStream errorStream) {
  if (errorStream == null) {
    return;
  }
  try {
    DataOutputBuffer errorBuffer = new DataOutputBuffer();
    IOUtils.copyBytes(errorStream, errorBuffer, 4096);
    IOUtils.closeStream(errorBuffer);
    IOUtils.closeStream(errorStream);
  } catch (IOException ioe) {
    // ignore
  }
}
 

@Test
public void testGzipCompatibility() throws IOException {
  Random r = new Random();
  long seed = r.nextLong();
  r.setSeed(seed);
  LOG.info("seed: " + seed);

  DataOutputBuffer dflbuf = new DataOutputBuffer();
  GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
  byte[] b = new byte[r.nextInt(128 * 1024 + 1)];
  r.nextBytes(b);
  gzout.write(b);
  gzout.close();

  DataInputBuffer gzbuf = new DataInputBuffer();
  gzbuf.reset(dflbuf.getData(), dflbuf.getLength());

  Configuration conf = new Configuration();
  conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
  CompressionCodec codec = ReflectionUtils.newInstance(GzipCodec.class, conf);
  Decompressor decom = codec.createDecompressor();
  assertNotNull(decom);
  assertEquals(BuiltInGzipDecompressor.class, decom.getClass());
  InputStream gzin = codec.createInputStream(gzbuf, decom);

  dflbuf.reset();
  IOUtils.copyBytes(gzin, dflbuf, 4096);
  final byte[] dflchk = Arrays.copyOf(dflbuf.getData(), dflbuf.getLength());
  assertArrayEquals(b, dflchk);
}
 

/**
 * @return URL contents as a byte array
 */
public static byte[] urlGetBytes(URL url) throws IOException {
  URLConnection conn = url.openConnection();
  HttpURLConnection hc = (HttpURLConnection)conn;
  
  assertEquals(HttpURLConnection.HTTP_OK, hc.getResponseCode());
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
  return out.toByteArray();
}
 

@Override
public void concat(final Path trg, final Path [] psrcs) throws IOException {
  final int bufferSize = 4096;
  try(FSDataOutputStream out = create(trg)) {
    for (Path src : psrcs) {
      try(FSDataInputStream in = open(src)) {
        IOUtils.copyBytes(in, out, bufferSize, false);
      }
    }
  }
}
 

/** Copy files between FileSystems. */
public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
                           FileSystem dstFS, Path dst,
                           boolean deleteSource,
                           boolean overwrite,
                           Configuration conf) throws IOException {
  Path src = srcStatus.getPath();
  dst = checkDest(src.getName(), dstFS, dst, overwrite);
  if (srcStatus.isDirectory()) {
    checkDependencies(srcFS, src, dstFS, dst);
    if (!dstFS.mkdirs(dst)) {
      return false;
    }
    FileStatus contents[] = srcFS.listStatus(src);
    for (int i = 0; i < contents.length; i++) {
      copy(srcFS, contents[i], dstFS,
           new Path(dst, contents[i].getPath().getName()),
           deleteSource, overwrite, conf);
    }
  } else {
    InputStream in=null;
    OutputStream out = null;
    try {
      in = srcFS.open(src);
      out = dstFS.create(dst, overwrite);
      IOUtils.copyBytes(in, out, conf, true);
    } catch (IOException e) {
      IOUtils.closeStream(out);
      IOUtils.closeStream(in);
      throw e;
    }
  }
  if (deleteSource) {
    return srcFS.delete(src, true);
  } else {
    return true;
  }

}
 

public void testDatanodeBlockScanner() throws IOException {
  
  long startTime = System.currentTimeMillis();
  
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  
  FileSystem fs = cluster.getFileSystem();
  Path file1 = new Path("/tmp/testBlockVerification/file1");
  Path file2 = new Path("/tmp/testBlockVerification/file2");
  
  /*
   * Write the first file and restart the cluster.
   */
  DFSTestUtil.createFile(fs, file1, 10, (short)1, 0);
  cluster.shutdown();
  cluster = new MiniDFSCluster(conf, 1, false, null);
  cluster.waitActive();
  
  DFSClient dfsClient =  new DFSClient(new InetSocketAddress("localhost", 
                                       cluster.getNameNodePort()), conf);
  fs = cluster.getFileSystem();
  DatanodeInfo dn = dfsClient.datanodeReport(DatanodeReportType.LIVE)[0];
  
  /*
   * The cluster restarted. The block should be verified by now.
   */
  assertTrue(waitForVerification(dn, fs, file1) > startTime);
  
  /*
   * Create a new file and read the block. The block should be marked 
   * verified since the client reads the block and verifies checksum. 
   */
  DFSTestUtil.createFile(fs, file2, 10, (short)1, 0);
  IOUtils.copyBytes(fs.open(file2), new IOUtils.NullOutputStream(), 
                    conf, true); 
  assertTrue(waitForVerification(dn, fs, file2) > startTime);
  
  cluster.shutdown();
}
 

/**
 * Download link and have it be the response.
 * @param req the http request
 * @param resp the http response
 * @param link the link to download
 * @param c the cookie to set if any
 * @throws IOException on any error.
 */
private static void proxyLink(HttpServletRequest req, 
    HttpServletResponse resp, URI link, Cookie c, String proxyHost)
    throws IOException {
  DefaultHttpClient client = new DefaultHttpClient();
  client
      .getParams()
      .setParameter(ClientPNames.COOKIE_POLICY,
          CookiePolicy.BROWSER_COMPATIBILITY)
      .setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true);
  // Make sure we send the request from the proxy address in the config
  // since that is what the AM filter checks against. IP aliasing or
  // similar could cause issues otherwise.
  InetAddress localAddress = InetAddress.getByName(proxyHost);
  if (LOG.isDebugEnabled()) {
    LOG.debug("local InetAddress for proxy host: {}", localAddress);
  }
  client.getParams()
      .setParameter(ConnRoutePNames.LOCAL_ADDRESS, localAddress);
  HttpGet httpGet = new HttpGet(link);
  @SuppressWarnings("unchecked")
  Enumeration<String> names = req.getHeaderNames();
  while(names.hasMoreElements()) {
    String name = names.nextElement();
    if(passThroughHeaders.contains(name)) {
      String value = req.getHeader(name);
      if (LOG.isDebugEnabled()) {
        LOG.debug("REQ HEADER: {} : {}", name, value);
      }
      httpGet.setHeader(name, value);
    }
  }

  String user = req.getRemoteUser();
  if (user != null && !user.isEmpty()) {
    httpGet.setHeader("Cookie",
        PROXY_USER_COOKIE_NAME + "=" + URLEncoder.encode(user, "ASCII"));
  }
  OutputStream out = resp.getOutputStream();
  try {
    HttpResponse httpResp = client.execute(httpGet);
    resp.setStatus(httpResp.getStatusLine().getStatusCode());
    for (Header header : httpResp.getAllHeaders()) {
      resp.setHeader(header.getName(), header.getValue());
    }
    if (c != null) {
      resp.addCookie(c);
    }
    InputStream in = httpResp.getEntity().getContent();
    if (in != null) {
      IOUtils.copyBytes(in, out, 4096, true);
    }
  } finally {
    httpGet.releaseConnection();
  }
}
 

/** Create and write the given file */
private static void createFile(File f, String s) throws IOException {
  InputStream is = new ByteArrayInputStream(s.getBytes());
  FileOutputStream fos = new FileOutputStream(f);
  IOUtils.copyBytes(is, fos, s.length(), true);
}
 

public void testDatanodeBlockScanner() throws IOException {
  
  long startTime = System.currentTimeMillis();
  
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  
  FileSystem fs = cluster.getFileSystem();
  Path file1 = new Path("/tmp/testBlockVerification/file1");
  Path file2 = new Path("/tmp/testBlockVerification/file2");
  
  /*
   * Write the first file and restart the cluster.
   */
  DFSTestUtil.createFile(fs, file1, 10, (short)1, 0);
  cluster.shutdown();
  cluster = new MiniDFSCluster(conf, 1, false, null);
  cluster.waitActive();
  
  DFSClient dfsClient =  new DFSClient(new InetSocketAddress("localhost", 
                                       cluster.getNameNodePort()), conf);
  fs = cluster.getFileSystem();
  DatanodeInfo dn = dfsClient.datanodeReport(DatanodeReportType.LIVE)[0];
  
  /*
   * The cluster restarted. The block should be verified by now.
   */
  assertTrue(waitForVerification(dn, fs, file1) > startTime);
  
  /*
   * Create a new file and read the block. The block should be marked 
   * verified since the client reads the block and verifies checksum. 
   */
  DFSTestUtil.createFile(fs, file2, 10, (short)1, 0);
  IOUtils.copyBytes(fs.open(file2), new IOUtils.NullOutputStream(), 
                    conf, true); 
  assertTrue(waitForVerification(dn, fs, file2) > startTime);
  
  cluster.shutdown();
}
 

@Override
protected void execute(OzoneClient client, OzoneAddress address)
    throws IOException, OzoneClientException {

  String volumeName = address.getVolumeName();
  String bucketName = address.getBucketName();

  OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
  OzoneBucket bucket = vol.getBucket(bucketName);

  if (replicationFactor == null) {
    replicationFactor = ReplicationFactor.valueOf(
        getConf().getInt(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT));
  }

  if (replicationType == null) {
    replicationType = ReplicationType.valueOf(
        getConf().get(OZONE_REPLICATION_TYPE,
            OZONE_REPLICATION_TYPE_DEFAULT));
  }

  OzoneKeyDetails keyDetail = bucket.getKey(fromKey);
  Map<String, String> keyMetadata = new HashMap<>(keyDetail.getMetadata());
  keyMetadata.remove(OzoneConsts.GDPR_SECRET);
  keyMetadata.remove(OzoneConsts.GDPR_ALGORITHM);

  String gdprEnabled = bucket.getMetadata().get(OzoneConsts.GDPR_FLAG);
  if (Boolean.parseBoolean(gdprEnabled)) {
    keyMetadata.put(OzoneConsts.GDPR_FLAG, Boolean.TRUE.toString());
  }

  int chunkSize = (int) getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
      OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
  try (InputStream input = bucket.readKey(fromKey);
       OutputStream output = bucket.createKey(toKey, input.available(),
           replicationType, replicationFactor, keyMetadata)) {
    IOUtils.copyBytes(input, output, chunkSize);
  }

  if (isVerbose()) {
    out().printf("Copy Key : %s to %s%n", fromKey, toKey);
  }
}