Java Code Examples for org.apache.commons.compress.archivers.tar.TarArchiveOutputStream#closeArchiveEntry()

private static void addToTar(TarArchiveOutputStream out, File file, String dir) throws IOException {
    String entry = dir + File.separator + file.getName();
    if (file.isFile()){
        out.putArchiveEntry(new TarArchiveEntry(file, entry));
        try (FileInputStream in = new FileInputStream(file)){
            org.apache.commons.compress.utils.IOUtils.copy(in, out);
        }
        out.closeArchiveEntry();
    } else if (file.isDirectory()) {
        File[] children = file.listFiles();
        if (children != null){
            for (File child : children){
                addToTar(out, child, entry);
            }
        }
    } else {
        System.out.println(file.getName() + " is not supported");
    }
}
 

private static void addToArchive(TarArchiveOutputStream taos, File dir, File root) throws IOException {
    byte[] buffer = new byte[1024];
    int length;
    int index = root.getAbsolutePath().length();
    for (File file : dir.listFiles()) {
        String name = file.getAbsolutePath().substring(index);
        if (file.isDirectory()) {
            if (file.listFiles().length != 0) {
                addToArchive(taos, file, root);
            } else {
                taos.putArchiveEntry(new TarArchiveEntry(name + File.separator));
                taos.closeArchiveEntry();
            }
        } else {
            try (FileInputStream fis = new FileInputStream(file)) {
                TarArchiveEntry entry = new TarArchiveEntry(name);
                entry.setSize(file.length());
                taos.putArchiveEntry(entry);
                while ((length = fis.read(buffer)) > 0) {
                    taos.write(buffer, 0, length);
                }
                taos.closeArchiveEntry();
            }
        }
    }
}
 

private static void addFilesToArchive(String source, File file,
                                      TarArchiveOutputStream
                                          tarFileOutputStream)
    throws IOException {
  tarFileOutputStream.putArchiveEntry(new TarArchiveEntry(file, source));
  if (file.isFile()) {
    try (FileInputStream fileInputStream = new FileInputStream(file)) {
      BufferedInputStream bufferedInputStream =
          new BufferedInputStream(fileInputStream);
      org.apache.commons.compress.utils.IOUtils.copy(bufferedInputStream,
          tarFileOutputStream);
      tarFileOutputStream.closeArchiveEntry();
    }
  } else if (file.isDirectory()) {
    tarFileOutputStream.closeArchiveEntry();
    File[] filesInDir = file.listFiles();
    if (filesInDir != null) {
      for (File cFile : filesInDir) {
        addFilesToArchive(cFile.getAbsolutePath(), cFile,
            tarFileOutputStream);
      }
    }
  }
}
 

public static File archiveTARFiles(File base, Iterable<File> files, String archiveNameWithOutExtension) throws IOException {
    File tarFile = new File(FileUtils.getTempDirectoryPath(), archiveNameWithOutExtension + ".tar");
    TarArchiveOutputStream tos = new TarArchiveOutputStream(new FileOutputStream(tarFile));
    try {
        tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
        for (File file : files) {
            TarArchiveEntry tarEntry = new TarArchiveEntry(file);
            tarEntry.setName(relativize(base, file));

            tos.putArchiveEntry(tarEntry);

            if (!file.isDirectory()) {
                FileUtils.copyFile(file, tos);
            }
            tos.closeArchiveEntry();
        }
    } finally {
        tos.close();
    }

    return tarFile;
}
 

private void writeAttributesEntry(final Map<String, String> attributes, final TarArchiveOutputStream tout) throws IOException {
    final StringBuilder sb = new StringBuilder();
    sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE properties\n  SYSTEM \"http://java.sun.com/dtd/properties.dtd\">\n");
    sb.append("<properties>");
    for (final Map.Entry<String, String> entry : attributes.entrySet()) {
        final String escapedKey = StringEscapeUtils.escapeXml11(entry.getKey());
        final String escapedValue = StringEscapeUtils.escapeXml11(entry.getValue());
        sb.append("\n  <entry key=\"").append(escapedKey).append("\">").append(escapedValue).append("</entry>");
    }
    sb.append("</properties>");

    final byte[] metaBytes = sb.toString().getBytes(StandardCharsets.UTF_8);
    final TarArchiveEntry attribEntry = new TarArchiveEntry(FILENAME_ATTRIBUTES);
    attribEntry.setMode(tarPermissions);
    attribEntry.setSize(metaBytes.length);
    tout.putArchiveEntry(attribEntry);
    tout.write(metaBytes);
    tout.closeArchiveEntry();
}
 

private void addControlContent ( final TarArchiveOutputStream out, final String name, final ContentProvider content, final int mode, final Supplier<Instant> timestampSupplier ) throws IOException
{
    if ( content == null || !content.hasContent () )
    {
        return;
    }

    final TarArchiveEntry entry = new TarArchiveEntry ( name );
    if ( mode >= 0 )
    {
        entry.setMode ( mode );
    }

    entry.setUserName ( "root" );
    entry.setGroupName ( "root" );
    entry.setSize ( content.getSize () );
    entry.setModTime ( timestampSupplier.get ().toEpochMilli () );
    out.putArchiveEntry ( entry );
    try ( InputStream stream = content.createInputStream () )
    {
        IOUtils.copy ( stream, out );
    }
    out.closeArchiveEntry ();
}
 

private void writeContentEntry(final TarArchiveOutputStream tarOut, final InputStream inStream, final long fileSize) throws IOException {
    final TarArchiveEntry entry = new TarArchiveEntry(FILENAME_CONTENT);
    entry.setMode(tarPermissions);
    entry.setSize(fileSize);
    tarOut.putArchiveEntry(entry);
    final byte[] buffer = new byte[512 << 10];//512KB
    int bytesRead = 0;
    while ((bytesRead = inStream.read(buffer)) != -1) { //still more data to read
        if (bytesRead > 0) {
            tarOut.write(buffer, 0, bytesRead);
        }
    }

    copy(inStream, tarOut);
    tarOut.closeArchiveEntry();
}
 

private static void addToArchiveCompression(TarArchiveOutputStream out, File file, String dir)
    throws IOException {
  if (file.isFile()){
    String archivePath = "." + dir;
    LOGGER.info("archivePath = " + archivePath);
    out.putArchiveEntry(new TarArchiveEntry(file, archivePath));
    try (FileInputStream in = new FileInputStream(file)) {
      IOUtils.copy(in, out);
    }
    out.closeArchiveEntry();
  } else if (file.isDirectory()) {
    File[] children = file.listFiles();
    if (children != null){
      for (File child : children){
        String appendDir = child.getAbsolutePath().replace(file.getAbsolutePath(), "");
        addToArchiveCompression(out, child, dir + appendDir);
      }
    }
  } else {
    LOGGER.error(file.getName() + " is not supported");
  }
}
 

private static void addFileToTar(String rootTarPath, File file, TarArchiveOutputStream tar)
  throws IOException {

  final String fileName =
    Optional.ofNullable(rootTarPath).orElse("") + TAR_PATH_DELIMITER + file.getName();
  tar.putArchiveEntry(new TarArchiveEntry(file, fileName));
  if (file.isFile()) {
    Files.copy(file.toPath(), tar);
    tar.closeArchiveEntry();
  } else if (file.isDirectory()) {
    tar.closeArchiveEntry();
    for (File fileInDirectory : file.listFiles()) {
      addFileToTar(fileName, fileInDirectory, tar);
    }
  }
}
 

/**
 * 目录归档
 *
 * @param dir
 * @param taos
 *            TarArchiveOutputStream
 * @param basePath
 * @throws Exception
 */
private static void archiveDir(File dir, TarArchiveOutputStream taos,
                               String basePath) throws Exception {

    File[] files = dir.listFiles();

    if (files.length < 1) {
        TarArchiveEntry entry = new TarArchiveEntry(basePath
                + dir.getName() + PATH);

        taos.putArchiveEntry(entry);
        taos.closeArchiveEntry();
    }

    for (File file : files) {

        // 递归归档
        archive(file, taos, basePath + dir.getName() + PATH);

    }
}
 

private static void addTarDir(
        TarArchiveOutputStream tout,
        Path base,
        File dir
) throws IOException {
    Preconditions.checkArgument(dir.isDirectory());
    String name = base.relativize(
            Paths.get(dir.getAbsolutePath())
    ).toString();
    ArchiveEntry entry = tout.createArchiveEntry(dir, name);
    tout.putArchiveEntry(entry);
    tout.closeArchiveEntry();
    for (File f : dir.listFiles()) {
        addTarEntry(tout, base, f);
    }
}
 

private static void addTarFile(
        TarArchiveOutputStream tout,
        Path base,
        File file
) throws IOException {
    Preconditions.checkArgument(
            file.isFile(),
            "given file" +
            " is not file: %s", file);
    checkFileSize(file.length());
    String name = base.relativize(
            Paths.get(file.getAbsolutePath())
    ).toString();
    ArchiveEntry entry = tout.createArchiveEntry(file, name);
    tout.putArchiveEntry(entry);
    try (InputStream in = new FileInputStream(file)) {
        IOUtils.copy(in, tout);
    }
    tout.closeArchiveEntry();
}
 

private void writeContentEntry(final TarArchiveOutputStream tarOut, final InputStream inStream, final long fileSize) throws IOException {
    final TarArchiveEntry entry = new TarArchiveEntry(FILENAME_CONTENT);
    entry.setMode(tarPermissions);
    entry.setSize(fileSize);
    tarOut.putArchiveEntry(entry);
    final byte[] buffer = new byte[512 << 10];//512KB
    int bytesRead = 0;
    while ((bytesRead = inStream.read(buffer)) != -1) { //still more data to read
        if (bytesRead > 0) {
            tarOut.write(buffer, 0, bytesRead);
        }
    }

    copy(inStream, tarOut);
    tarOut.closeArchiveEntry();
}
 

private static void writeToTarFile(TarArchiveOutputStream tos, String name,
                                   String content) throws IOException {
  TarArchiveEntry tarArchiveEntry = new TarArchiveEntry(name);
  if (null != content) {
    tarArchiveEntry.setSize(content.length());
  }
  tarArchiveEntry.setModTime(System.currentTimeMillis());
  tos.putArchiveEntry(tarArchiveEntry);
  if (null != content) {
    byte[] buf = content.getBytes();
    tos.write(buf, 0, content.length());
    tos.flush();
  }
  tos.closeArchiveEntry();
}
 

@Override
protected void sendCompressedDirectory(MCRPath file, BasicFileAttributes attrs,
    TarArchiveOutputStream container) throws IOException {
    TarArchiveEntry entry = new TarArchiveEntry(getFilename(file), TarConstants.LF_DIR);
    entry.setModTime(attrs.lastModifiedTime().toMillis());
    container.putArchiveEntry(entry);
    container.closeArchiveEntry();
}
 

static LocalResource createTgzFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File gzipFile = new File(p.toUri().getPath() + ".tar.gz");
  gzipFile.createNewFile();
  TarArchiveOutputStream out = new TarArchiveOutputStream(
      new GZIPOutputStream(new FileOutputStream(gzipFile)));
  TarArchiveEntry entry = new TarArchiveEntry(p.getName());
  entry.setSize(bytes.length);
  out.putArchiveEntry(entry);
  out.write(bytes);
  out.closeArchiveEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".tar.gz")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz"))
      .getModificationTime());
  return ret;
}
 

private String getReefTarUri(final String jobIdentifier) {
  try {
    // Create REEF_TAR
    final FileOutputStream fileOutputStream = new FileOutputStream(REEF_TAR);
    final TarArchiveOutputStream tarArchiveOutputStream =
        new TarArchiveOutputStream(new GZIPOutputStream(fileOutputStream));
    final File globalFolder = new File(this.fileNames.getGlobalFolderPath());
    final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(globalFolder.toPath());

    for (final Path path : directoryStream) {
      tarArchiveOutputStream.putArchiveEntry(new TarArchiveEntry(path.toFile(),
          globalFolder + "/" + path.getFileName()));

      final BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(path.toFile()));
      IOUtils.copy(bufferedInputStream, tarArchiveOutputStream);
      bufferedInputStream.close();

      tarArchiveOutputStream.closeArchiveEntry();
    }
    directoryStream.close();
    tarArchiveOutputStream.close();
    fileOutputStream.close();

    // Upload REEF_TAR to HDFS
    final FileSystem fileSystem = FileSystem.get(new Configuration());
    final org.apache.hadoop.fs.Path src = new org.apache.hadoop.fs.Path(REEF_TAR);
    final String reefTarUriValue = fileSystem.getUri().toString() + this.jobSubmissionDirectoryPrefix + "/" +
        jobIdentifier + "/" + REEF_TAR;
    final org.apache.hadoop.fs.Path dst = new org.apache.hadoop.fs.Path(reefTarUriValue);
    fileSystem.copyFromLocalFile(src, dst);

    return reefTarUriValue;
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
}
 

public static void addTarEntry(TarArchiveOutputStream tos, String path, byte[] content) throws IOException {
    TarArchiveEntry entry = new TarArchiveEntry(path);
    entry.setSize(content.length);
    tos.putArchiveEntry(entry);
    tos.write(content);
    tos.closeArchiveEntry();
}
 

static LocalResource createTgzFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File gzipFile = new File(p.toUri().getPath() + ".tar.gz");
  gzipFile.createNewFile();
  TarArchiveOutputStream out = new TarArchiveOutputStream(
      new GZIPOutputStream(new FileOutputStream(gzipFile)));
  TarArchiveEntry entry = new TarArchiveEntry(p.getName());
  entry.setSize(bytes.length);
  out.putArchiveEntry(entry);
  out.write(bytes);
  out.closeArchiveEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".tar.gz")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz"))
      .getModificationTime());
  return ret;
}
 

private void recursiveTar(String entryFilename, String rootPath, String itemPath, TarArchiveOutputStream tarArchive) {
    try {
        final File sourceFile = new File(itemPath).getCanonicalFile();     // e.g. /foo/bar/baz
        final File sourceRootFile = new File(rootPath).getCanonicalFile();     // e.g. /foo
        final String relativePathToSourceFile = sourceRootFile.toPath().relativize(sourceFile.toPath()).toFile().toString();    // e.g. /bar/baz

        final String tarEntryFilename;
        if (relativePathToSourceFile.isEmpty()) {
            tarEntryFilename = entryFilename; // entry filename e.g. xyz => xyz
        } else {
            tarEntryFilename = entryFilename + "/" + relativePathToSourceFile; // entry filename e.g. /xyz/bar/baz => /foo/bar/baz
        }

        final TarArchiveEntry tarEntry = new TarArchiveEntry(sourceFile, tarEntryFilename.replaceAll("^/", ""));

        // TarArchiveEntry automatically sets the mode for file/directory, but we can update to ensure that the mode is set exactly (inc executable bits)
        tarEntry.setMode(getUnixFileMode(itemPath));
        tarArchive.putArchiveEntry(tarEntry);

        if (sourceFile.isFile()) {
            Files.copy(sourceFile.toPath(), tarArchive);
        }
        // a directory entry merely needs to exist in the TAR file - there is no data stored yet
        tarArchive.closeArchiveEntry();

        final File[] children = sourceFile.listFiles();
        if (children != null) {
            // recurse into child files/directories
            for (final File child : children) {
                recursiveTar(entryFilename, sourceRootFile.getCanonicalPath(), child.getCanonicalPath(), tarArchive);
            }
        }
    } catch (IOException e) {
        log.error("Error when copying TAR file entry: {}", itemPath, e);
        throw new UncheckedIOException(e); // fail fast
    }
}