Java Code Examples for org.apache.commons.compress.archivers.tar.TarArchiveInputStream#close()

/**
 * given tar.gz file will be copied to this tar.gz file.
 * all files will be transferred to new tar.gz file one by one.
 * original directory structure will be kept intact
 *
 * @param tarGzipFile the archive file to be copied to the new archive
 */
public boolean addTarGzipToArchive(String tarGzipFile) {
  try {
    // construct input stream
    InputStream fin = Files.newInputStream(Paths.get(tarGzipFile));
    BufferedInputStream in = new BufferedInputStream(fin);
    GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in);
    TarArchiveInputStream tarInputStream = new TarArchiveInputStream(gzIn);

    // copy the existing entries from source gzip file
    ArchiveEntry nextEntry;
    while ((nextEntry = tarInputStream.getNextEntry()) != null) {
      tarOutputStream.putArchiveEntry(nextEntry);
      IOUtils.copy(tarInputStream, tarOutputStream);
      tarOutputStream.closeArchiveEntry();
    }

    tarInputStream.close();
    return true;
  } catch (IOException ioe) {
    LOG.log(Level.SEVERE, "Archive File can not be added: " + tarGzipFile, ioe);
    return false;
  }
}
 

public boolean isTared() {
	try {
		TarArchiveInputStream tarIn = new TarArchiveInputStream(new FileInputStream(sinfile));
		try {
			while ((tarIn.getNextTarEntry()) != null) {
				break;
			}
			tarIn.close();
			return true;
		} catch (IOException ioe) {
			try { tarIn.close(); } catch (Exception e) {}
			return false;
		}
	} catch (FileNotFoundException fne) {
		return false;
	}
}
 

/** {@inheritDoc} */
@Override
public Path downloadArtifact(Artifact a) throws Exception {
	Path p = null;
	
	if(!a.isReadyForDownload())
		throw new IllegalArgumentException("Artifact not fully specified: " + a);

	// Already downloaded?
	if(a.isCached()) {
		log.debug(a.toString() + " available in local m2 repo: [" + a.getAbsM2Path() + "]");
		p = a.getAbsM2Path();
		
		try{
			if(a.getProgrammingLanguage()==ProgrammingLanguage.JAVA){
				JarFile j = new JarFile(p.toFile(), false, java.util.zip.ZipFile.OPEN_READ);
				j.close();
			}
			else if(a.getProgrammingLanguage()==ProgrammingLanguage.PY && a.getPackaging().equals("sdist") ){
				TarArchiveInputStream t =new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(p.toFile()))));
				t.getNextTarEntry();
				t.close();
			}
		}catch(Exception e){
			e.printStackTrace();
			log.error("Exception when opening archive file [" + p.toFile() + "]: " + e.getMessage());
			boolean deleted = p.toFile().delete();
			if(!deleted)
				log.warn("Couldn't delete presumibly corrupted archive [" + p +"]");
			p=this.downloadArtifactFile(a);
		}
	}
	// No, download!
	else {
		p=this.downloadArtifactFile(a);
			
	}
	return p;
	
}
 

/** Untar an input file into an output file.

     * The output file is created in the output folder, having the same name
     * as the input file, minus the '.tar' extension.
     *
     * @param inputFile     the input .tar file
     * @param outputDir     the output directory file.
     * @throws IOException
     * @throws FileNotFoundException
     *
     * @return  The {@link List} of {@link File}s with the untared content.
     * @throws ArchiveException
     */
    public static List<File> unTar(final File inputFile, final File outputDir) throws FileNotFoundException, IOException, ArchiveException {

        logger.info(String.format("Untaring %s to dir %s.", inputFile.getAbsolutePath(), outputDir.getAbsolutePath()));

        final List<File> untaredFiles = new LinkedList<File>();
        final InputStream is = new FileInputStream(inputFile);
        final TarArchiveInputStream debInputStream = (TarArchiveInputStream) new ArchiveStreamFactory().createArchiveInputStream("tar", is);
        TarArchiveEntry entry = null;
        while ((entry = (TarArchiveEntry)debInputStream.getNextEntry()) != null) {
            final File outputFile = new File(outputDir, entry.getName());
            if (entry.isDirectory()) {
                logger.info(String.format("Attempting to write output directory %s.", outputFile.getAbsolutePath()));
                if (!outputFile.exists()) {
                    logger.info(String.format("Attempting to create output directory %s.", outputFile.getAbsolutePath()));
                    if (!outputFile.mkdirs()) {
                        throw new IllegalStateException(String.format("Couldn't create directory %s.", outputFile.getAbsolutePath()));
                    }
                }
            } else {
                logger.info(String.format("Creating output file %s.", outputFile.getAbsolutePath()));
                final OutputStream outputFileStream = new FileOutputStream(outputFile);
                IOUtils.copy(debInputStream, outputFileStream);
                outputFileStream.close();
            }
            untaredFiles.add(outputFile);
        }
        debInputStream.close();

        return untaredFiles;
    }
 

private static List<File> unTar(final File inputFile, final File outputDir) throws FileNotFoundException, IOException, ArchiveException {
    Log.i(TAG, String.format("Untaring %s to dir %s", inputFile.getAbsolutePath(), outputDir.getAbsolutePath()));

    if (!outputDir.exists()) {
        outputDir.mkdirs();
    }
    final List<File> untaredFiles = new LinkedList<File>();
    final InputStream is = new FileInputStream(inputFile);
    final TarArchiveInputStream debInputStream = (TarArchiveInputStream) new ArchiveStreamFactory().createArchiveInputStream("tar", is);
    TarArchiveEntry entry = null;
    while ((entry = (TarArchiveEntry)debInputStream.getNextEntry()) != null) {
        final File outputFile = new File(outputDir, entry.getName());
        if (entry.isDirectory()) {
            if (!outputFile.exists()) {
                if (!outputFile.mkdirs()) {
                    throw new IllegalStateException(String.format("Couldn't create directory %s.", outputFile.getAbsolutePath()));
                }
            }
        } else {
            final OutputStream outputFileStream = new FileOutputStream(outputFile);
            IOUtils.copy(debInputStream, outputFileStream);
            outputFileStream.close();
        }
        untaredFiles.add(outputFile);
    }
    debInputStream.close();

    return untaredFiles;
}
 

public static void extractTarGz(InputStream inputTarGzStream, String outDir,
    boolean logging) {
  try {
    GzipCompressorInputStream gzIn = new GzipCompressorInputStream(
        inputTarGzStream);
    TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn);

    // read Tar entries
    TarArchiveEntry entry = null;
    while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
      if (logging) {
        LOG.info("Extracting: " + outDir + File.separator + entry.getName());
      }
      if (entry.isDirectory()) { // create directory
        File f = new File(outDir + File.separator + entry.getName());
        f.mkdirs();
      } else { // decompress file
        int count;
        byte data[] = new byte[EXTRACT_BUFFER_SIZE];

        FileOutputStream fos = new FileOutputStream(outDir + File.separator
            + entry.getName());
        BufferedOutputStream dest = new BufferedOutputStream(fos,
            EXTRACT_BUFFER_SIZE);
        while ((count = tarIn.read(data, 0, EXTRACT_BUFFER_SIZE)) != -1) {
          dest.write(data, 0, count);
        }
        dest.close();
      }
    }

    // close input stream
    tarIn.close();

  } catch (IOException e) {
    LOG.error("IOException: " + e.getMessage());
  }
}
 

private static String readGzipStreamAsString(InputStream is) throws Exception {
  TarArchiveInputStream tarIn = new TarArchiveInputStream(is);
  try {
    TarArchiveEntry tarEntry;
    while ((tarEntry = tarIn.getNextTarEntry()) != null) {
      if (tarEntry.isFile() && tarEntry.getName().endsWith(".txt")) {
        return IOUtils.toString(tarIn, "UTF-8");
      }
    }
  } finally {
    tarIn.close();
  }

  return Strings.EMPTY;
}
 

@Scheduled(fixedDelay = 86400 * 1000) //Every 24H
@PostConstruct
private void downloadIPLocationDatabase() throws Exception {
    if(config.getGeoIP().isEnabled()){
        logger.info("Downloading GEOIP Database from: "+config.getGeoIP().getUrl());

        File tmpGEOIPFileRound = File.createTempFile("geoIP", "mmdb");
        FileOutputStream fos = new FileOutputStream(tmpGEOIPFileRound);
        HttpURLConnection connection = getURLConnection(config.getGeoIP().getUrl());
        if (connection.getResponseCode() != 200) {
            return;
        }
        TarArchiveInputStream tarGzGeoIPStream = new TarArchiveInputStream(new GZIPInputStream(connection.getInputStream()));
        TarArchiveEntry entry= null;
        int offset;
        long pointer=0;
        while ((entry = tarGzGeoIPStream.getNextTarEntry()) != null) {
            pointer+=entry.getSize();
            if(entry.getName().endsWith("GeoLite2-City.mmdb")){
                byte[] content = new byte[(int) entry.getSize()];
                offset=0;
                //FileInputStream fis = new FileInputStream(entry.getFile());
                //IOUtils.copy(fis,fos);
                //tarGzGeoIPStream.skip(pointer);
                //tarGzGeoIPStream.read(content,offset,content.length-offset);
                //IOUtils.write(content,fos);
                int r;
                byte[] b = new byte[1024];
                while ((r = tarGzGeoIPStream.read(b)) != -1) {
                    fos.write(b, 0, r);
                }
                //fis.close();
                break;
            }
        }
        tarGzGeoIPStream.close();
        fos.flush();
        fos.close();
        connection.disconnect();
        geoIPDB = new DatabaseReader.Builder(tmpGEOIPFileRound).withCache(new CHMCache()).build();
        if (tmpGEOIPFile != null && tmpGEOIPFile.exists()) {
            tmpGEOIPFile.delete();
        }
        tmpGEOIPFile = tmpGEOIPFileRound;
    }
}
 

/**
 * Untars the passed in {@link FileAwareInputStream} to the task's staging directory. Uses the name of the root
 * {@link TarArchiveEntry} in the stream as the directory name for the untarred file. The method also commits the data
 * by moving the file from staging to output directory.
 *
 * @see org.apache.gobblin.data.management.copy.writer.FileAwareInputStreamDataWriter#write(org.apache.gobblin.data.management.copy.FileAwareInputStream)
 */
@Override
public void writeImpl(InputStream inputStream, Path writeAt, CopyableFile copyableFile, FileAwareInputStream record)
    throws IOException {
  this.closer.register(inputStream);

  TarArchiveInputStream tarIn = new TarArchiveInputStream(inputStream);
  final ReadableByteChannel inputChannel = Channels.newChannel(tarIn);
  TarArchiveEntry tarEntry;

  // flush the first entry in the tar, which is just the root directory
  tarEntry = tarIn.getNextTarEntry();
  String tarEntryRootName = StringUtils.remove(tarEntry.getName(), Path.SEPARATOR);

  log.info("Unarchiving at " + writeAt);

  try {
    while ((tarEntry = tarIn.getNextTarEntry()) != null) {

      // the API tarEntry.getName() is misleading, it is actually the path of the tarEntry in the tar file
      String newTarEntryPath = tarEntry.getName().replace(tarEntryRootName, writeAt.getName());
      Path tarEntryStagingPath = new Path(writeAt.getParent(), newTarEntryPath);
      if (!FileUtils.isSubPath(writeAt.getParent(), tarEntryStagingPath)) {
        throw new IOException(String.format("Extracted file: %s is trying to write outside of output directory: %s",
            tarEntryStagingPath, writeAt.getParent()));
      }

      if (tarEntry.isDirectory() && !this.fs.exists(tarEntryStagingPath)) {
        this.fs.mkdirs(tarEntryStagingPath);
      } else if (!tarEntry.isDirectory()) {
        FSDataOutputStream out = this.fs.create(tarEntryStagingPath, true);
        final WritableByteChannel outputChannel = Channels.newChannel(out);
        try {
          StreamCopier copier = new StreamCopier(inputChannel, outputChannel);
          if (isInstrumentationEnabled()) {
            copier.withCopySpeedMeter(this.copySpeedMeter);
          }
          this.bytesWritten.addAndGet(copier.copy());
          if (isInstrumentationEnabled()) {
            log.info("File {}: copied {} bytes, average rate: {} B/s", copyableFile.getOrigin().getPath(), this.copySpeedMeter.getCount(), this.copySpeedMeter.getMeanRate());
          } else {
            log.info("File {} copied.", copyableFile.getOrigin().getPath());
          }
        } finally {
          out.close();
          outputChannel.close();
        }
      }
    }
  } finally {
    tarIn.close();
    inputChannel.close();
    inputStream.close();
  }
}
 

public void flashImage(SinFile sin,String partitionname) throws X10FlashException, IOException {
	//wrotedata=true;
	String command="";
	logger.info("processing "+sin.getName());
	command = "signature:"+HexDump.toHex(sin.getHeader().length);
	logger.info("   signature:"+HexDump.toHex(sin.getHeader().length));
	CommandPacket p=null;
	if (!_bundle.simulate()) {
		USBFlash.write(command.getBytes());
		p = USBFlash.readCommandReply(false);
		//logger.info("   signature reply : "+p.getResponse());
		USBFlash.write(sin.getHeader());
		p = USBFlash.readCommandReply(true);
		command="signature";
		USBFlash.write(command.getBytes());
		p = USBFlash.readCommandReply(true);
		logger.info("   signature status : "+p.getResponse());
	}
	command="erase:"+partitionname;
	logger.info("   "+command);
	if (!_bundle.simulate()) {
		USBFlash.write(command.getBytes());
		p = USBFlash.readCommandReply(true);
		logger.info("   erase status : "+p.getResponse());
	}
	TarArchiveInputStream tarIn = sin.getTarInputStream();
	TarArchiveEntry entry=null;
	while ((entry = tarIn.getNextTarEntry()) != null) {
		if (!entry.getName().endsWith("cms")) {
			logger.info("   sending "+entry.getName());
			if (!_bundle.simulate()) {
				command = "download:"+HexDump.toHex((int)entry.getSize());
				logger.info("      "+command);
				USBFlash.write(command.getBytes());
				p = USBFlash.readCommandReply(false);
				//logger.info("      Download reply : "+p.getResponse());
			}
			CircularByteBuffer cb = new CircularByteBuffer(CircularByteBuffer.INFINITE_SIZE);
			IOUtils.copy(tarIn, cb.getOutputStream());
			LogProgress.initProgress(cb.getAvailable()/USBFlash.getUSBBufferSize()+1);
			while (cb.getAvailable()>0) {
				byte[] buffer = new byte[cb.getAvailable()>=USBFlash.getUSBBufferSize()?USBFlash.getUSBBufferSize():cb.getAvailable()];
				cb.getInputStream().read(buffer);
				LogProgress.updateProgress();
				if (!_bundle.simulate()) {
					USBFlash.write(buffer);
				}
				else {
					try {
						Thread.sleep(10);} catch (Exception e) {}
				}
			}
			if (!_bundle.simulate()) {
				p = USBFlash.readCommandReply(true);
				logger.info("      download status : "+p.getResponse());
			}
			command="flash:"+partitionname;
			logger.info("      "+command);
			if (!_bundle.simulate()) {
				USBFlash.write(command.getBytes());
				p = USBFlash.readCommandReply(true);
				logger.info("      flash status : "+p.getResponse());
			}
			LogProgress.initProgress(0);
		}
	}
	
	tarIn.close();
}
 

/**
 * Extracts files to the specified destination
 *
 * @param file the file to extract to
 * @param dest the destination directory
 * @throws IOException
 */
public static void unzipFileTo(String file, String dest) throws IOException {
    File target = new File(file);
    if (!target.exists())
        throw new IllegalArgumentException("Archive doesnt exist");
    FileInputStream fin = new FileInputStream(target);
    int BUFFER = 2048;
    byte data[] = new byte[BUFFER];

    if (file.endsWith(".zip") || file.endsWith(".jar")) {
        try(ZipInputStream zis = new ZipInputStream(fin)) {
            //get the zipped file list entry
            ZipEntry ze = zis.getNextEntry();

            while (ze != null) {
                String fileName = ze.getName();
                File newFile = new File(dest + File.separator + fileName);

                if (ze.isDirectory()) {
                    newFile.mkdirs();
                    zis.closeEntry();
                    ze = zis.getNextEntry();
                    continue;
                }

                FileOutputStream fos = new FileOutputStream(newFile);

                int len;
                while ((len = zis.read(data)) > 0) {
                    fos.write(data, 0, len);
                }

                fos.close();
                ze = zis.getNextEntry();
                log.debug("File extracted: " + newFile.getAbsoluteFile());
            }

            zis.closeEntry();
        }
    } else if (file.endsWith(".tar.gz") || file.endsWith(".tgz")) {

        BufferedInputStream in = new BufferedInputStream(fin);
        GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in);
        TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn);

        TarArchiveEntry entry;
        /* Read the tar entries using the getNextEntry method **/
        while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
            log.info("Extracting: " + entry.getName());
            /* If the entry is a directory, create the directory. */

            if (entry.isDirectory()) {
                File f = new File(dest + File.separator + entry.getName());
                f.mkdirs();
            }
            /*
             * If the entry is a file,write the decompressed file to the disk
             * and close destination stream.
             */
            else {
                int count;
                try(FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName());
                    BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER);) {
                    while ((count = tarIn.read(data, 0, BUFFER)) != -1) {
                        destStream.write(data, 0, count);
                    }

                    destStream.flush();
                    IOUtils.closeQuietly(destStream);
                }
            }
        }

        // Close the input stream
        tarIn.close();
    } else if (file.endsWith(".gz")) {
        File extracted = new File(target.getParent(), target.getName().replace(".gz", ""));
        if (extracted.exists())
            extracted.delete();
        extracted.createNewFile();
        try(GZIPInputStream is2 = new GZIPInputStream(fin); OutputStream fos = FileUtils.openOutputStream(extracted)) {
            IOUtils.copyLarge(is2, fos);
            fos.flush();
        }
    } else {
        throw new IllegalStateException("Unable to infer file type (compression format) from source file name: " +
                file);
    }
    target.delete();
}
 

public static void untargz(final InputStream read, final File destination, final boolean noparent, final FileFilter fileFilter) throws IOException {
        Objects.requireNonNull(fileFilter, "'fileFilter' is required.");

        Files.dir(destination);
        Files.writable(destination);

        try {
            GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(read);
            TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn);

            TarArchiveEntry entry;
            while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
                String path = entry.getName();
                if (noparent) {
                    path = path.replaceFirst("^[^/]+/", "");
                }

                File file = new File(destination, path);
                if (!fileFilter.accept(file)) continue;

                if (entry.isDirectory()) {
                    Files.mkdir(file);
                } else {
                    Files.mkdir(file.getParentFile());
                    IO.copy(tarIn, file);
                    long lastModified = entry.getLastModifiedDate().getTime();
                    if (lastModified > 0L) {
                        file.setLastModified(lastModified);
                    }

//                    if (entry.getMode() != 420) System.out.printf("%s  %s%n", entry.getMode(), entry.getName());
                    // DMB: I have no idea how to respect the mod.
                    // Elasticsearch tar has entries with 33261 that are executable
                    if (33261 == entry.getMode()) {
                        file.setExecutable(true);
                    }
                    // DMB: I have no idea how to respect the mod.
                    // Kibana tar has entries with 493 that are executable
                    if (493 == entry.getMode()) {
                        file.setExecutable(true);
                    }
                }
            }

            tarIn.close();
        } catch (IOException var9) {
            throw new IOException("Unable to unzip " + read, var9);
        }
    }
 

/**
 * 解归档
 *
 * @param srcFile
 * @param destFile
 * @throws Exception
 */
public static void dearchive(File srcFile, File destFile) throws Exception {

    TarArchiveInputStream tais = new TarArchiveInputStream(
            new FileInputStream(srcFile));
    dearchive(destFile, tais);

    tais.close();

}