org.apache.commons.compress.utils.IOUtils Java Examples

public static byte[] decompress(byte[] bytes, int len) throws IOException
{
	byte[] data = new byte[len + BZIP_HEADER.length];

	// add header
	System.arraycopy(BZIP_HEADER, 0, data, 0, BZIP_HEADER.length);
	System.arraycopy(bytes, 0, data, BZIP_HEADER.length, len);

	ByteArrayOutputStream os = new ByteArrayOutputStream();

	try (InputStream is = new BZip2CompressorInputStream(new ByteArrayInputStream(data)))
	{
		IOUtils.copy(is, os);
	}

	return os.toByteArray();
}
 

public static void unzip(final File zip, final File dest, final Charset charset) throws IOException {
    final Path destPath = dest.toPath();
    try (final ZipFile zipFile = new ZipFile(zip, ZipFile.OPEN_READ, charset)) {
        final Enumeration<? extends ZipEntry> entries = zipFile.entries();
        while (entries.hasMoreElements()) {
            final ZipEntry entry = entries.nextElement();
            final Path entryPath = destPath.resolve(entry.getName());
            if (!entryPath.normalize().startsWith(dest.toPath()))
                throw new IOException("Zip entry contained path traversal");
            if (entry.isDirectory()) {
                Files.createDirectories(entryPath);
            } else {
                Files.createDirectories(entryPath.getParent());
                try (final InputStream in = zipFile.getInputStream(entry)) {
                    try (final OutputStream out = new FileOutputStream(entryPath.toFile())) {
                        IOUtils.copy(in, out);
                    }
                }
            }
        }
    }
}
 

@Test
public void shouldStoreContentCompressed() throws IOException {
    // given
 boolean compressData = true;
 String folder = tempFolder.newFolder().toString();
    Page target = new Page(new URL(url), html);
    FileSystemTargetRepository repository = new FileSystemTargetRepository(Paths.get(folder), DataFormat.HTML, false, compressData);
    
    // when
    repository.insert(target);
    
    // then
    Path path = Paths.get(folder, "example.com", "http%3A%2F%2Fexample.com");
    assertThat(path.toFile().exists(), is(true));
    
    byte[] fileBytes = Files.readAllBytes(path);
    assertThat(fileBytes, is(notNullValue()));
    assertThat(fileBytes.length < html.getBytes().length, is(true));
    
    InputStream gzip = new InflaterInputStream(new ByteArrayInputStream(fileBytes));
    byte[] uncompressedBytes = IOUtils.toByteArray(gzip);
    String content = new String(uncompressedBytes);
    assertThat(content, is(html));
}
 

@Test
// verifies backward compatibility
public void testConcatenatedStreamsReadFirstOnly() throws IOException {
    EmbeddedChannel chDecoderGZip = new EmbeddedChannel(createDecoder(ZlibWrapper.GZIP));

    try {
        byte[] bytes = IOUtils.toByteArray(getClass().getResourceAsStream("/multiple.gz"));

        assertTrue(chDecoderGZip.writeInbound(Unpooled.copiedBuffer(bytes)));
        Queue<Object> messages = chDecoderGZip.inboundMessages();
        assertEquals(1, messages.size());

        ByteBuf msg = (ByteBuf) messages.poll();
        assertEquals("a", msg.toString(CharsetUtil.UTF_8));
        ReferenceCountUtil.release(msg);
    } finally {
        assertFalse(chDecoderGZip.finish());
        chDecoderGZip.close();
    }
}
 

@Test
public void testConcatenatedStreamsReadFully() throws IOException {
    EmbeddedChannel chDecoderGZip = new EmbeddedChannel(new JdkZlibDecoder(true));

    try {
        byte[] bytes = IOUtils.toByteArray(getClass().getResourceAsStream("/multiple.gz"));

        assertTrue(chDecoderGZip.writeInbound(Unpooled.copiedBuffer(bytes)));
        Queue<Object> messages = chDecoderGZip.inboundMessages();
        assertEquals(2, messages.size());

        for (String s : Arrays.asList("a", "b")) {
            ByteBuf msg = (ByteBuf) messages.poll();
            assertEquals(s, msg.toString(CharsetUtil.UTF_8));
            ReferenceCountUtil.release(msg);
        }
    } finally {
        assertFalse(chDecoderGZip.finish());
        chDecoderGZip.close();
    }
}
 

public String doRestPUT(String method, Object parameter) throws IOException, RuntimeException {
	HttpPut request = new HttpPut("http://localhost:16043/rest/v1/" + method);

	if (parameter != null) {
		request.setEntity(new StringEntity(marshal(parameter), ContentType.APPLICATION_JSON));
	}

	HttpResponse response = client.execute(targetHost, request, context);

	ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
	IOUtils.copy(response.getEntity().getContent(), byteArrayOutputStream);

	String responseString = new String(byteArrayOutputStream.toByteArray(), "UTF8");

	if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 203) {
		throw new RuntimeException(responseString);
	}

	return responseString;
}
 

private void maybeAddPostFilterCmdStep(
    BuildContext context,
    BuildableContext buildableContext,
    ImmutableList.Builder<Step> steps,
    ImmutableBiMap<Path, Path> inResDirToOutResDirMap) {
  postFilterResourcesCmd.ifPresent(
      cmd -> {
        OutputStream filterResourcesDataOutputStream = null;
        try {
          Path filterResourcesDataPath = getFilterResourcesDataPath();
          getProjectFilesystem().createParentDirs(filterResourcesDataPath);
          filterResourcesDataOutputStream =
              getProjectFilesystem().newFileOutputStream(filterResourcesDataPath);
          writeFilterResourcesData(filterResourcesDataOutputStream, inResDirToOutResDirMap);
          buildableContext.recordArtifact(filterResourcesDataPath);
          addPostFilterCommandSteps(cmd, context.getSourcePathResolver(), steps);
        } catch (IOException e) {
          throw new RuntimeException("Could not generate/save filter resources data json", e);
        } finally {
          IOUtils.closeQuietly(filterResourcesDataOutputStream);
        }
      });
}
 

/**
 * given tar.gz file will be copied to this tar.gz file.
 * all files will be transferred to new tar.gz file one by one.
 * original directory structure will be kept intact
 *
 * @param zipFile the archive file to be copied to the new archive
 * @param dirPrefixForTar sub path inside the archive
 */
public boolean addZipToArchive(String zipFile, String dirPrefixForTar) {
  try {
    // construct input stream
    ZipFile zipFileObj = new ZipFile(zipFile);
    Enumeration<? extends ZipEntry> entries = zipFileObj.entries();

    // copy the existing entries from source gzip file
    while (entries.hasMoreElements()) {
      ZipEntry nextEntry = entries.nextElement();
      TarArchiveEntry entry = new TarArchiveEntry(dirPrefixForTar + nextEntry.getName());
      entry.setSize(nextEntry.getSize());
      entry.setModTime(nextEntry.getTime());

      tarOutputStream.putArchiveEntry(entry);
      IOUtils.copy(zipFileObj.getInputStream(nextEntry), tarOutputStream);
      tarOutputStream.closeArchiveEntry();
    }

    zipFileObj.close();
    return true;
  } catch (IOException ioe) {
    LOG.log(Level.SEVERE, "Archive File can not be added: " + zipFile, ioe);
    return false;
  }
}
 

private static void extractZipFile(final File destination, final ZipFile zipFile) throws IOException {
    final Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();

    while (entries.hasMoreElements()) {
        final ZipArchiveEntry entry = entries.nextElement();
        final File entryDestination = getDestinationFile(destination, entry.getName());

        if (entry.isDirectory()) {
            entryDestination.mkdirs();
        } else {
            entryDestination.getParentFile().mkdirs();
            final InputStream in = zipFile.getInputStream(entry);
            try (final OutputStream out = new FileOutputStream(entryDestination)) {
                IOUtils.copy(in, out);
                IOUtils.closeQuietly(in);
            }
        }
    }
}
 

/**
 * add one file to tar.gz file
 * file is created from the given byte array
 *
 * @param filename file to be added to the tar.gz
 */
public boolean addFileToArchive(String filename, byte[] contents) {

  String filePathInTar = JOB_ARCHIVE_DIRECTORY + "/" + filename;
  try {
    TarArchiveEntry entry = new TarArchiveEntry(filePathInTar);
    entry.setSize(contents.length);
    tarOutputStream.putArchiveEntry(entry);
    IOUtils.copy(new ByteArrayInputStream(contents), tarOutputStream);
    tarOutputStream.closeArchiveEntry();

    return true;
  } catch (IOException e) {
    LOG.log(Level.SEVERE, "File can not be added: " + filePathInTar, e);
    return false;
  }
}
 

/**
 * 入力したバイト配列をBZip2で圧縮して返します。
 * 
 * @param input
 * @return
 */
public static byte[] compress(byte[] input) {
    ByteArrayOutputStream ref = null;

    try (val bais = new ByteArrayInputStream(input);
            val baos = new ByteArrayOutputStream(input.length);
            val bzip2cos = new BZip2CompressorOutputStream(baos)) {
        IOUtils.copy(bais, bzip2cos);
        ref = baos;
    } catch (IOException e) {
        log.error("failed to encode.", e);
        throw new RuntimeException(e);
    }

    return ref.toByteArray();
}
 

/**
 * 入力したバイト配列をBZip2で展開して返します。
 * 
 * @param input
 * @return
 */
public static byte[] decompress(byte[] input) {
    ByteArrayOutputStream ref = null;

    try (val bais = new ByteArrayInputStream(input);
            val bzip2cis = new BZip2CompressorInputStream(bais);
            val baos = new ByteArrayOutputStream()) {
        IOUtils.copy(bzip2cis, baos);
        ref = baos;
    } catch (IOException e) {
        log.error("failed to decode.", e);
        throw new RuntimeException(e);
    }

    return ref.toByteArray();
}
 

@Override
public Set<String> load(final InputStream imagePayload, final ProgressHandler handler)
    throws DockerException, InterruptedException {
  final WebTarget resource = resource()
          .path("images")
          .path("load")
          .queryParam("quiet", "false");

  final LoadProgressHandler loadProgressHandler = new LoadProgressHandler(handler);
  final Entity<InputStream> entity = Entity.entity(imagePayload, APPLICATION_OCTET_STREAM);

  try (final ProgressStream load =
          request(POST, ProgressStream.class, resource,
                  resource.request(APPLICATION_JSON_TYPE), entity)) {
    load.tail(loadProgressHandler, POST, resource.getUri());
    return loadProgressHandler.getImageNames();
  } catch (IOException e) {
    throw new DockerException(e);
  } finally {
    IOUtils.closeQuietly(imagePayload);
  }
}
 

private static void extract(final Path path, final String destDirectory) throws IOException {
  try (TarArchiveInputStream fin =
      new TarArchiveInputStream(
          new GzipCompressorInputStream(new FileInputStream(path.toAbsolutePath().toString())))) {
    TarArchiveEntry entry;
    while ((entry = fin.getNextTarEntry()) != null) {
      if (entry.isDirectory()) {
        continue;
      }
      final File curfile = new File(destDirectory, entry.getName());
      final File parent = curfile.getParentFile();
      if (!parent.exists()) {
        parent.mkdirs();
      }
      IOUtils.copy(fin, new FileOutputStream(curfile));
    }
  }
}
 

@Test
@Ignore
// TODO need to regen key now that we're using CryptoSuite
public void testSign() {

    byte[] plainText = "123456".getBytes(UTF_8);
    byte[] signature;
    try {
        PrivateKey key = (PrivateKey) crypto.getTrustStore().getKey("key", "123456".toCharArray());
        signature = crypto.sign(key, plainText);

        BufferedInputStream bis = new BufferedInputStream(
                this.getClass().getResourceAsStream("/keypair-signed.crt"));
        byte[] cert = IOUtils.toByteArray(bis);
        bis.close();

        assertTrue(crypto.verify(cert, SIGNING_ALGORITHM, signature, plainText));
    } catch (KeyStoreException | CryptoException | IOException | UnrecoverableKeyException
            | NoSuchAlgorithmException e) {
        fail("Could not verify signature. Error: " + e.getMessage());
    }
}
 

@Test
public void testSetTrustStoreDuplicateCertUsingFile() {
    try {
        // Read the certificate data
        java.net.URL certUrl = this.getClass().getResource("/ca.crt");
        String certData = org.apache.commons.io.IOUtils.toString(certUrl, "UTF-8");

        // Write this to a temp file
        File tempFile = tempFolder.newFile("temp.txt");
        Path tempPath = Paths.get(tempFile.getAbsolutePath());
        Files.write(tempPath, certData.getBytes());

        crypto.addCACertificateToTrustStore(tempFile, "ca"); //KeyStore overrides existing cert if same alias
    } catch (Exception e) {
        fail("testSetTrustStoreDuplicateCert should not have thrown Exception. Error: " + e.getMessage());
    }
}
 

public static void makeOnlyUnZip() throws ArchiveException, IOException{
		final InputStream is = new FileInputStream("D:/中文名字.zip");
		ArchiveInputStream in = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.ZIP, is);
		ZipArchiveEntry entry = entry = (ZipArchiveEntry) in.getNextEntry();
		
		String dir = "D:/cnname";
		File filedir = new File(dir);
		if(!filedir.exists()){
			filedir.mkdir();
		}
//		OutputStream out = new FileOutputStream(new File(dir, entry.getName()));
		OutputStream out = new FileOutputStream(new File(filedir, entry.getName()));
		IOUtils.copy(in, out);
		out.close();
		in.close();
	}
 

/**
 * 把一个ZIP文件解压到一个指定的目录中
 * @param zipfilename ZIP文件抽象地址
 * @param outputdir 目录绝对地址
 */
public static void unZipToFolder(String zipfilename, String outputdir) throws IOException {
    File zipfile = new File(zipfilename);
    if (zipfile.exists()) {
        outputdir = outputdir + File.separator;
        FileUtils.forceMkdir(new File(outputdir));

        ZipFile zf = new ZipFile(zipfile, "UTF-8");
        Enumeration zipArchiveEntrys = zf.getEntries();
        while (zipArchiveEntrys.hasMoreElements()) {
            ZipArchiveEntry zipArchiveEntry = (ZipArchiveEntry) zipArchiveEntrys.nextElement();
            if (zipArchiveEntry.isDirectory()) {
                FileUtils.forceMkdir(new File(outputdir + zipArchiveEntry.getName() + File.separator));
            } else {
                IOUtils.copy(zf.getInputStream(zipArchiveEntry), FileUtils.openOutputStream(new File(outputdir + zipArchiveEntry.getName())));
            }
        }
    } else {
        throw new IOException("指定的解压文件不存在：\t" + zipfilename);
    }
}
 

public static byte[] gZip(byte[] data) throws IOException
{
	byte[] ret = null;
	ByteArrayOutputStream bos = null;
	GZIPOutputStream gzip = null;
	try
	{
		bos = new ByteArrayOutputStream();
		gzip = new GZIPOutputStream(bos);
		gzip.write(data);
		gzip.finish();
		ret = bos.toByteArray();
	} finally
	{
		IOUtils.closeQuietly(gzip);
		IOUtils.closeQuietly(bos);
	}
	return ret;
}
 

default String login(String address) throws IOException, KeyManagementException, NoSuchAlgorithmException {
    HttpURLConnection loginConnection = null;
    String cookie = "";
    try {
        loginConnection = connector.getConnection("GET", address + "/auth/login?user=test&password=testPass");
        try (InputStream in = loginConnection.getInputStream()) {
            String responseBody = new String(IOUtils.toByteArray(in));
            assertTrue(responseBody.contains("\"success\":true"), () -> "Not successful: " + responseBody);
            cookie = loginConnection.getHeaderField("Set-Cookie").split(";")[0];
            System.out.println("Got cookie: " + cookie);
        }
    } finally {
        loginConnection.disconnect();
    }
    return cookie;
}
 

private void findAndCopyFromTar(TarArchiveInputStream tarIn, FileOutputStream fos) throws IOException {
    // Breadth first search
    Queue<TarArchiveEntry> entries = new ArrayDeque<>();
    entries.add(tarIn.getNextTarEntry());
    while (!entries.isEmpty()) {
        TarArchiveEntry entry = entries.poll();
        if (entry.isDirectory()) {
            entries.addAll(Arrays.asList(entry.getDirectoryEntries()));
        }

        // Looking for this file
        if (entry.getName().endsWith("GeoLite2-Country.mmdb")) {
            IOUtils.copy(tarIn, fos);
            break; // Found it
        }

        TarArchiveEntry next = tarIn.getNextTarEntry();
        if (next != null) entries.add(next);
    }
}
 

public static void untar(
        InputStream tar,
        File parentDir
) throws IOException {
    TarArchiveInputStream tin = new TarArchiveInputStream(tar);
    ArchiveEntry e;
    while ((e = tin.getNextEntry()) != null) {
        File f = new File(parentDir, e.getName());
        f.setLastModified(e.getLastModifiedDate().getTime());
        f.getParentFile().mkdirs();
        if (e.isDirectory()) {
            f.mkdir();
            continue;
        }
        long size = e.getSize();
        checkFileSize(size);
        try (OutputStream out = new FileOutputStream(f)) {
            /* TarInputStream pretends each
               entry's EOF is the stream's EOF */
            IOUtils.copy(tin, out);
        }
    }
}
 

@SuppressWarnings("unchecked")
private <T> T unserializeData(InputStream inputStream) {
    T object = null;
    try {
        if (dataFormat.equals(DataFormat.CBOR)) {
                object = (T) cborMapper.readValue(inputStream, TargetModelCbor.class);
        } else if (dataFormat.equals(DataFormat.JSON)) {
            object = (T) jsonMapper.readValue(inputStream, TargetModelJson.class);
        } else if (dataFormat.equals(DataFormat.HTML)) {
            byte[] fileData = IOUtils.toByteArray(inputStream);
            object = (T) new String(fileData);
        }
    } catch (IOException e) {
        throw new RuntimeException("Failed to unserialize object.", e);
    }
    return object;
}
 

public void publish(BrokerStats brokerStats) throws IOException {
  try {
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = avroEncoderFactory.binaryEncoder(stream, null);

    avroEventWriter.write(brokerStats, binaryEncoder);
    binaryEncoder.flush();
    IOUtils.closeQuietly(stream);

    String key = brokerStats.getName() + "_" + System.currentTimeMillis();
    int numPartitions = kafkaProducer.partitionsFor(destTopic).size();
    int partition = brokerStats.getId() % numPartitions;

    Future<RecordMetadata> future = kafkaProducer.send(
        new ProducerRecord<>(destTopic, partition, key.getBytes(), stream.toByteArray()));
    future.get();

    OpenTsdbMetricConverter.incr("kafka.stats.collector.success", 1, "host=" + HOSTNAME);
  } catch (Exception e) {
    LOG.error("Failure in publish stats", e);
    OpenTsdbMetricConverter.incr("kafka.stats.collector.failure", 1, "host=" + HOSTNAME);
    throw new RuntimeException("Avro serialization failure", e);
  }
}
 

@Test(priority = 10, expectedExceptions = InsightsCustomException.class)
public void testFileSizeException() throws InsightsCustomException, IOException {

	FileInputStream input = new FileInputStream(fileSize);
	MultipartFile multipartFile = new MockMultipartFile("file", fileSize.getName(), "text/plain",
			IOUtils.toByteArray(input));

	boolean response = bulkUploadService.uploadDataInDatabase(multipartFile, bulkUploadTestData.toolName,
			bulkUploadTestData.label, bulkUploadTestData.insightTimeField,
			bulkUploadTestData.nullInsightTimeFormat);

}
 

private static List<File> unTar(final File inputFile, final File outputDir) throws FileNotFoundException, IOException, ArchiveException {
    Log.i(TAG, String.format("Untaring %s to dir %s", inputFile.getAbsolutePath(), outputDir.getAbsolutePath()));

    if (!outputDir.exists()) {
        outputDir.mkdirs();
    }
    final List<File> untaredFiles = new LinkedList<File>();
    final InputStream is = new FileInputStream(inputFile);
    final TarArchiveInputStream debInputStream = (TarArchiveInputStream) new ArchiveStreamFactory().createArchiveInputStream("tar", is);
    TarArchiveEntry entry = null;
    while ((entry = (TarArchiveEntry)debInputStream.getNextEntry()) != null) {
        final File outputFile = new File(outputDir, entry.getName());
        if (entry.isDirectory()) {
            if (!outputFile.exists()) {
                if (!outputFile.mkdirs()) {
                    throw new IllegalStateException(String.format("Couldn't create directory %s.", outputFile.getAbsolutePath()));
                }
            }
        } else {
            final OutputStream outputFileStream = new FileOutputStream(outputFile);
            IOUtils.copy(debInputStream, outputFileStream);
            outputFileStream.close();
        }
        untaredFiles.add(outputFile);
    }
    debInputStream.close();

    return untaredFiles;
}
 

public Payload read(final Repository repository, final String path) throws IOException {
  HttpGet get = new HttpGet(url(repository, path));
  try (CloseableHttpResponse response = client.execute(get)) {
    if (response.getStatusLine().getStatusCode() >= 300) {
      return null;
    }

    HttpEntity entity = response.getEntity();
    try (InputStream in = entity.getContent()) {
      byte[] content = IOUtils.toByteArray(in);
      String contentType = entity.getContentType().toString();

      return new Payload()
      {
        @Override
        public InputStream openInputStream() throws IOException {
          return new ByteArrayInputStream(content);
        }

        @Override
        public long getSize() {
          return content.length;
        }

        @Override
        public String getContentType() {
          return contentType;
        }
      };
    }
  }
}
 

/**
 * Zip file.
 *
 * @param filePath the file path
 * @return the string
 */
public String zipFile(String filePath) {
	try {
		/* Create Output Stream that will have final zip files */
		OutputStream zipOutput = new FileOutputStream(new File(filePath
				+ ".zip"));
		/*
		 * Create Archive Output Stream that attaches File Output Stream / and
		 * specifies type of compression
		 */
		ArchiveOutputStream logicalZip = new ArchiveStreamFactory()
				.createArchiveOutputStream(ArchiveStreamFactory.ZIP, zipOutput);
		/* Create Archieve entry - write header information */
		logicalZip.putArchiveEntry(new ZipArchiveEntry(FilenameUtils.getName(filePath)));
		/* Copy input file */
		IOUtils.copy(new FileInputStream(new File(filePath)), logicalZip);
		/* Close Archieve entry, write trailer information */
		logicalZip.closeArchiveEntry();

		/* Finish addition of entries to the file */
		logicalZip.finish();
		/* Close output stream, our files are zipped */
		zipOutput.close();
	} catch (Exception e) {
		System.err.println(e.getMessage());
		return null;
	}
	return filePath + ".zip";
}
 

@Test(priority = 11, expectedExceptions = InsightsCustomException.class)
public void testIncorrectFileException() throws InsightsCustomException, IOException {

	FileInputStream input = new FileInputStream(incorrectDataFile);
	MultipartFile multipartFile = new MockMultipartFile("file", incorrectDataFile.getName(), "text/plain",
			IOUtils.toByteArray(input));

	boolean response = bulkUploadService.uploadDataInDatabase(multipartFile, bulkUploadTestData.toolName,
			bulkUploadTestData.label, bulkUploadTestData.insightTimeField,
			bulkUploadTestData.nullInsightTimeFormat);

}
 

@Test(priority = 9, expectedExceptions = InsightsCustomException.class)
public void testUploadDataWithNullInsightTimeFieldInDatabase() throws InsightsCustomException, IOException {
	FileInputStream input = new FileInputStream(fileWithVariedEpochTimes);
	MultipartFile multipartFile = new MockMultipartFile("file", fileWithVariedEpochTimes.getName(), "text/plain",
			IOUtils.toByteArray(input));
	boolean response = bulkUploadService.uploadDataInDatabase(multipartFile, bulkUploadTestData.toolName,
			bulkUploadTestData.label, bulkUploadTestData.nullInsightTimeField,
			bulkUploadTestData.nullInsightTimeFormat);
	String expectedOutcome = "Insight Time Field not present in the file";
	Assert.assertEquals(response, expectedOutcome);
	Assert.assertFalse(multipartFile.isEmpty());
	Assert.assertTrue(multipartFile.getSize() < filesizeMaxValue);
}