net.jpountz.lz4.LZ4BlockOutputStream Java Examples

private void convertAndWriteToFile( final InputStream inputStream, final File tmpFile ) throws IOException {
	try ( final LZ4BlockOutputStream out = new LZ4BlockOutputStream( new FileOutputStream( tmpFile ) ) ) {
		reset();

		final XMLInputFactory inputFactory = XMLInputFactory.newInstance();
		inputFactory.setProperty( XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE );
		final XMLEventReader eventReader =
				inputFactory.createXMLEventReader( inputStream, StandardCharsets.UTF_8.name() );
		final XMLEventWriter eventWriter =
				XMLOutputFactory.newInstance().createXMLEventWriter( out, StandardCharsets.UTF_8.name() );

		while ( eventReader.hasNext() ) {
			final XMLEvent nextEvent = eventReader.nextEvent();
			convert( nextEvent, eventWriter );
		}
		eventReader.close();
		eventWriter.flush();
		eventWriter.close();

	} catch ( final XMLStreamException | FactoryConfigurationError e ) {
		throw new RuntimeException( e );
	}
}
 

@Override
protected byte[] compress(byte[] data) throws Exception {
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    int size = MAX_BLOCK_SIZE + 1;
    LZ4BlockOutputStream lz4Os = new LZ4BlockOutputStream(os,
            rand.nextInt(size - MIN_BLOCK_SIZE) + MIN_BLOCK_SIZE);
    lz4Os.write(data);
    lz4Os.close();

    return os.toByteArray();
}
 

@Override
public final OutputStream chainOutput(final OutputStream out) throws IOException {
  switch (compression) {
    case GZIP:
      return new GZIPOutputStream(out);
    case LZ4:
      return new LZ4BlockOutputStream(out);
    case NONE:
      return out;
    default:
      throw new UnsupportedCompressionException("Not supported compression method");
  }
}
 

@Override
public byte[] compress(byte[] data) throws IOException {
	LZ4Factory factory = LZ4Factory.fastestInstance();
	ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
	LZ4Compressor compressor = factory.fastCompressor();
	LZ4BlockOutputStream compressedOutput = new LZ4BlockOutputStream(byteOutput, 2048, compressor);
	compressedOutput.write(data);
	compressedOutput.close();
	
	return byteOutput.toByteArray();
}
 

private void openWriter() throws IOException {
    if (written) {
        throw new IllegalStateException("list is already closed");
    }
    if (writing) {
        throw new IllegalStateException("already writing on this list");
    }
    if (compressionEnabled) {
        this.tmpFile = Files.createTempFile(tmpDir, "listswap", ".tmp.gz");
    } else {
        this.tmpFile = Files.createTempFile(tmpDir, "listswap", ".tmp");
    }
    logger.log(Level.FINE, "opening tmp swap file {0}", tmpFile.toAbsolutePath());
    writing = true;
    try {
        out = Files.newOutputStream(tmpFile);
        bout = new SimpleBufferedOutputStream(out, DISK_BUFFER_SIZE);
        if (compressionEnabled) {
            zippedout = new LZ4BlockOutputStream(out);
            oout = new ExtendedDataOutputStream(zippedout);
        } else {
            oout = new ExtendedDataOutputStream(bout);
        }

    } catch (IOException ex) {
        closeWriter();
        throw new RuntimeException(ex);
    }
}
 

@Override
public final OutputStream chainOutput(final OutputStream out) throws IOException {
  switch (compression) {
    case Gzip:
      return new GZIPOutputStream(out);
    case LZ4:
      return new LZ4BlockOutputStream(out);
    default:
      throw new UnsupportedCompressionException("Not supported compression method");
  }
}
 

public void update(byte[] key, byte[] value) throws IOException {
    numKeys++;
    try {
        updateUnsafe(key, value);
    } catch (NullPointerException e) {
        file.getParentFile().mkdirs();
        file.createNewFile();
        this.os = new FaweOutputStream(new BufferedOutputStream(new ZstdOutputStream(new LZ4BlockOutputStream(new FileOutputStream(file)))));
        updateUnsafe(key, value);
    }
}
 

default void flushChanges(File file) throws IOException {
        try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(file))) {
            try (LZ4BlockOutputStream compressed = new LZ4BlockOutputStream(out)) {
//                compressed.setLevel(Deflater.BEST_SPEED);
                try (FaweOutputStream fos = new FaweOutputStream(compressed)) {
                    flushChanges(fos);
                }
            }
        }
    }
 

public final void open() throws IOException {
		
		if (!file.exists() && !writeable) {
			throw new IOException("Could not find file: " + file);
		} else {
			if (writeable) {
				if (uselz4) {
					
					LZ4BlockOutputStream os = new LZ4BlockOutputStream(new FileOutputStream(file), buffersize);
					out = new BufferedWriter(new OutputStreamWriter(os,"US-ASCII"), buffersize);
					
				} else if (gzipped) {
//					ParallelGZIPOutputStream gzipOutputStream = new ParallelGZIPOutputStream(new FileOutputStream(file), buffersize));
					GZIPOutputStream gzipOutputStream = new GZIPOutputStream(new FileOutputStream(file), buffersize) {{
						def.setLevel(Deflater.BEST_SPEED);
					}};
					out = new BufferedWriter(new OutputStreamWriter(gzipOutputStream, "US-ASCII"), buffersize);
					
				} else {
					out = new BufferedWriter(new FileWriter(file), buffersize);
				}
			} else {
				if (uselz4) {
					LZ4BlockInputStream is = new LZ4BlockInputStream(new FileInputStream(file));
					in = new BufferedReader(new InputStreamReader(is, "US-ASCII"));
				} else if (gzipped) {
					GZIPInputStream gzipInputStream = new GZIPInputStream(new FileInputStream(file), buffersize);
					in = new BufferedReader(new InputStreamReader(gzipInputStream, "US-ASCII"));
				} else {
//                System.out.println("Opening file: "+file);
					in = new BufferedReader(new InputStreamReader(new FileInputStream(file), ENCODING), buffersize);
				}
			}
		}
	}
 

@Override
protected OutputStream getCompressOutputStream(OutputStream outputStream) {
    return new LZ4BlockOutputStream(outputStream);
}