Python lzma.compress() Examples

def run(results, raw_paste_data, paste_object):
    if config['outputs']['json_output']['store_raw']:
        original = raw_paste_data
        orig_size = len(original.encode())
        logger.debug("Compressing paste... Pre-compression size: {}", orig_size)
        compressed = base64.b64encode(lzma.compress(raw_paste_data.encode()))
        compressed_size = len(compressed)
        logger.debug("Compressing paste... Post-compression size: {}", compressed_size)

        # In some cases compressed blobs may be larger
        # if not much data is compressed
        if orig_size > compressed_size:
            paste_object['raw_paste'] = compressed.decode('utf-8')
            logger.debug("Compressed data smaller than original blob. Keeping compressed.")
        else:
            logger.debug("Original smaller than compressed blob. Keeping original.")

    # Regardless of modification, return the paste object
    return paste_object 

def test_snappy_read(self, tmpdir):
        comp = snappy.StreamCompressor()
        # generate two chunks with their own framing
        compressed = comp.compress(b"hello, ") + comp.compress(b"world")
        file_path = tmpdir.join("foo").strpath
        with open(file_path, "wb") as fp:
            fp.write(compressed)

        out = []
        with SnappyFile(open(file_path, "rb"), "rb") as fp:
            while True:
                chunk = fp.read()
                if not chunk:
                    break
                out.append(chunk)

        full = b"".join(out)
        assert full == b"hello, world" 

def CompressLZMA(string):
    """
    CompressBrotli(string) -> str
    Returns without delimiters the given string compressed \
using the lzstring compression method.

    """
    compressed = lzma.compress(
        string.encode("ascii"),
        format=lzma.FORMAT_RAW,
        filters=[{'id': lzma.FILTER_LZMA2, 'preset': 9 | lzma.PRESET_EXTREME}]
    )
    number = 1
    for c in compressed:
        number = number * 256 + c
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[LZMA_ENCODING] + result 

def CompressBrotli(string):
    """
    CompressBrotli(string) -> str
    Returns without delimiters the given string compressed \
using Google's brotli compression method.

    """
    compressed = brotli.compress(string.encode("ascii"))
    number = 1
    for c in compressed:
        number = number * 256 + c
    result = ""
    while number:
        result = Codepage[number % 255] + result
        number //= 255
    return Codepage[BROTLI_ENCODING] + result 

def test_decompression_event(self):
        ifile = WALTester(self.incoming_path, "00000001000000000000000A", "random")
        callback_queue = Queue()
        local_filepath = os.path.join(self.temp_dir, "00000001000000000000000A")
        self.compression_queue.put({
            "blob": self.compress(ifile.contents),
            "callback_queue": callback_queue,
            "filetype": "xlog",
            "local_path": local_filepath,
            "metadata": {
                "compression-algorithm": self.algorithm,
                "compression-level": 0,
                "host": socket.gethostname(),
                "original-file-size": ifile.size,
                "pg-version": 90500,
            },
            "site": self.test_site,
            "type": "DECOMPRESSION",
        })
        callback_queue.get(timeout=5.0)
        assert os.path.exists(local_filepath) is True
        with open(local_filepath, "rb") as fp:
            fdata = fp.read()
        assert fdata[:100] == ifile.contents[:100]
        assert fdata == ifile.contents 

def __generic_compressor(
    input_file_path: pathlib.Path,
    file_ending: str,
    compress_function: Callable[[Any], Any],
) -> None:
    """
    Compress a single file
    :param input_file_path: File to compress
    :param file_ending: Ending to add onto archive
    :param compress_function: Function that compresses
    """
    output_file_path = pathlib.Path(str(input_file_path) + file_ending)

    with input_file_path.open("rb") as f_in, output_file_path.open("wb") as f_out:
        f_out.write(compress_function(f_in.read())) 

def bz2_pack(source):
    """
    Returns `source` as bzip2-compressed Python script
    """
    import bz2
    compressed = base64.b64encode(bz2.compress(
        source.encode('utf-8'))).decode('utf-8')
    return f'import bz2,base64;exec(bz2.decompress(base64.b64decode("{compressed}")))' 

def _write_current_bin(self):
        self.f_store_positions.write_long(self.f_store.tell())
        self.current_bin.finalize(self.f_store, self.compress)
        self.current_bin = None 

def bz2_pack(source):
    """
    Returns 'source' as a bzip2-compressed, self-extracting python script.

    .. note::

        This method uses up more space than the zip_pack method but it has the
        advantage in that the resulting .py file can still be imported into a
        python program.
    """
    import bz2, base64
    out = ""
    # Preserve shebangs (don't care about encodings for this)
    first_line = source.split('\n')[0]
    if analyze.shebang.match(first_line):
        if py3:
            if first_line.rstrip().endswith('python'): # Make it python3
                first_line = first_line.rstrip()
                first_line += '3' #!/usr/bin/env python3
        out = first_line + '\n'
    compressed_source = bz2.compress(source.encode('utf-8'))
    out += 'import bz2, base64\n'
    out += "exec(bz2.decompress(base64.b64decode('"
    out += base64.b64encode(compressed_source).decode('utf-8')
    out += "')))\n"
    return out 

def gz_pack(source):
    """
    Returns 'source' as a gzip-compressed, self-extracting python script.

    .. note::

        This method uses up more space than the zip_pack method but it has the
        advantage in that the resulting .py file can still be imported into a
        python program.
    """
    import zlib, base64
    out = ""
    # Preserve shebangs (don't care about encodings for this)
    first_line = source.split('\n')[0]
    if analyze.shebang.match(first_line):
        if py3:
            if first_line.rstrip().endswith('python'): # Make it python3
                first_line = first_line.rstrip()
                first_line += '3' #!/usr/bin/env python3
        out = first_line + '\n'
    compressed_source = zlib.compress(source.encode('utf-8'))
    out += 'import zlib, base64\n'
    out += "exec(zlib.decompress(base64.b64decode('"
    out += base64.b64encode(compressed_source).decode('utf-8')
    out += "')))\n"
    return out 

def lzma_pack(source):
    """
    Returns 'source' as a lzma-compressed, self-extracting python script.

    .. note::

        This method uses up more space than the zip_pack method but it has the
        advantage in that the resulting .py file can still be imported into a
        python program.
    """
    import lzma, base64
    out = ""
    # Preserve shebangs (don't care about encodings for this)
    first_line = source.split('\n')[0]
    if analyze.shebang.match(first_line):
        if py3:
            if first_line.rstrip().endswith('python'): # Make it python3
                first_line = first_line.rstrip()
                first_line += '3' #!/usr/bin/env python3
        out = first_line + '\n'
    compressed_source = lzma.compress(source.encode('utf-8'))
    out += 'import lzma, base64\n'
    out += "exec(lzma.decompress(base64.b64decode('"
    out += base64.b64encode(compressed_source).decode('utf-8')
    out += "')))\n"
    return out 

def set(self, key: str, payload: bytes, ttl: int = None, format: str = 'html') -> None:
        compressed = lzma.compress(payload)
        self._cache.set(key + format, compressed, expire=ttl) 

def compress_directory(files_to_compress: List[pathlib.Path], output_name: str) -> None:
    """
    Compress a directory using all supported compression methods
    :param files_to_compress: Files to compress into a single archive
    :param output_name: Name to give compressed archive
    """
    temp_dir = mtgjson4.COMPILED_OUTPUT_DIR.joinpath(output_name)

    # Copy files to temporary folder
    LOGGER.info(f"Creating temporary directory for {output_name}")
    temp_dir.mkdir(parents=True, exist_ok=True)
    for file in files_to_compress:
        shutil.copy(str(file), str(temp_dir))

    # Compress the archives
    LOGGER.info(f"Compressing {temp_dir.name}")
    with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
        # Compression methods
        pool.apply_async(shutil.make_archive, (temp_dir, "bztar", str(temp_dir)))
        pool.apply_async(shutil.make_archive, (temp_dir, "gztar", str(temp_dir)))
        pool.apply_async(shutil.make_archive, (temp_dir, "xztar", str(temp_dir)))
        pool.apply_async(shutil.make_archive, (temp_dir, "zip", str(temp_dir)))

        # Wait for compressions to finish
        pool.close()
        pool.join()

    # Delete the temporary folder
    LOGGER.info(f"Removing temporary directory for {output_name}")
    shutil.rmtree(temp_dir, ignore_errors=True) 

def compress_file(file_path: pathlib.Path) -> None:
    """
    Compress a single file using all supported compression methods
    :param file_path: Path of file to compress
    """
    LOGGER.info(f"Compressing {file_path.name}")
    __generic_compressor(file_path, ".bz2", bz2.compress)
    __generic_compressor(file_path, ".gz", gzip.compress)
    __generic_compressor(file_path, ".xz", lzma.compress)

    # Zip files are done a bit differently
    with zipfile.ZipFile(str(file_path) + ".zip", "w") as zip_out:
        zip_out.write(file_path, file_path.name, zipfile.ZIP_DEFLATED) 

def finalize(self, fout: 'output file', compress: 'function'):
        count = len(self)
        fout.write(pack(U_INT, count))
        for content_type_id in self.content_type_ids:
            fout.write(pack(U_CHAR, content_type_id))
        content = b''.join(self.item_dir + self.items)
        compressed = compress(content)
        fout.write(pack(U_INT, len(compressed)))
        fout.write(compressed)
        self.content_type_ids.clear()
        self.item_dir.clear()
        self.items.clear() 

def compress(self, data):
        raise NotImplementedError 

def compress(self, data):
        return lzma.compress(data) 

def compress(self, data):
        return snappy.StreamCompressor().compress(data) 

def process(self, data):
        super(DeenPluginLzma, self).process(data)
        if not lzma:
            self.log.error('lzma module not found')
            return data
        try:
            data = lzma.compress(data)
        except lzma.LZMAError as e:
            self.error = e
            self.log.error(self.error)
            self.log.debug(self.error, exc_info=True)
        return data 

def packz(obj: Any) -> bytes:
    """
    Serialize and compress.
    """
    return lzma.compress(pack(obj), format=lzma.FORMAT_XZ) 

def to_pickle(obj):
    buff = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
    cbuff = lzma.compress(buff, format=lzma.FORMAT_XZ)
    return io.BytesIO(cbuff) 

def _get_content(self, exts):
        if isinstance(exts, str):
            exts = [exts]
        content = b''
        for ext in exts:
            filename = self.task_file[:-5] + ext
            if not os.path.isfile(filename):
                continue
            with open(filename, 'rb') as fh:
                content += fh.read()
        if not content:
            return b''
        return lzma.compress(content) 

def _set_params(self, params):
        params_block = lzma.compress(pickle.dumps(params))
        #env.logger.error(f'updating {self.task_id} params of size {len(params_block)}')
        with fasteners.InterProcessLock(
                os.path.join(env.temp_dir, self.task_id + '.lck')):
            with open(self.task_file, 'r+b') as fh:
                header = self._read_header(fh)
                if len(params_block) == header.params_size:
                    fh.seek(self.header_size, 0)
                    fh.write(params_block)
                else:
                    fh.read(header.params_size)
                    runtime = fh.read(header.runtime_size)
                    shell = fh.read(header.shell_size)
                    pulse = fh.read(header.pulse_size)
                    stdout = fh.read(header.stdout_size)
                    stderr = fh.read(header.stderr_size)
                    result = fh.read(header.result_size)
                    signature = fh.read(header.signature_size)
                    header = header._replace(params_size=len(params_block))
                    self._write_header(fh, header)
                    fh.write(params_block)
                    if runtime:
                        fh.write(runtime)
                    if shell:
                        fh.write(shell)
                    if pulse:
                        fh.write(pulse)
                    if stdout:
                        fh.write(stdout)
                    if stderr:
                        fh.write(stderr)
                    if result:
                        fh.write(result)
                    if signature:
                        fh.write(signature)
                    fh.truncate(self.header_size + header.params_size +
                                header.runtime_size + header.shell_size +
                                header.pulse_size + header.stdout_size +
                                header.stderr_size + header.result_size +
                                header.signature_size) 

def set(self, step_id: str, signature: dict):
        try:
            self._write((step_id, lzma.compress(pickle.dumps(signature))))
        except sqlite3.DatabaseError as e:
            env.logger.warning(
                f'Failed to set step signature for step {step_id}: {e}') 

def raw_deflate(self):
        """Raw deflate data
        
        Returns:
            Chepy: The Chepy object. 
        """
        self.state = zlib.compress(self._convert_to_bytes())[2:-4]
        return self 

def gz_pack(source):
    """
    Returns `source` as gzip-compressed Python script
    """
    import zlib
    compressed = base64.b64encode(zlib.compress(source.encode('utf-8'))).decode('utf-8')
    return f'import zlib,base64;exec(zlib.decompress(base64.b64decode("{compressed}")))' 

def lzma_pack(source):
    """
    Returns `source` as lzma-compressed Python script
    """
    import lzma
    compressed = base64.b64encode(lzma.compress(source.encode('utf-8'))).decode('utf-8')
    return f'import lzma,base64;exec(lzma.decompress(base64.b64decode("{compressed}")))' 

def test_index_gzip(self, urlopen_mock):
    if not Compression.GZIP.available:
      self.skipTest('(gzip compression unavailable)')

    import zlib
    urlopen_mock.return_value = io.BytesIO(zlib.compress(EXAMPLE_INDEX_JSON))

    collector = CollecTor()
    self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.GZIP))
    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None) 

def test_index_bz2(self, urlopen_mock):
    if not Compression.BZ2.available:
      self.skipTest('(bz2 compression unavailable)')

    import bz2
    urlopen_mock.return_value = io.BytesIO(bz2.compress(EXAMPLE_INDEX_JSON))

    collector = CollecTor()
    self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.BZ2))
    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None) 

def test_index_lzma(self, urlopen_mock):
    if not Compression.LZMA.available:
      self.skipTest('(lzma compression unavailable)')

    import lzma
    urlopen_mock.return_value = io.BytesIO(lzma.compress(EXAMPLE_INDEX_JSON))

    collector = CollecTor()
    self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.LZMA))
    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.xz', timeout = None)