Python lzma.decompress() Examples

def download_latest_frida(device: Device):
    latest_release = requests.get(FRIDA_LATEST_RELEASE_URL).json()
    arch = get_architecture(device)

    for asset in latest_release['assets']:
        release_name = asset['name']
        if "server" in release_name and "android-{}.xz".format(arch) in release_name:
            eprint("⚡ Downloading {}...".format(release_name))
            xz_file = requests.get(asset['browser_download_url'])

            eprint("⚡ Extracting {}...".format(release_name))
            server_binary = lzma.decompress(xz_file.content)

            eprint("⚡ Writing {}...".format(release_name))
            with open(release_name[:-3], "wb") as f:  # remove extension
                f.write(server_binary)
            return release_name[:-3] 

def get(self, step_id: str):
        try:
            cur = self.conn.cursor()
            cur.execute('SELECT signature FROM steps WHERE step_id=? ',
                        (step_id,))
            res = cur.fetchone()
        except sqlite3.DatabaseError as e:
            env.logger.warning(
                f'Failed to get step signature for step {step_id}: {e}')
            return None
        if res:
            try:
                return pickle.loads(lzma.decompress(res[0]))
            except Exception as e:
                env.logger.warning(
                    f'Failed to load signature for step {step_id}: {e}')
                return None
        else:
            return None 

def __init__(self, file_or_filenames):
        self._f = MultiFileReader(file_or_filenames)

        try:
            self._header = read_header(self._f)
            if (self._f.size != self._header.size):
                raise IncorrectFileSize(
                    'File size should be {0}, {1} bytes found'
                    .format(self._header.size, self._f.size))
        except FileFormatException:
            self._f.close()
            raise

        self._refs = RefList(self._f,
                             self._header.encoding,
                             offset=self._header.refs_offset)

        self._g = MultiFileReader(file_or_filenames)
        self._store = Store(self._g,
                            self._header.store_offset,
                            COMPRESSIONS[self._header.compression].decompress,
                            self._header.content_types) 

def DecompressBrotli(string):
    """
    DecompressBrotli(string) -> str
    Returns the original form of the given string compressed \
using Google's brotli compression method., passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    compressed = []
    while number > 1:
        compressed = [number % 256] + compressed
        number //= 256
    return brotli.decompress(bytes(compressed)).decode("ascii") 

def DecompressLZMA(string):
    """
    DecompressBrotli(string) -> str
    Returns the original form of the given string compressed \
using Google's brotli compression method., passed without delimiters.

    """
    number = 0
    for character in string:
        ordinal = OrdinalLookup.get(character, ord(character))
        number = number * 255 + ordinal - (ordinal > gap)
    compressed = []
    while number > 1:
        compressed = [number % 256] + compressed
        number //= 256
    return lzma.decompress(
        bytes(compressed),
        format=lzma.FORMAT_RAW,
        filters=[{'id': lzma.FILTER_LZMA2, 'preset': 9 | lzma.PRESET_EXTREME}]
    ).decode("ascii") 

def lzma_pack(source):
    """
    Returns 'source' as a lzma-compressed, self-extracting python script.

    .. note::

        This method uses up more space than the zip_pack method but it has the
        advantage in that the resulting .py file can still be imported into a
        python program.
    """
    import lzma, base64
    out = ""
    # Preserve shebangs (don't care about encodings for this)
    first_line = source.split('\n')[0]
    if analyze.shebang.match(first_line):
        if py3:
            if first_line.rstrip().endswith('python'): # Make it python3
                first_line = first_line.rstrip()
                first_line += '3' #!/usr/bin/env python3
        out = first_line + '\n'
    compressed_source = lzma.compress(source.encode('utf-8'))
    out += 'import lzma, base64\n'
    out += "exec(lzma.decompress(base64.b64decode('"
    out += base64.b64encode(compressed_source).decode('utf-8')
    out += "')))\n"
    return out 

def test_compress_to_memory(self):
        ifile = WALTester(self.incoming_path, "0000000100000000000000FF", "random")
        self.compression_queue.put({
            "compress_to_memory": True,
            "delete_file_after_compression": False,
            "full_path": ifile.path,
            "src_path": ifile.path_partial,
            "type": "MOVE",
        })
        expected = {
            "callback_queue": None,
            "filetype": "xlog",
            "local_path": ifile.path,
            "metadata": {
                "compression-algorithm": self.algorithm,
                "compression-level": 0,
                "host": socket.gethostname(),
                "original-file-size": ifile.size,
                "pg-version": 90500,
            },
            "site": self.test_site,
        }
        transfer_event = self.transfer_queue.get(timeout=3.0)
        for key, value in expected.items():
            if key == "metadata":
                assert transfer_event[key].pop("hash")
                assert transfer_event[key].pop("hash-algorithm") == "sha1"
            assert transfer_event[key] == value

        result = self.decompress(transfer_event["blob"])
        assert result[:100] == ifile.contents[:100]
        assert result == ifile.contents 

def decompress(self, data):
        raise NotImplementedError 

def get(self, key: str, format: str = 'html') -> Optional[bytes]:
        loop = asyncio.get_event_loop()
        cache_get = self._cache.get
        data = await loop.run_in_executor(None, cache_get, key + format)
        if data is not None:
            res = await loop.run_in_executor(None, lzma.decompress, data)
            return res 

def decompress(self, source, cursor, compressedbytes, uncompressedbytes=None):
        if self.algo == uproot.const.kZLIB:
            from zlib import decompress as zlib_decompress
            return zlib_decompress(cursor.bytes(source, compressedbytes))

        elif self.algo == uproot.const.kLZMA:
            try:
                from lzma import decompress as lzma_decompress
            except ImportError:
                try:
                    from backports.lzma import decompress as lzma_decompress
                except ImportError:
                    raise ImportError("install lzma package with:\n    pip install backports.lzma\nor\n    conda install backports.lzma\n(or just use Python >= 3.3).")
            return lzma_decompress(cursor.bytes(source, compressedbytes))

        elif self.algo == uproot.const.kOldCompressionAlgo:
            raise NotImplementedError("ROOT's \"old\" algorithm (fCompress 300) is not supported")

        elif self.algo == uproot.const.kLZ4:
            try:
                from lz4.block import decompress as lz4_decompress
            except ImportError:
                raise ImportError("install lz4 package with:\n    pip install lz4\nor\n    conda install lz4")

            if uncompressedbytes is None:
                raise ValueError("lz4 needs to know the uncompressed number of bytes")
            return lz4_decompress(cursor.bytes(source, compressedbytes), uncompressed_size=uncompressedbytes)

        elif self.algo == uproot.const.kZSTD:
            try:
                import zstandard as zstd
            except ImportError:
                raise ImportError("install zstd package with:\n    pip install zstandard\nor\n    conda install zstandard")
            dctx = zstd.ZstdDecompressor()
            return dctx.decompress(cursor.bytes(source, compressedbytes))

        else:
            raise ValueError("unrecognized compression algorithm: {0}".format(self.algo)) 

def decompress(self, source, cursor, compressedbytes, uncompressedbytes=None):
        if self.algo == uproot.const.kZLIB:
            from zlib import decompress as zlib_decompress
            return zlib_decompress(cursor.bytes(source, compressedbytes))

        elif self.algo == uproot.const.kLZMA:
            try:
                from lzma import decompress as lzma_decompress
            except ImportError:
                try:
                    from backports.lzma import decompress as lzma_decompress
                except ImportError:
                    raise ImportError("install lzma package with:\n    pip install backports.lzma\nor\n    conda install backports.lzma\n(or just use Python >= 3.3).")
            return lzma_decompress(cursor.bytes(source, compressedbytes))

        elif self.algo == uproot.const.kOldCompressionAlgo:
            raise NotImplementedError("ROOT's \"old\" algorithm (fCompress 300) is not supported")

        elif self.algo == uproot.const.kLZ4:
            try:
                from lz4.block import decompress as lz4_decompress
            except ImportError:
                raise ImportError("install lz4 package with:\n    pip install lz4\nor\n    conda install lz4")

            if uncompressedbytes is None:
                raise ValueError("lz4 needs to know the uncompressed number of bytes")
            return lz4_decompress(cursor.bytes(source, compressedbytes), uncompressed_size=uncompressedbytes)

        elif self.algo == uproot.const.kZSTD:
            try:
                import zstandard as zstd
            except ImportError:
                raise ImportError("install zstd package with:\n    pip install zstandard\nor\n    conda install zstandard")
            dctx = zstd.ZstdDecompressor()
            return dctx.decompress(cursor.bytes(source, compressedbytes))

        else:
            raise ValueError("unrecognized compression algorithm: {0}".format(self.algo)) 

def _read_block_header(self):
        self.block_count = self.raw_decoder.read_long()
        if self.codec == "null":
            # Skip a long; we don't need to use the length.
            self.raw_decoder.skip_long()
            self._datum_decoder = self._raw_decoder
        elif self.codec == 'deflate':
            # Compressed data is stored as (length, data), which
            # corresponds to how the "bytes" type is encoded.
            data = self.raw_decoder.read_bytes()
            # -15 is the log of the window size; negative indicates
            # "raw" (no zlib headers) decompression.    See zlib.h.
            uncompressed = zlib.decompress(data, -15)
            self._datum_decoder = io.BinaryDecoder(StringIO(uncompressed))
        elif self.codec == 'snappy':
            # Compressed data includes a 4-byte CRC32 checksum
            length = self.raw_decoder.read_long()
            data = self.raw_decoder.read(length - 4)
            uncompressed = snappy.decompress(data)
            self._datum_decoder = io.BinaryDecoder(StringIO(uncompressed))
            self.raw_decoder.check_crc32(uncompressed);
        elif self.codec == 'xz':
            # Compressed data is stored as (length, data), which
            # corresponds to how the "bytes" type is encoded.
            data = self.raw_decoder.read_bytes()
            uncompressed = lzma.decompress(data)
            self._datum_decoder = io.BinaryDecoder(StringIO(uncompressed))
        else:
            raise DataFileException("Unknown codec: %r" % self.codec) 

def test_compress_decompress_stream(self):
        plaintext = WALTester(self.incoming_path, "00000001000000000000000E", "random").contents
        compressed_stream = self.make_compress_stream(io.BytesIO(plaintext))
        result_data = io.BytesIO()
        while True:
            bytes_requested = random.randrange(1, 12345)
            data = compressed_stream.read(bytes_requested)
            if not data:
                break
            result_data.write(data)
            # Must return exactly the amount of data requested except when reaching end of stream
            if len(data) < bytes_requested:
                assert not compressed_stream.read(1)
                break
            assert len(data) == bytes_requested
        assert result_data.tell() > 0
        assert compressed_stream.tell() > 0
        result_data.seek(0)
        decompressed = self.decompress(result_data.read())
        assert plaintext == decompressed

        compressed_stream = self.make_compress_stream(io.BytesIO(plaintext))
        compressed_data = compressed_stream.read()
        assert compressed_stream.tell() == len(compressed_data)
        decompressed = self.decompress(compressed_data)
        assert plaintext == decompressed 

def gz_pack(source):
    """
    Returns 'source' as a gzip-compressed, self-extracting python script.

    .. note::

        This method uses up more space than the zip_pack method but it has the
        advantage in that the resulting .py file can still be imported into a
        python program.
    """
    import zlib, base64
    out = ""
    # Preserve shebangs (don't care about encodings for this)
    first_line = source.split('\n')[0]
    if analyze.shebang.match(first_line):
        if py3:
            if first_line.rstrip().endswith('python'): # Make it python3
                first_line = first_line.rstrip()
                first_line += '3' #!/usr/bin/env python3
        out = first_line + '\n'
    compressed_source = zlib.compress(source.encode('utf-8'))
    out += 'import zlib, base64\n'
    out += "exec(zlib.decompress(base64.b64decode('"
    out += base64.b64encode(compressed_source).decode('utf-8')
    out += "')))\n"
    return out 

def bz2_pack(source):
    """
    Returns 'source' as a bzip2-compressed, self-extracting python script.

    .. note::

        This method uses up more space than the zip_pack method but it has the
        advantage in that the resulting .py file can still be imported into a
        python program.
    """
    import bz2, base64
    out = ""
    # Preserve shebangs (don't care about encodings for this)
    first_line = source.split('\n')[0]
    if analyze.shebang.match(first_line):
        if py3:
            if first_line.rstrip().endswith('python'): # Make it python3
                first_line = first_line.rstrip()
                first_line += '3' #!/usr/bin/env python3
        out = first_line + '\n'
    compressed_source = bz2.compress(source.encode('utf-8'))
    out += 'import bz2, base64\n'
    out += "exec(bz2.decompress(base64.b64decode('"
    out += base64.b64encode(compressed_source).decode('utf-8')
    out += "')))\n"
    return out 

def parse_play_data(self, replay_data):
        offset_end = self.offset+self.__replay_length
        if self.game_mode != GameMode.Standard:
            self.play_data = None
        else:
            datastring = lzma.decompress(replay_data[self.offset:offset_end], format=lzma.FORMAT_AUTO).decode('ascii')[:-1]
            events = [eventstring.split('|') for eventstring in datastring.split(',')]
            self.play_data = [ReplayEvent(int(event[0]), float(event[1]), float(event[2]), int(event[3])) for event in events]
        self.offset = offset_end 

def _decompress(self, bin_index):
        store_item = self[bin_index]
        return self.decompress(store_item.compressed_content) 

def test_archive_command_compression(self):
        zero = WALTester(self.incoming_path, "00000001000000000000000D", "zero")
        callback_queue = Queue()
        event = {
            "callback_queue": callback_queue,
            "compress_to_memory": True,
            "delete_file_after_compression": False,
            "full_path": zero.path,
            "src_path": zero.path_partial,
            "type": "MOVE",
        }
        self.compression_queue.put(event)
        transfer_event = self.transfer_queue.get(timeout=5.0)
        expected = {
            "callback_queue": callback_queue,
            "filetype": "xlog",
            "local_path": zero.path,
            "metadata": {
                "compression-algorithm": self.algorithm,
                "compression-level": 0,
                "host": socket.gethostname(),
                "original-file-size": zero.size,
                "pg-version": 90500,
            },
            "site": self.test_site,
        }
        for key, value in expected.items():
            if key == "metadata":
                assert transfer_event[key].pop("hash")
                assert transfer_event[key].pop("hash-algorithm") == "sha1"
            assert transfer_event[key] == value

        assert self.decompress(transfer_event["blob"]) == zero.contents 

def read_file(self, f, flip_hr=False):
        self.mode, self.version = struct.unpack('<BI', f.read(5))
        assert self.mode == 0, "%s support not added yet" % MODES[self.mode]
        self.beatmap_hash = parse_string(f)
        self.player = parse_string(f)
        self.replay_hash = parse_string(f)
        self.n300, self.n100, self.n50, self.ngeki, self.nkatu, self.nmiss, \
            self.score, self.combo, self.perfect, self.mods = \
            struct.unpack('<HHHHHHIH?I', f.read(23))
        self.life_events = deque()
        for rec in parse_string(f).split(','):
            if rec:
                u, v = rec.split('|')
                self.life_events.append((int(u), float(v)))
        self.timestamp, self.length = struct.unpack('<QI', f.read(12))
        self.replay = replay = deque()
        flip = flip_hr and self.has_mod(16)
        t = 0
        for rec in lzma.decompress(f.read(self.length)).decode().split(','):
            if rec:
                w, x, y, z = rec.split('|')
                w, x, y, z = int(w), float(x), float(y), int(z)
                t += w
                if flip:
                    y = 384 - y
                p = ReplayPoint(t, x, y, z)
                replay.append(p) 

def decompress(self, data):
        return lzma.decompress(data) 

def decompress(self, data):
        return zstd.ZstdDecompressor().decompressobj().decompress(data) 

def unpackz(buff: bytes) -> Any:
    """
    Decompress and unserialize.
    """
    return unpack(lzma.decompress(buff, format=lzma.FORMAT_XZ)) 

def from_pickle(buff):
    return pickle.loads(lzma.decompress(buff.read(), format=lzma.FORMAT_XZ)) 

def _get_params(self):
        with open(self.task_file, 'rb') as fh:
            header = self._read_header(fh)
            if header.params_size == 0 and header.runtime_size == 0:
                return {}
            fh.seek(self.header_size, 0)
            if header.params_size == 0:
                return {}
            else:
                try:
                    return pickle.loads(
                        lzma.decompress(fh.read(header.params_size)))
                except Exception as e:
                    raise RuntimeError(
                        f'Failed to obtain params of task {self.task_id}: {e}') 

def _get_runtime(self):
        with open(self.task_file, 'rb') as fh:
            header = self._read_header(fh)
            if header.runtime_size == 0:
                return {}
            fh.seek(self.header_size + header.params_size, 0)
            try:
                return pickle.loads(
                    lzma.decompress(fh.read(header.runtime_size)))
            except Exception as e:
                env.logger.error(
                    f'Failed to obtain runtime of task {self.task_id}: {e}')
                return {'_runtime': {}} 

def get_params_and_runtime(self):
        with open(self.task_file, 'rb') as fh:
            header = self._read_header(fh)
            if header.params_size == 0 and header.runtime_size == 0:
                return {}
            fh.seek(self.header_size, 0)
            if header.params_size == 0:
                params = {}
            else:
                try:
                    params = pickle.loads(
                        lzma.decompress(fh.read(header.params_size)))
                except Exception as e:
                    env.logger.error(
                        f'Failed to obtain params with runtime of task {self.task_id}: {e}'
                    )
                    params = {}
            if '_runtime' not in params.sos_dict:
                params.sos_dict['_runtime'] = {}
            if header.runtime_size > 0:
                try:
                    runtime = pickle.loads(
                        lzma.decompress(fh.read(header.runtime_size)))
                except Exception as e:
                    env.logger.error(
                        f'Failed to obtain runtime of task {self.task_id}: {e}')
                    runtime = {'_runtime': {}}
            else:
                runtime = {'_runtime': {}}
            return params, runtime 

def _get_shell(self):
        with open(self.task_file, 'rb') as fh:
            header = self._read_header(fh)
            if header.shell_size == 0:
                return ''
            fh.seek(self.header_size + header.params_size + header.runtime_size,
                    0)
            try:
                return lzma.decompress(fh.read(header.shell_size)).decode()
            except Exception as e:
                env.logger.warning(f'Failed to decode shell: {e}')
                return '' 

def _get_pulse(self):
        with open(self.task_file, 'rb') as fh:
            header = self._read_header(fh)
            if header.pulse_size == 0:
                return ''
            fh.seek(
                self.header_size + header.params_size + header.runtime_size +
                header.shell_size, 0)
            try:
                return lzma.decompress(fh.read(header.pulse_size)).decode()
            except Exception as e:
                env.logger.warning(f'Failed to decode pulse: {e}')
                return '' 

def _get_stderr(self):
        with open(self.task_file, 'rb') as fh:
            header = self._read_header(fh)
            if header.stderr_size == 0:
                return ''
            fh.seek(
                self.header_size + header.params_size + header.runtime_size +
                header.shell_size + header.pulse_size + header.stdout_size, 0)
            try:
                return lzma.decompress(fh.read(header.stderr_size)).decode()
            except Exception as e:
                env.logger.warning(f'Failed to decode stderr: {e}')
                return '' 

def _get_result(self):
        with open(self.task_file, 'rb') as fh:
            header = self._read_header(fh)
            if header.result_size == 0:
                return {}
            fh.seek(
                self.header_size + header.params_size + header.runtime_size +
                header.shell_size + header.pulse_size + header.stdout_size +
                header.stderr_size, 0)
            try:
                return pickle.loads(
                    lzma.decompress(fh.read(header.result_size)))
            except Exception as e:
                env.logger.warning(f'Failed to decode result: {e}')
                return {'ret_code': 1}