Python lzma.LZMADecompressor() Examples

def test_roundtrip(self):
        cdata = lzma.compress(INPUT)
        ddata = lzma.decompress(cdata)
        self.assertEqual(ddata, INPUT)

        cdata = lzma.compress(INPUT, lzma.FORMAT_XZ)
        ddata = lzma.decompress(cdata)
        self.assertEqual(ddata, INPUT)

        cdata = lzma.compress(INPUT, lzma.FORMAT_ALONE)
        ddata = lzma.decompress(cdata)
        self.assertEqual(ddata, INPUT)

        cdata = lzma.compress(INPUT, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
        ddata = lzma.decompress(cdata, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
        self.assertEqual(ddata, INPUT)

    # Unlike LZMADecompressor, decompress() *does* handle concatenated streams. 

def test_decompressor_inputbuf_2(self):
        # Test reusing input buffer by appending data at the
        # end right away
        lzd = LZMADecompressor()
        out = []

        # Create input buffer and empty it
        self.assertEqual(lzd.decompress(COMPRESSED_XZ[:200],
                                        max_length=0), b'')
        out.append(lzd.decompress(b''))

        # Fill buffer with new data
        out.append(lzd.decompress(COMPRESSED_XZ[200:280], 2))

        # Append some more data, not enough to require resize
        out.append(lzd.decompress(COMPRESSED_XZ[280:300], 2))

        # Decompress rest of data
        out.append(lzd.decompress(COMPRESSED_XZ[300:]))
        self.assertEqual(b''.join(out), INPUT) 

def decompress(self, buf):
		if not self.compressed:
			return buf
		ty = self.compression_type
		if ty == CompressionType.LZMA:
			props, dict_size = struct.unpack("<BI", buf.read(5))
			lc = props % 9
			props = int(props / 9)
			pb = int(props / 5)
			lp = props % 5
			dec = lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=[{
				"id": lzma.FILTER_LZMA1,
				"dict_size": dict_size,
				"lc": lc,
				"lp": lp,
				"pb": pb,
			}])
			res = dec.decompress(buf.read())
			return BytesIO(res)
		if ty in (CompressionType.LZ4, CompressionType.LZ4HC):
			res = lz4_decompress(buf.read(self.compressed_size), self.uncompressed_size)
			return BytesIO(res)
		raise NotImplementedError("Unimplemented compression method: %r" % (ty)) 

def test_decompressor_inputbuf_1(self):
        # Test reusing input buffer after moving existing
        # contents to beginning
        lzd = LZMADecompressor()
        out = []

        # Create input buffer and fill it
        self.assertEqual(lzd.decompress(COMPRESSED_XZ[:100],
                                        max_length=0), b'')

        # Retrieve some results, freeing capacity at beginning
        # of input buffer
        out.append(lzd.decompress(b'', 2))

        # Add more data that fits into input buffer after
        # moving existing data to beginning
        out.append(lzd.decompress(COMPRESSED_XZ[100:105], 15))

        # Decompress rest of data
        out.append(lzd.decompress(COMPRESSED_XZ[105:]))
        self.assertEqual(b''.join(out), INPUT) 

def decompress_lzma(data):
    results = []
    len(data)
    while True:
        decomp = LZMADecompressor(FORMAT_AUTO, None, None)
        try:
            res = decomp.decompress(data)
        except LZMAError:
            if results:
                break  # Leftover data is not a valid LZMA/XZ stream; ignore it.
            else:
                raise  # Error on the first iteration; bail out.
        results.append(res)
        data = decomp.unused_data
        if not data:
            break
        if not decomp.eof:
            raise LZMAError("Compressed data ended before the end-of-stream marker was reached")
    return b"".join(results) 

def test_roundtrip(self):
        cdata = lzma.compress(INPUT)
        ddata = lzma.decompress(cdata)
        self.assertEqual(ddata, INPUT)

        cdata = lzma.compress(INPUT, lzma.FORMAT_XZ)
        ddata = lzma.decompress(cdata)
        self.assertEqual(ddata, INPUT)

        cdata = lzma.compress(INPUT, lzma.FORMAT_ALONE)
        ddata = lzma.decompress(cdata)
        self.assertEqual(ddata, INPUT)

        cdata = lzma.compress(INPUT, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
        ddata = lzma.decompress(cdata, lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
        self.assertEqual(ddata, INPUT)

    # Unlike LZMADecompressor, decompress() *does* handle concatenated streams. 

def test_decompressor_inputbuf_2(self):
        # Test reusing input buffer by appending data at the
        # end right away
        lzd = LZMADecompressor()
        out = []

        # Create input buffer and empty it
        self.assertEqual(lzd.decompress(COMPRESSED_XZ[:200],
                                        max_length=0), b'')
        out.append(lzd.decompress(b''))

        # Fill buffer with new data
        out.append(lzd.decompress(COMPRESSED_XZ[200:280], 2))

        # Append some more data, not enough to require resize
        out.append(lzd.decompress(COMPRESSED_XZ[280:300], 2))

        # Decompress rest of data
        out.append(lzd.decompress(COMPRESSED_XZ[300:]))
        self.assertEqual(b''.join(out), INPUT) 

def test_decompressor_inputbuf_1(self):
        # Test reusing input buffer after moving existing
        # contents to beginning
        lzd = LZMADecompressor()
        out = []

        # Create input buffer and fill it
        self.assertEqual(lzd.decompress(COMPRESSED_XZ[:100],
                                        max_length=0), b'')

        # Retrieve some results, freeing capacity at beginning
        # of input buffer
        out.append(lzd.decompress(b'', 2))

        # Add more data that fits into input buffer after
        # moving existing data to beginning
        out.append(lzd.decompress(COMPRESSED_XZ[100:105], 15))

        # Decompress rest of data
        out.append(lzd.decompress(COMPRESSED_XZ[105:]))
        self.assertEqual(b''.join(out), INPUT) 

def test_decompressor_bigmem(self, size):
        lzd = LZMADecompressor()
        blocksize = 10 * 1024 * 1024
        block = random.getrandbits(blocksize * 8).to_bytes(blocksize, "little")
        try:
            input = block * (size // blocksize + 1)
            cdata = lzma.compress(input)
            ddata = lzd.decompress(cdata)
            self.assertEqual(ddata, input)
        finally:
            input = cdata = ddata = None

    # Pickling raises an exception; there's no way to serialize an lzma_stream. 

def test_decompressor_raw_4(self):
        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4)
        self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE) 

def test_decompressor_chunks(self):
        lzd = LZMADecompressor()
        out = []
        for i in range(0, len(COMPRESSED_XZ), 10):
            self.assertFalse(lzd.eof)
            out.append(lzd.decompress(COMPRESSED_XZ[i:i+10]))
        out = b"".join(out)
        self.assertEqual(out, INPUT)
        self.assertEqual(lzd.check, lzma.CHECK_CRC64)
        self.assertTrue(lzd.eof)
        self.assertEqual(lzd.unused_data, b"") 

def get_lzma_decompressor(coders: List[Dict[str, Any]]):
    filters = []  # type: List[Dict[str, Any]]
    for coder in coders:
        if coder['numinstreams'] != 1 or coder['numoutstreams'] != 1:
            raise UnsupportedCompressionMethodError('Only a simple compression method is currently supported.')
        if not SupportedMethods.is_native_coder(coder):
            raise UnsupportedCompressionMethodError
        properties = coder.get('properties', None)
        filter_id = SupportedMethods.get_filter_id(coder)
        if properties is not None:
            filters[:0] = [lzma._decode_filter_properties(filter_id, properties)]  # type: ignore
        else:
            filters[:0] = [{'id': filter_id}]
    return lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters) 

def test_decompressor_raw_1(self):
        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1)
        self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE) 

def test_pickle(self):
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            with self.assertRaises(TypeError):
                pickle.dumps(LZMACompressor(), proto)
            with self.assertRaises(TypeError):
                pickle.dumps(LZMADecompressor(), proto) 

def test_decompress_memlimit(self):
        with self.assertRaises(LZMAError):
            lzma.decompress(COMPRESSED_XZ, memlimit=1024)
        with self.assertRaises(LZMAError):
            lzma.decompress(
                    COMPRESSED_XZ, format=lzma.FORMAT_XZ, memlimit=1024)
        with self.assertRaises(LZMAError):
            lzma.decompress(
                    COMPRESSED_ALONE, format=lzma.FORMAT_ALONE, memlimit=1024)

    # Test LZMADecompressor on known-good input data. 

def get_alternative_decompressor(coder: Dict[str, Any], password=None) -> Union[bz2.BZ2Decompressor, lzma.LZMADecompressor, ISevenZipDecompressor]:  # noqa
    if SupportedMethods.is_native_coder(coder):
        raise UnsupportedCompressionMethodError('Unknown method code:{}'.format(coder['method']))
    filter_id = SupportedMethods.get_filter_id(coder)
    if filter_id not in algorithm_class_map:
        raise UnsupportedCompressionMethodError('Unknown method filter_id:{}'.format(filter_id))
    if SupportedMethods.is_crypto_id(filter_id):
        return algorithm_class_map[filter_id][1](coder['properties'], password)
    elif SupportedMethods.need_property(filter_id):
        return algorithm_class_map[filter_id][1](coder['properties'])
    else:
        return algorithm_class_map[filter_id][1]() 

def test_lzma_lzma2bcj_compressor():
    filters = [{'id': 4}, {'id': 33, 'dict_size': 16777216}]
    assert lzma.LZMADecompressor(format=lzma.FORMAT_RAW, filters=filters) is not None 

def test_lzmadecompressor_lzmabcj():
    indata = b'This file is located in the root.'
    compressor = lzma.LZMACompressor(format=lzma.FORMAT_RAW,
                                     filters=[{'id': lzma.FILTER_X86}, {'id': lzma.FILTER_LZMA1}])
    compressed = compressor.compress(indata)
    compressed += compressor.flush()
    decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_RAW,
                                         filters=[{'id': lzma.FILTER_X86}, {'id': lzma.FILTER_LZMA1}])
    outdata = decompressor.decompress(data=compressed)
    assert outdata == indata 

def test_decompressor_raw_3(self):
        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3)
        self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE) 

def test_decompressor_raw_2(self):
        lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2)
        self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE) 

def test_pickle(self):
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            with self.assertRaises(TypeError):
                pickle.dumps(LZMACompressor(), proto)
            with self.assertRaises(TypeError):
                pickle.dumps(LZMADecompressor(), proto) 

def test_decompressor_xz(self):
        lzd = LZMADecompressor(lzma.FORMAT_XZ)
        self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) 

def test_decompressor_auto(self):
        lzd = LZMADecompressor()
        self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64)

        lzd = LZMADecompressor()
        self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) 

def test_decompressor_memlimit(self):
        lzd = LZMADecompressor(memlimit=1024)
        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)

        lzd = LZMADecompressor(lzma.FORMAT_XZ, memlimit=1024)
        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ)

        lzd = LZMADecompressor(lzma.FORMAT_ALONE, memlimit=1024)
        self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE)

    # Test LZMADecompressor on known-good input data. 

def test_decompressor_after_eof(self):
        lzd = LZMADecompressor()
        lzd.decompress(COMPRESSED_XZ)
        self.assertRaises(EOFError, lzd.decompress, b"nyan") 

def test_simple_bad_args(self):
        self.assertRaises(TypeError, LZMACompressor, [])
        self.assertRaises(TypeError, LZMACompressor, format=3.45)
        self.assertRaises(TypeError, LZMACompressor, check="")
        self.assertRaises(TypeError, LZMACompressor, preset="asdf")
        self.assertRaises(TypeError, LZMACompressor, filters=3)
        # Can't specify FORMAT_AUTO when compressing.
        self.assertRaises(ValueError, LZMACompressor, format=lzma.FORMAT_AUTO)
        # Can't specify a preset and a custom filter chain at the same time.
        with self.assertRaises(ValueError):
            LZMACompressor(preset=7, filters=[{"id": lzma.FILTER_LZMA2}])

        self.assertRaises(TypeError, LZMADecompressor, ())
        self.assertRaises(TypeError, LZMADecompressor, memlimit=b"qw")
        with self.assertRaises(TypeError):
            LZMADecompressor(lzma.FORMAT_RAW, filters="zzz")
        # Cannot specify a memory limit with FILTER_RAW.
        with self.assertRaises(ValueError):
            LZMADecompressor(lzma.FORMAT_RAW, memlimit=0x1000000)
        # Can only specify a custom filter chain with FILTER_RAW.
        self.assertRaises(ValueError, LZMADecompressor, filters=FILTERS_RAW_1)
        with self.assertRaises(ValueError):
            LZMADecompressor(format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1)
        with self.assertRaises(ValueError):
            LZMADecompressor(format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1)

        lzc = LZMACompressor()
        self.assertRaises(TypeError, lzc.compress)
        self.assertRaises(TypeError, lzc.compress, b"foo", b"bar")
        self.assertRaises(TypeError, lzc.flush, b"blah")
        empty = lzc.flush()
        self.assertRaises(ValueError, lzc.compress, b"quux")
        self.assertRaises(ValueError, lzc.flush)

        lzd = LZMADecompressor()
        self.assertRaises(TypeError, lzd.decompress)
        self.assertRaises(TypeError, lzd.decompress, b"foo", b"bar")
        lzd.decompress(empty)
        self.assertRaises(EOFError, lzd.decompress, b"quux") 

def decode_file(path):
    basename = os.path.splitext(path)[0]
    decodedname = basename + ".decoded.csv"

    print("process:", path, "->", decodedname)

    with open(path, 'rb') as f:
        data = f.read()

    tempdata = bytearray()

    for i in range(0, 8):
        tempdata.append(data[i])

    for i in range(0, 4):
        tempdata.append(0)

    for i in range(8, len(data)):
        tempdata.append(data[i])

    try:
        with open(decodedname, 'wb') as f:
            decompressor = lzma.LZMADecompressor()
            unpack_data = decompressor.decompress(tempdata)
            f.write(unpack_data)
    except:
        print("invalid input:", path)

#
# decode_file
# 

def from_bundle(cls, bundle, buf):
		ret = cls()
		ret.bundle = bundle
		ret.environment = bundle.environment
		offset = buf.tell()
		ret._buf = BinaryReader(buf, endian=">")

		if bundle.is_unityfs:
			ret._buf_ofs = buf.tell()
			return ret

		if not bundle.compressed:
			ret.name = buf.read_string()
			header_size = buf.read_uint()
			buf.read_uint()  # size
		else:
			header_size = bundle.asset_header_size

		# FIXME: this offset needs to be explored more
		ofs = buf.tell()
		if bundle.compressed:
			dec = lzma.LZMADecompressor()
			data = dec.decompress(buf.read())
			ret._buf = BinaryReader(BytesIO(data[header_size:]), endian=">")
			ret._buf_ofs = 0
			buf.seek(ofs)
		else:
			ret._buf_ofs = offset + header_size - 4
			if ret.is_resource:
				ret._buf_ofs -= len(ret.name)

		return ret 

def test_decompress_memlimit(self):
        with self.assertRaises(LZMAError):
            lzma.decompress(COMPRESSED_XZ, memlimit=1024)
        with self.assertRaises(LZMAError):
            lzma.decompress(
                    COMPRESSED_XZ, format=lzma.FORMAT_XZ, memlimit=1024)
        with self.assertRaises(LZMAError):
            lzma.decompress(
                    COMPRESSED_ALONE, format=lzma.FORMAT_ALONE, memlimit=1024)

    # Test LZMADecompressor on known-good input data. 

def test_decompressor_bigmem(self, size):
        lzd = LZMADecompressor()
        blocksize = 10 * 1024 * 1024
        block = random.getrandbits(blocksize * 8).to_bytes(blocksize, "little")
        try:
            input = block * (size // blocksize + 1)
            cdata = lzma.compress(input)
            ddata = lzd.decompress(cdata)
            self.assertEqual(ddata, input)
        finally:
            input = cdata = ddata = None

    # Pickling raises an exception; there's no way to serialize an lzma_stream.