Python zlib.DEFLATED Examples

def ibytes2icompressed(source):
    yield (
        b'\037\213\010\000' +  # Gzip file, deflate, no filename
        struct.pack('<L', long(time.time())) +  # compression start time
        b'\002\377'  # maximum compression, no OS specified
    )

    crc = zlib.crc32(b"")
    length = 0
    compressor = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
    for d in source:
        crc = zlib.crc32(d, crc) & 0xffffffff
        length += len(d)
        chunk = compressor.compress(d)
        if chunk:
            yield chunk
    yield compressor.flush()
    yield struct.pack("<2L", crc, length & 0xffffffff) 

def compress_readable_output(src_file, compress_level=6):
    crc = zlib.crc32(b"")
    size = 0
    zobj = zlib.compressobj(compress_level, zlib.DEFLATED, -zlib.MAX_WBITS,
                            zlib.DEF_MEM_LEVEL, zlib.Z_DEFAULT_STRATEGY)
    prefix_written = False
    while True:
        data = src_file.read(DEFAULT_BUFFER_SIZE)
        if not data:
            break
        size += len(data)
        crc = zlib.crc32(data, crc)
        data = zobj.compress(data)
        if not prefix_written:
            prefix_written = True
            data = gzip_prefix() + data
        yield data
    yield zobj.flush() + struct.pack(b"<LL", crc & CRC_MASK, size) 

def _write_part(self):
        z = zlib.compressobj(6, zlib.DEFLATED, zlib.MAX_WBITS + 16)

        offset = self.gzip_temp.tell()

        buff = '\n'.join(self.curr_lines) + '\n'
        self.curr_lines = []

        buff = z.compress(buff)
        self.gzip_temp.write(buff)

        buff = z.flush()
        self.gzip_temp.write(buff)
        self.gzip_temp.flush()

        length = self.gzip_temp.tell() - offset

        partline = '{0}\t{1}\t{2}\t{3}'.format(self.curr_key, self.part_name, offset, length)

        return partline 

def gzipStream(input, compressLevel=6):
    crc, size = zlib.crc32(''), 0
    # magic header, compression method, no flags
    header = '\037\213\010\000'
    # timestamp
    header += struct.pack('<L', 0)
    # uh.. stuff
    header += '\002\377'
    yield header

    compress = zlib.compressobj(compressLevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
    _compress = compress.compress
    _crc32 = zlib.crc32

    yield input.wait
    for buf in input:
        if len(buf) != 0:
            crc = _crc32(buf, crc)
            size += len(buf)
            yield _compress(buf)
        yield input.wait

    yield compress.flush()
    yield struct.pack('<LL', crc & 0xFFFFFFFFL, size & 0xFFFFFFFFL) 

def enc(self, data, final):
        buf = []
        if not self.writeheader:
            h = header.new()
            h.mtime = int(time.time())
            h.fname = self.fname
            buf.append(header.tobytes(h))
            self.compobj = zlib.compressobj(self.level, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL)
            self.writeheader = True
        buf.append(self.compobj.compress(data))
        self.crc = zlib.crc32(data, self.crc) & 0xffffffff
        self.size += len(data)
        if final:
            buf.append(self.compobj.flush())
            t = tail.new()
            t.crc32 = self.crc
            t.isize = self.size
            buf.append(tail.tobytes(t))
        return b''.join(buf) 

def handle(self, connection):

        exported = self.createPlaylist(connection.headers['Host'], connection.reqtype, query_get(connection.query, 'fmt')).encode('utf-8')

        connection.send_response(200)
        connection.send_header('Content-Type', 'audio/mpegurl; charset=utf-8')
        connection.send_header('Access-Control-Allow-Origin', '*')
        try:
           h = connection.headers.get('Accept-Encoding').split(',')[0]
           compress_method = { 'zlib': zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS),
                               'deflate': zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS),
                               'gzip': zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) }
           exported = compress_method[h].compress(exported) + compress_method[h].flush()
           connection.send_header('Content-Encoding', h)
        except: pass
        connection.send_header('Content-Length', len(exported))
        connection.send_header('Connection', 'close')
        connection.end_headers()
        connection.wfile.write(exported) 

def test_receive_message_deflate(self):
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        data = '\xc1%c' % (len(compressed_hello) | 0x80)
        data += _mask_hybi(compressed_hello)

        # Close frame
        data += '\x88\x8a' + _mask_hybi(struct.pack('!H', 1000) + 'Good bye')

        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_request_from_rawdata(
                data, permessage_deflate_request=extension)
        self.assertEqual('Hello', msgutil.receive_message(request))

        self.assertEqual(None, msgutil.receive_message(request)) 

def compress_readable_output(src_file, compress_level=6):
    crc = zlib.crc32(b"")
    size = 0
    zobj = zlib.compressobj(compress_level, zlib.DEFLATED, -zlib.MAX_WBITS,
                            zlib.DEF_MEM_LEVEL, zlib.Z_DEFAULT_STRATEGY)
    prefix_written = False
    while True:
        data = src_file.read(DEFAULT_BUFFER_SIZE)
        if not data:
            break
        size += len(data)
        crc = zlib.crc32(data, crc)
        data = zobj.compress(data)
        if not prefix_written:
            prefix_written = True
            data = gzip_prefix() + data
        yield data
    yield zobj.flush() + struct.pack(b"<LL", crc & CRC_MASK, size) 

def test_send_message_fragmented_empty_last_frame(self):
        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_request_from_rawdata(
                '', permessage_deflate_request=extension)
        msgutil.send_message(request, 'Hello', end=False)
        msgutil.send_message(request, '')

        compress = zlib.compressobj(
                zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        expected = '\x41%c' % len(compressed_hello)
        expected += compressed_hello
        compressed_empty = compress.compress('')
        compressed_empty += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_empty = compressed_empty[:-4]
        expected += '\x80%c' % len(compressed_empty)
        expected += compressed_empty
        self.assertEqual(expected, request.connection.written_data()) 

def test_send_message_fragmented(self):
        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_request_from_rawdata(
                '', permessage_deflate_request=extension)
        msgutil.send_message(request, 'Hello', end=False)
        msgutil.send_message(request, 'Goodbye', end=False)
        msgutil.send_message(request, 'World')

        compress = zlib.compressobj(
                zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        expected = '\x41%c' % len(compressed_hello)
        expected += compressed_hello
        compressed_goodbye = compress.compress('Goodbye')
        compressed_goodbye += compress.flush(zlib.Z_SYNC_FLUSH)
        expected += '\x00%c' % len(compressed_goodbye)
        expected += compressed_goodbye
        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_world = compressed_world[:-4]
        expected += '\x80%c' % len(compressed_world)
        expected += compressed_world
        self.assertEqual(expected, request.connection.written_data()) 

def test_send_message(self):
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION)
        request = _create_request_from_rawdata(
            '', deflate_frame_request=extension)
        msgutil.send_message(request, 'Hello')
        msgutil.send_message(request, 'World')

        expected = ''

        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        expected += '\xc1%c' % len(compressed_hello)
        expected += compressed_hello

        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_world = compressed_world[:-4]
        expected += '\xc1%c' % len(compressed_world)
        expected += compressed_world

        self.assertEqual(expected, request.connection.written_data()) 

def gzip_app_iter(app_iter):
    size = 0
    crc = zlib.crc32(b"") & 0xffffffff
    compress = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS,
                                zlib.DEF_MEM_LEVEL, 0)

    yield _gzip_header
    for item in app_iter:
        size += len(item)
        crc = zlib.crc32(item, crc) & 0xffffffff

        # The compress function may return zero length bytes if the input is
        # small enough; it buffers the input for the next iteration or for a
        # flush.
        result = compress.compress(item)
        if result:
            yield result

    # Similarly, flush may also not yield a value.
    result = compress.flush()
    if result:
        yield result
    yield struct.pack("<2L", crc, size & 0xffffffff) 

def test_send_message_no_context_takeover_parameter(self):
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION)
        extension.add_parameter('no_context_takeover', None)
        request = _create_request_from_rawdata(
            '', deflate_frame_request=extension)
        for i in xrange(3):
            msgutil.send_message(request, 'Hello')

        compressed_message = compress.compress('Hello')
        compressed_message += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_message = compressed_message[:-4]
        expected = '\xc1%c' % len(compressed_message)
        expected += compressed_message

        self.assertEqual(
            expected + expected + expected, request.connection.written_data()) 

def _encode(self, data, con, path):
        compressobj = zlib.compressobj(
            6,
            zlib.DEFLATED,
            16 + 15,
            zlib.DEF_MEM_LEVEL,
            0
        )
        data = compressobj.compress(data)
        data += compressobj.flush()
        return data


# -------------------- Cipher Enums --------------------

# payload encryption method
# https://github.com/keepassxreboot/keepassxc/blob/8324d03f0a015e62b6182843b4478226a5197090/src/format/KeePass2.cpp#L24-L26 

def _post(self, data, url, session=None, timeout=None):
        session = session or self._session
        timeout = timeout or self._timeout
        _logger.debug('Raw datastream being sent: %s', pprint.pformat(data))

        if self._compress:
            uncompressed_bytes = len(data)
            c = zlib.compressobj(_COMPRESSION_LEVEL, zlib.DEFLATED,
                                 zlib.MAX_WBITS | 16)
            data = c.compress(data) + c.flush()
            _logger.debug('Compressed payload from %d to %d bytes',
                          uncompressed_bytes, len(data))

        response = session.post(url, data=data, timeout=timeout)
        _logger.debug('Sending to SignalFx %s (%d %s)',
                      'succeeded' if response.ok else 'failed',
                      response.status_code, response.text) 

def gzip_encode(self, content):
        gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)
        data = gzip_compress.compress(content) + gzip_compress.flush()
        return data 

def test_send_message_fragmented_bfinal(self):
        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_request_from_rawdata(
                '', permessage_deflate_request=extension)
        self.assertEquals(1, len(request.ws_extension_processors))
        request.ws_extension_processors[0].set_bfinal(True)
        msgutil.send_message(request, 'Hello', end=False)
        msgutil.send_message(request, 'World', end=True)

        expected = ''

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_FINISH)
        compressed_hello = compressed_hello + chr(0)
        expected += '\x41%c' % len(compressed_hello)
        expected += compressed_hello

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_FINISH)
        compressed_world = compressed_world + chr(0)
        expected += '\x80%c' % len(compressed_world)
        expected += compressed_world

        self.assertEqual(expected, request.connection.written_data()) 

def test_compressoptions(self):
        # specify lots of options to compressobj()
        level = 2
        method = zlib.DEFLATED
        wbits = -12
        memlevel = 9
        strategy = zlib.Z_FILTERED
        co = zlib.compressobj(level, method, wbits, memlevel, strategy)
        x1 = co.compress(HAMLET_SCENE)
        x2 = co.flush()
        dco = zlib.decompressobj(wbits)
        y1 = dco.decompress(x1 + x2)
        y2 = dco.flush()
        self.assertEqual(HAMLET_SCENE, y1 + y2) 

def gzipb(bytes):
    """bytes -> bytes"""
    compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)
    return compress.compress(bytes) + compress.flush() 

def deflateStream(input, compressLevel=6):
    # NOTE: this produces RFC-conformant but some-browser-incompatible output.
    # The RFC says that you're supposed to output zlib-format data, but many
    # browsers expect raw deflate output. Luckily all those browsers support
    # gzip, also, so they won't even see deflate output.
    compress = zlib.compressobj(compressLevel, zlib.DEFLATED, zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
    _compress = compress.compress
    yield input.wait
    for buf in input:
        if len(buf) != 0:
            yield _compress(buf)
        yield input.wait

    yield compress.flush() 

def test_badcompressobj(self):
        # verify failure on building compress object with bad params
        self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0) 

def add_binary(self, data, compressed=True, protected=True):
        if self.version >= (4, 0):
            # add protected flag byte
            if protected:
                data = b'\x01' + data
            else:
                data = b'\x00' + data
            # add binary element to inner header
            c = Container(type='binary', data=data)
            self.kdbx.body.payload.inner_header.binary.append(c)
        else:
            binaries = self._xpath(
                '/KeePassFile/Meta/Binaries',
                first=True
            )
            if compressed:
                # gzip compression
                compressor = zlib.compressobj(
                    zlib.Z_DEFAULT_COMPRESSION,
                    zlib.DEFLATED,
                    zlib.MAX_WBITS | 16
                )
                data = compressor.compress(data)
                data += compressor.flush()
            data = base64.b64encode(data).decode()

            # set ID for Binary Element
            binary_id = len(self.binaries)

            # add binary element to XML
            binaries.append(
                E.Binary(data, ID=str(binary_id), Compressed=str(compressed))
            )

        # return binary id
        return len(self.binaries) - 1 

def __init__(self, window_bits):
        self._logger = get_class_logger(self)

        self._compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -window_bits) 

def test_permessage_deflate_fragmented_message(self):
        extensions = common.parse_extensions(
            common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_mock_request(
            logical_channel_extensions=extensions)
        dispatcher = _MuxMockDispatcher()
        mux_handler = mux._MuxHandler(request, dispatcher)
        mux_handler.start()
        mux_handler.add_channel_slots(mux._INITIAL_NUMBER_OF_CHANNEL_SLOTS,
                                      mux._INITIAL_QUOTA_FOR_CLIENT)

        # Send compressed 'HelloHelloHello' as fragmented message.
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_hello = compress.compress('HelloHelloHello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]

        m = len(compressed_hello) / 2
        request.connection.put_bytes(
            _create_logical_frame(channel_id=1,
                                  message=compressed_hello[:m],
                                  fin=False, rsv1=True,
                                  opcode=common.OPCODE_TEXT))
        request.connection.put_bytes(
            _create_logical_frame(channel_id=1,
                                  message=compressed_hello[m:],
                                  fin=True, rsv1=False,
                                  opcode=common.OPCODE_CONTINUATION))

        request.connection.put_bytes(
            _create_logical_frame(channel_id=1, message='Goodbye'))

        self.assertTrue(mux_handler.wait_until_done(timeout=2))

        self.assertEqual(['HelloHelloHello'],
                         dispatcher.channel_events[1].messages)
        messages = request.connection.get_written_messages(1)
        self.assertEqual(1, len(messages))
        self.assertEqual(compressed_hello, messages[0]) 

def test_logarchive_successful(self):
        self.login("public", "password")
        data = b"my_file_stream"
        mock_file = BytesIO(zlib.compressobj(-1, zlib.DEFLATED, WINDOW_BUFFER_SIZE).compress(data))
        with patch("endpoints.web.log_archive._storage.stream_read_file", return_value=mock_file):
            self.getResponse("web.logarchive", file_id=self.build_uuid, expected_code=200) 

def test_send_message_no_context_takeover_parameter(self):
        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        extension.add_parameter('server_no_context_takeover', None)
        request = _create_request_from_rawdata(
                '', permessage_deflate_request=extension)
        for i in xrange(3):
            msgutil.send_message(request, 'Hello', end=False)
            msgutil.send_message(request, 'Hello', end=True)

        compress = zlib.compressobj(
                zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        first_hello = compress.compress('Hello')
        first_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        expected = '\x41%c' % len(first_hello)
        expected += first_hello
        second_hello = compress.compress('Hello')
        second_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        second_hello = second_hello[:-4]
        expected += '\x80%c' % len(second_hello)
        expected += second_hello

        self.assertEqual(
                expected + expected + expected,
                request.connection.written_data()) 

def test_send_two_messages(self):
        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_request_from_rawdata(
                '', permessage_deflate_request=extension)
        msgutil.send_message(request, 'Hello')
        msgutil.send_message(request, 'World')

        compress = zlib.compressobj(
                zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        expected = ''

        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        expected += '\xc1%c' % len(compressed_hello)
        expected += compressed_hello

        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_world = compressed_world[:-4]
        expected += '\xc1%c' % len(compressed_world)
        expected += compressed_world

        self.assertEqual(expected, request.connection.written_data()) 

def test_send_message(self):
        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_request_from_rawdata(
                '', permessage_deflate_request=extension)
        msgutil.send_message(request, 'Hello')

        compress = zlib.compressobj(
                zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        expected = '\xc1%c' % len(compressed_hello)
        expected += compressed_hello
        self.assertEqual(expected, request.connection.written_data()) 

def test_receive_message_various_btype(self):
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        data = ''

        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        data += '\xc1%c' % (len(compressed_hello) | 0x80)
        data += _mask_hybi(compressed_hello)

        compressed_websocket = compress.compress('WebSocket')
        compressed_websocket += compress.flush(zlib.Z_FINISH)
        compressed_websocket += '\x00'
        data += '\xc1%c' % (len(compressed_websocket) | 0x80)
        data += _mask_hybi(compressed_websocket)

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_world = compressed_world[:-4]
        data += '\xc1%c' % (len(compressed_world) | 0x80)
        data += _mask_hybi(compressed_world)

        # Close frame
        data += '\x88\x8a' + _mask_hybi(struct.pack('!H', 1000) + 'Good bye')

        extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION)
        request = _create_request_from_rawdata(
            data, deflate_frame_request=extension)
        self.assertEqual('Hello', msgutil.receive_message(request))
        self.assertEqual('WebSocket', msgutil.receive_message(request))
        self.assertEqual('World', msgutil.receive_message(request))

        self.assertEqual(None, msgutil.receive_message(request)) 

def test_receive_message_comp_bit(self):
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        data = ''

        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        data += '\xc1%c' % (len(compressed_hello) | 0x80)
        data += _mask_hybi(compressed_hello)

        data += '\x81\x85' + _mask_hybi('Hello')

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        compressed_2nd_hello = compress.compress('Hello')
        compressed_2nd_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_2nd_hello = compressed_2nd_hello[:-4]
        data += '\xc1%c' % (len(compressed_2nd_hello) | 0x80)
        data += _mask_hybi(compressed_2nd_hello)

        extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION)
        request = _create_request_from_rawdata(
            data, deflate_frame_request=extension)
        for i in xrange(3):
            self.assertEqual('Hello', msgutil.receive_message(request))