Python zlib.Z_FINISH Examples

def method_get(self):
        self.wfile.write(self.default_headers)

        # Compressed data loop
        if self.server.compression:
            c = zlib.compressobj(level=1, wbits=16 + zlib.MAX_WBITS)

            while True:
                data = sys.stdin.buffer.read(65536)

                if data is None or len(data) == 0:
                    self.wfile.write(c.flush(zlib.Z_FINISH))
                    break

                self.wfile.write(c.compress(data))

        # Normal data loop
        else:
            while True:
                data = sys.stdin.buffer.read(65536)

                if data is None or len(data) == 0:
                    break

                self.wfile.write(data) 

def close(self):
        fileobj = self.fileobj
        if fileobj is None:
            return
        self.fileobj = None
        try:
            if self.mode == gzip.WRITE:
                fileobj.write(self.compress.flush(Z_FINISH))
                gzip.write32u(fileobj, self.crc)
                # self.size may exceed 2GB, or even 4GB
                gzip.write32u(fileobj, self.size & 0xffffffff)
                fileobj.flush()
        finally:
            myfileobj = self.myfileobj
            if myfileobj:
                self.myfileobj = None
                myfileobj.close() 

def flush(self, lib_mode=FLUSH):
        self._buffer.flush()
        GzipFile.flush(self, lib_mode) 

def close(self):
        self._socket.sendall(self._compressor.flush(zlib.Z_FINISH))
        self._socket.close() 

def testZLibFlushRecord(self):
    fn = self._WriteRecordsToFile([b"small record"], "small_record")
    with open(fn, "rb") as h:
      buff = h.read()

    # creating more blocks and trailing blocks shouldn't break reads
    compressor = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS)

    output = b""
    for c in buff:
      if isinstance(c, int):
        c = six.int2byte(c)
      output += compressor.compress(c)
      output += compressor.flush(zlib.Z_FULL_FLUSH)

    output += compressor.flush(zlib.Z_FULL_FLUSH)
    output += compressor.flush(zlib.Z_FULL_FLUSH)
    output += compressor.flush(zlib.Z_FINISH)

    # overwrite the original file with the compressed data
    with open(fn, "wb") as h:
      h.write(output)

    with self.test_session() as sess:
      options = tf.python_io.TFRecordOptions(
          compression_type=TFRecordCompressionType.ZLIB)
      reader = tf.TFRecordReader(name="test_reader", options=options)
      queue = tf.FIFOQueue(1, [tf.string], shapes=())
      key, value = reader.read(queue)
      queue.enqueue(fn).run()
      queue.close().run()
      k, v = sess.run([key, value])
      self.assertTrue(tf.compat.as_text(k).startswith("%s:" % fn))
      self.assertAllEqual(b"small record", v) 

def test_send_message_bfinal(self):
        extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION)
        request = _create_request_from_rawdata(
            '', deflate_frame_request=extension)
        self.assertEquals(1, len(request.ws_extension_processors))
        deflate_frame_processor = request.ws_extension_processors[0]
        deflate_frame_processor.set_bfinal(True)
        msgutil.send_message(request, 'Hello')
        msgutil.send_message(request, 'World')

        expected = ''

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_FINISH)
        compressed_hello = compressed_hello + chr(0)
        expected += '\xc1%c' % len(compressed_hello)
        expected += compressed_hello

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_FINISH)
        compressed_world = compressed_world + chr(0)
        expected += '\xc1%c' % len(compressed_world)
        expected += compressed_world

        self.assertEqual(expected, request.connection.written_data()) 

def test_receive_message(self):
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        data = ''

        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        data += '\xc1%c' % (len(compressed_hello) | 0x80)
        data += _mask_hybi(compressed_hello)

        compressed_websocket = compress.compress('WebSocket')
        compressed_websocket += compress.flush(zlib.Z_FINISH)
        compressed_websocket += '\x00'
        data += '\xc1%c' % (len(compressed_websocket) | 0x80)
        data += _mask_hybi(compressed_websocket)

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_world = compressed_world[:-4]
        data += '\xc1%c' % (len(compressed_world) | 0x80)
        data += _mask_hybi(compressed_world)

        # Close frame
        data += '\x88\x8a' + _mask_hybi(struct.pack('!H', 1000) + 'Good bye')

        extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION)
        request = _create_request_from_rawdata(
            data, deflate_frame_request=extension)
        self.assertEqual('Hello', msgutil.receive_message(request))
        self.assertEqual('WebSocket', msgutil.receive_message(request))
        self.assertEqual('World', msgutil.receive_message(request))

        self.assertEqual(None, msgutil.receive_message(request)) 

def test_receive_message_various_btype(self):
        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        data = ''

        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_hello = compressed_hello[:-4]
        data += '\xc1%c' % (len(compressed_hello) | 0x80)
        data += _mask_hybi(compressed_hello)

        compressed_websocket = compress.compress('WebSocket')
        compressed_websocket += compress.flush(zlib.Z_FINISH)
        compressed_websocket += '\x00'
        data += '\xc1%c' % (len(compressed_websocket) | 0x80)
        data += _mask_hybi(compressed_websocket)

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)

        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_SYNC_FLUSH)
        compressed_world = compressed_world[:-4]
        data += '\xc1%c' % (len(compressed_world) | 0x80)
        data += _mask_hybi(compressed_world)

        # Close frame
        data += '\x88\x8a' + _mask_hybi(struct.pack('!H', 1000) + 'Good bye')

        extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION)
        request = _create_request_from_rawdata(
            data, deflate_frame_request=extension)
        self.assertEqual('Hello', msgutil.receive_message(request))
        self.assertEqual('WebSocket', msgutil.receive_message(request))
        self.assertEqual('World', msgutil.receive_message(request))

        self.assertEqual(None, msgutil.receive_message(request)) 

def test_send_message_fragmented_bfinal(self):
        extension = common.ExtensionParameter(
                common.PERMESSAGE_DEFLATE_EXTENSION)
        request = _create_request_from_rawdata(
                '', permessage_deflate_request=extension)
        self.assertEquals(1, len(request.ws_extension_processors))
        request.ws_extension_processors[0].set_bfinal(True)
        msgutil.send_message(request, 'Hello', end=False)
        msgutil.send_message(request, 'World', end=True)

        expected = ''

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_hello = compress.compress('Hello')
        compressed_hello += compress.flush(zlib.Z_FINISH)
        compressed_hello = compressed_hello + chr(0)
        expected += '\x41%c' % len(compressed_hello)
        expected += compressed_hello

        compress = zlib.compressobj(
            zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS)
        compressed_world = compress.compress('World')
        compressed_world += compress.flush(zlib.Z_FINISH)
        compressed_world = compressed_world + chr(0)
        expected += '\x80%c' % len(compressed_world)
        expected += compressed_world

        self.assertEqual(expected, request.connection.written_data()) 

def compress_and_finish(self, bytes):
        compressed_bytes = self._compress.compress(bytes)
        compressed_bytes += self._compress.flush(zlib.Z_FINISH)
        self._logger.debug('Compress input %r', bytes)
        self._logger.debug('Compress result %r', compressed_bytes)
        return compressed_bytes 

def flush(self, lib_mode=FLUSH):
        self._buffer.flush()
        GzipFile.flush(self, lib_mode) 

def zip_compress(content_generator, level=7):
  """Reads chunks from |content_generator| and yields zip compressed chunks."""
  compressor = zlib.compressobj(level)
  for chunk in content_generator:
    compressed = compressor.compress(chunk)
    if compressed:
      yield compressed
  tail = compressor.flush(zlib.Z_FINISH)
  if tail:
    yield tail 

def read(self, size = None):
        assert size != 0

        if size is None:
            size = -1

        # Keep reading chunks until we have a bit of compressed data to return,
        # since returning an empty byte string would be interpreted as EOF.
        while not self.__buffer and self.__gzip:
            chunk = self.stream.read(size)

            self.__buffer += self.__gzip.compress(chunk)

            if not chunk or size < 0:
                # Read to EOF on underlying stream, so flush any remaining
                # compressed data and return whatever we have.  We'll return an
                # empty byte string as EOF ourselves on any subsequent calls.
                self.__buffer += self.__gzip.flush(zlib.Z_FINISH)
                self.__gzip = None

        if size > 0 and len(self.__buffer) > size:
            # This should be pretty rare since we're reading N bytes and then
            # *compressing* to fewer bytes.  It could happen in the rare case
            # of lots of data still stuck in the buffer from a previous call.
            compressed = self.__buffer[0:size]
            self.__buffer = self.__buffer[size:]
        else:
            compressed = self.__buffer
            self.__buffer = b''

        return compressed 

def flush(self, lib_mode=FLUSH):
        self._nltk_buffer.flush()
        GzipFile.flush(self, lib_mode)