Python zlib.Z_BEST_COMPRESSION Examples

def marshall_key(self, key):
        """
        Marshalls a Crash key to be used in the database.

        @see: L{__init__}

        @type  key: L{Crash} key.
        @param key: Key to convert.

        @rtype:  str or buffer
        @return: Converted key.
        """
        if key in self.__keys:
            return self.__keys[key]
        skey = pickle.dumps(key, protocol = 0)
        if self.compressKeys:
            skey = zlib.compress(skey, zlib.Z_BEST_COMPRESSION)
        if self.escapeKeys:
            skey = skey.encode('hex')
        if self.binaryKeys:
            skey = buffer(skey)
        self.__keys[key] = skey
        return skey 

def gz_compress(driver, data):
        """
        Params:
            driver (PersistenceDriver)
            data (Bytes): Uncompressed data

        Return: Bytes
        """
        try:
            original_size = sys.getsizeof(data)
            data = zlib.compress(data, level=zlib.Z_BEST_COMPRESSION)
            LOGGER.debug(
                'LookupTable (%s): Successfully compressed input data from %d to %d bytes',
                driver.id,
                original_size,
                sys.getsizeof(data)
            )
            return data
        except zlib.error:
            LOGGER.exception('LookupTable (%s): Data compression error.', driver.id) 

def marshall_key(self, key):
        """
        Marshalls a Crash key to be used in the database.

        @see: L{__init__}

        @type  key: L{Crash} key.
        @param key: Key to convert.

        @rtype:  str or buffer
        @return: Converted key.
        """
        if key in self.__keys:
            return self.__keys[key]
        skey = pickle.dumps(key, protocol = 0)
        if self.compressKeys:
            skey = zlib.compress(skey, zlib.Z_BEST_COMPRESSION)
        if self.escapeKeys:
            skey = skey.encode('hex')
        if self.binaryKeys:
            skey = buffer(skey)
        self.__keys[key] = skey
        return skey 

def run(self):
        print("VEDIO client starts...")
        while True:
            try:
                self.sock.connect(self.ADDR)
                break
            except:
                time.sleep(3)
                continue
        print("VEDIO client connected...")
        while self.cap.isOpened():
            ret, frame = self.cap.read()
            sframe = cv2.resize(frame, (0,0), fx=self.fx, fy=self.fx)
            data = pickle.dumps(sframe)
            zdata = zlib.compress(data, zlib.Z_BEST_COMPRESSION)
            try:
                self.sock.sendall(struct.pack("L", len(zdata)) + zdata)
            except:
                break
            for i in range(self.interval):
                self.cap.read() 

def marshall_key(self, key):
        """
        Marshalls a Crash key to be used in the database.

        @see: L{__init__}

        @type  key: L{Crash} key.
        @param key: Key to convert.

        @rtype:  str or buffer
        @return: Converted key.
        """
        if key in self.__keys:
            return self.__keys[key]
        skey = pickle.dumps(key, protocol = 0)
        if self.compressKeys:
            skey = zlib.compress(skey, zlib.Z_BEST_COMPRESSION)
        if self.escapeKeys:
            skey = skey.encode('hex')
        if self.binaryKeys:
            skey = buffer(skey)
        self.__keys[key] = skey
        return skey 

def encode_mask(mask_to_encode):
  """Encodes a binary mask into the Kaggle challenge text format.

  The encoding is done in three stages:
   - COCO RLE-encoding,
   - zlib compression,
   - base64 encoding (to use as entry in csv file).

  Args:
    mask_to_encode: binary np.ndarray of dtype bool and 2d shape.

  Returns:
    A (base64) text string of the encoded mask.
  """
  mask_to_encode = np.squeeze(mask_to_encode)
  mask_to_encode = mask_to_encode.reshape(mask_to_encode.shape[0],
                                          mask_to_encode.shape[1], 1)
  mask_to_encode = mask_to_encode.astype(np.uint8)
  mask_to_encode = np.asfortranarray(mask_to_encode)
  encoded_mask = coco_mask.encode(mask_to_encode)[0]['counts']
  compressed_mask = zlib.compress(six.ensure_binary(encoded_mask),
                                  zlib.Z_BEST_COMPRESSION)
  base64_mask = base64.b64encode(compressed_mask)
  return base64_mask 

def run(self):
        while True:
            try:
                self.sock.connect(self.ADDR)
                break
            except:
                time.sleep(3)
                continue
        if self.showme:
            cv2.namedWindow('You', cv2.WINDOW_NORMAL)
        print("VEDIO client connected...")
        while self.cap.isOpened():
            ret, frame = self.cap.read()
            if self.showme:
                cv2.imshow('You', frame)
                if cv2.waitKey(1) & 0xFF == 27:
                    self.showme = False
                    cv2.destroyWindow('You')
            sframe = cv2.resize(frame, (0, 0), fx=self.fx, fy=self.fx)
            data = pickle.dumps(sframe)
            zdata = zlib.compress(data, zlib.Z_BEST_COMPRESSION)
            try:
                self.sock.sendall(struct.pack("L", len(zdata)) + zdata)
            except:
                break
            for i in range(self.interval):
                self.cap.read() 

def _upload_logs(dagster_log_dir, log_size, dagster_log_queue_dir):
    '''Send POST request to telemetry server with the contents of $DAGSTER_HOME/logs/ directory '''
    try:
        if log_size > 0:
            # Delete contents of dagster_log_queue_dir so that new logs can be copied over
            for f in os.listdir(dagster_log_queue_dir):
                # Todo: there is probably a way to try to upload these logs without introducing
                # too much complexity...
                os.remove(os.path.join(dagster_log_queue_dir, f))

            os.rename(dagster_log_dir, dagster_log_queue_dir)

        for curr_path in os.listdir(dagster_log_queue_dir):
            curr_full_path = os.path.join(dagster_log_queue_dir, curr_path)
            retry_num = 0
            max_retries = 3
            success = False

            while not success and retry_num <= max_retries:
                with open(curr_full_path, 'rb') as curr_file:
                    byte = curr_file.read()

                    data = zlib.compress(byte, zlib.Z_BEST_COMPRESSION)
                    headers = {'content-encoding': 'gzip'}
                    r = requests.post(DAGSTER_TELEMETRY_URL, data=data, headers=headers)
                    if r.status_code == 200:
                        success = True
                    retry_num += 1

            if success:
                os.remove(curr_full_path)

    except Exception:  # pylint: disable=broad-except
        pass 

def compress(data):
        """
        Compresses the given data with the DEFLATE algorithm. The first four bytes contain the length of the
        uncompressed data.

        :param data: uncompressed data
        :type data: bytes or str
        :return: compressed data
        :rtype: bytes
        """
        compress_object = zlib.compressobj(
            zlib.Z_BEST_COMPRESSION,
            zlib.DEFLATED,
            zlib.MAX_WBITS,
            zlib.DEF_MEM_LEVEL,
            zlib.Z_DEFAULT_STRATEGY)
        if type(data) == str:
            compressed_data = compress_object.compress(data.encode('utf-8'))
            compressed_data += compress_object.flush()
            return struct.pack('!I', len(data.encode('utf-8'))) + compressed_data
        elif type(data) == bytes:
            compressed_data = compress_object.compress(data)
            compressed_data += compress_object.flush()
            return struct.pack('!I', len(data)) + compressed_data
        else:
            raise TypeError("Please pass a str or bytes to the packer.") 

def run(self):
        print ("VEDIO client starts...")
        while True:
            try:
                self.sock.connect(self.ADDR)
                break
            except:
                time.sleep(3)
                continue
        print ("video client <-> remote server success connected...")
        check = "F"
        check = self.sock.recv(1)
        if check.decode("utf-8") != "S":
            return
        print ("receive authend")
        #self.cap = cv2.VideoCapture(0)
        self.cap = cv2.VideoCapture("test.mp4")
        if self.showme:
            cv2.namedWindow('You', cv2.WINDOW_NORMAL)
        print ("remote VEDIO client connected...")
        while self.cap.isOpened():
            ret, frame = self.cap.read()
            if self.showme:
                cv2.imshow('You', frame)
                if cv2.waitKey(1) & 0xFF == 27:
                    self.showme = False
                    cv2.destroyWindow('You')
            if self.level > 0:
                frame = cv2.resize(frame, (0,0), fx=self.fx, fy=self.fx)
            data = pickle.dumps(frame)
            zdata = zlib.compress(data, zlib.Z_BEST_COMPRESSION)
            try:
                self.sock.sendall(struct.pack("L", len(zdata)) + zdata)
                print("video send ", len(zdata))
            except:
                break
            for i in range(self.interval):
                self.cap.read() 

def run(self):
        while True:
            try:
                self.sock.connect(self.ADDR)
                break
            except:
                time.sleep(3)
                continue
        if self.showme:
            cv2.namedWindow('You', cv2.WINDOW_NORMAL)
        print("VEDIO client connected...")
        while self.cap.isOpened():
            ret, frame = self.cap.read()
            if self.showme:
                cv2.imshow('You', frame)
                if cv2.waitKey(1) & 0xFF == 27:
                    self.showme = False
                    cv2.destroyWindow('You')
            sframe = cv2.resize(frame, (0,0), fx=self.fx, fy=self.fx)
            data = pickle.dumps(sframe)
            zdata = zlib.compress(data, zlib.Z_BEST_COMPRESSION)
            try:
                self.sock.sendall(struct.pack("L", len(zdata)) + zdata)
            except:
                break
            for i in range(self.interval):
                self.cap.read() 

def __init__(self, stream: BinaryIO):
        if not stream.readable():
            raise ValueError('"stream" argument must be readable.')

        self.stream = stream
        self.__buffer = b''
        self.__gzip = zlib.compressobj(
            level = zlib.Z_BEST_COMPRESSION,
            wbits = 16 + zlib.MAX_WBITS,    # Offset of 16 is gzip encapsulation
            memLevel = 9,                   # Memory is ~cheap; use it for better compression
        ) 

def save(self):
        if not self.__packet_frame__:
            self.__packet_frame__ = self.handler.pack()
        self.__packet_frame__['__globalprog__'] = self.pack()

        packet = zlib.compress(str.encode(str(self.__packet_frame__)), zlib.Z_BEST_COMPRESSION)

        with open(os.path.join(self.handler.file.path, self.handler.file.name + '.nbdler'), 'wb') as f:
            f.write(packet)
            f.flush() 

def marshall_value(self, value, storeMemoryMap = False):
        """
        Marshalls a Crash object to be used in the database.
        By default the C{memoryMap} member is B{NOT} stored here.

        @warning: Setting the C{storeMemoryMap} argument to C{True} can lead to
            a severe performance penalty!

        @type  value: L{Crash}
        @param value: Object to convert.

        @type  storeMemoryMap: bool
        @param storeMemoryMap: C{True} to store the memory map, C{False}
            otherwise.

        @rtype:  str
        @return: Converted object.
        """
        if hasattr(value, 'memoryMap'):
            crash = value
            memoryMap = crash.memoryMap
            try:
                crash.memoryMap = None
                if storeMemoryMap and memoryMap is not None:
                    # convert the generator to a list
                    crash.memoryMap = list(memoryMap)
                if self.optimizeValues:
                    value = pickle.dumps(crash, protocol = HIGHEST_PROTOCOL)
                    value = optimize(value)
                else:
                    value = pickle.dumps(crash, protocol = 0)
            finally:
                crash.memoryMap = memoryMap
                del memoryMap
                del crash
        if self.compressValues:
            value = zlib.compress(value, zlib.Z_BEST_COMPRESSION)
        if self.escapeValues:
            value = value.encode('hex')
        if self.binaryValues:
            value = buffer(value)
        return value 

def marshall_value(self, value, storeMemoryMap = False):
        """
        Marshalls a Crash object to be used in the database.
        By default the C{memoryMap} member is B{NOT} stored here.

        @warning: Setting the C{storeMemoryMap} argument to C{True} can lead to
            a severe performance penalty!

        @type  value: L{Crash}
        @param value: Object to convert.

        @type  storeMemoryMap: bool
        @param storeMemoryMap: C{True} to store the memory map, C{False}
            otherwise.

        @rtype:  str
        @return: Converted object.
        """
        if hasattr(value, 'memoryMap'):
            crash = value
            memoryMap = crash.memoryMap
            try:
                crash.memoryMap = None
                if storeMemoryMap and memoryMap is not None:
                    # convert the generator to a list
                    crash.memoryMap = list(memoryMap)
                if self.optimizeValues:
                    value = pickle.dumps(crash, protocol = HIGHEST_PROTOCOL)
                    value = optimize(value)
                else:
                    value = pickle.dumps(crash, protocol = 0)
            finally:
                crash.memoryMap = memoryMap
                del memoryMap
                del crash
        if self.compressValues:
            value = zlib.compress(value, zlib.Z_BEST_COMPRESSION)
        if self.escapeValues:
            value = value.encode('hex')
        if self.binaryValues:
            value = buffer(value)
        return value 

def marshall_value(self, value, storeMemoryMap = False):
        """
        Marshalls a Crash object to be used in the database.
        By default the C{memoryMap} member is B{NOT} stored here.

        @warning: Setting the C{storeMemoryMap} argument to C{True} can lead to
            a severe performance penalty!

        @type  value: L{Crash}
        @param value: Object to convert.

        @type  storeMemoryMap: bool
        @param storeMemoryMap: C{True} to store the memory map, C{False}
            otherwise.

        @rtype:  str
        @return: Converted object.
        """
        if hasattr(value, 'memoryMap'):
            crash = value
            memoryMap = crash.memoryMap
            try:
                crash.memoryMap = None
                if storeMemoryMap and memoryMap is not None:
                    # convert the generator to a list
                    crash.memoryMap = list(memoryMap)
                if self.optimizeValues:
                    value = pickle.dumps(crash, protocol = HIGHEST_PROTOCOL)
                    value = optimize(value)
                else:
                    value = pickle.dumps(crash, protocol = 0)
            finally:
                crash.memoryMap = memoryMap
                del memoryMap
                del crash
        if self.compressValues:
            value = zlib.compress(value, zlib.Z_BEST_COMPRESSION)
        if self.escapeValues:
            value = value.encode('hex')
        if self.binaryValues:
            value = buffer(value)
        return value