Python snappy.compress() Examples

def _zlib_no_compress(data):
    """Compress data with zlib level 0."""
    cobj = zlib.compressobj(0)
    return b"".join([cobj.compress(data), cobj.flush()]) 

def _pack_msgpack_snappy(obj):
    # print "pack", obj
    tmp = msgpack.dumps(obj, encoding='utf-8')
    if len(tmp) > 1000:
        return b'S' + snappy.compress(tmp)
    else:
        return b'\0' + tmp 

def encode(self, cmd_id: int, snappy_support: bool) -> MessageAPI:
        raw_payload_data = self.serialization_codec.encode(self.payload)

        if snappy_support:
            payload_data = self.compression_codec.compress(raw_payload_data)
        else:
            payload_data = raw_payload_data

        cmd_id_data = rlp.encode(cmd_id, sedes=rlp.sedes.big_endian_int)
        frame_size = len(cmd_id_data) + len(payload_data)

        if frame_size.bit_length() > 24:
            raise ValueError("Frame size has to fit in a 3-byte integer")

        # Frame-size is a 3-bit integer
        header = frame_size.to_bytes(3, 'big') + RLPX_HEADER_DATA
        body = cmd_id_data + payload_data

        return Message(header, body) 

def compress(self, data: bytes) -> bytes:
        return data 

def compress(self, data: bytes) -> bytes:
        return snappy.compress(data) 

def _serialize_gossip(operation: ssz.Serializable) -> bytes:
    encoded_operation = ssz.encode(operation)
    return snappy.compress(encoded_operation) 

def _pack_msgpack_snappy(obj):
    # print "pack", obj
    tmp = msgpack.dumps(obj, encoding='utf-8')
    if len(tmp) > 1000:
        return b'S' + snappy.compress(tmp)
    else:
        return b'\0' + tmp 

def _compress_event(self, event: BaseEvent) -> Union[BaseEvent, bytes]:
        if self.has_snappy_support:
            import snappy

            return cast(
                bytes,
                snappy.compress(pickle.dumps(event, protocol=pickle.HIGHEST_PROTOCOL)),
            )
        else:
            return event 

def compress(data):
    """
    Compresses given data via the snappy algorithm.

    The result is preceded with a header containing the string 'SNAPPY' and the
    default and min-compat versions (both ``1``).

    The block size for the compression is hard-coded at 32kb.

    If ``python-snappy`` is not installed a ``RuntimeError`` is raised.
    """
    if not snappy_available:
        raise RuntimeError("Snappy compression unavailable.")

    buff = BytesIO()
    buff.write(raw_header)

    for block_num in range(0, len(data), BLOCK_SIZE):
        block = data[block_num:block_num + BLOCK_SIZE]
        compressed = snappy.compress(block)

        buff.write(struct.pack("!i", len(compressed)))
        buff.write(compressed)

    result = buff.getvalue()

    buff.close()

    return result 

def compress(data):
        # ZstdCompressor is not thread safe.
        # TODO: Use a pool?
        return ZstdCompressor().compress(data) 

def __init__(self, level):
        # Jython zlib.compress doesn't support -1
        if level == -1:
            self.compress = zlib.compress
        # Jython zlib.compress also doesn't support 0
        elif level == 0:
            self.compress = _zlib_no_compress
        else:
            self.compress = lambda data: zlib.compress(data, level) 

def compress(data):
        return snappy.compress(data) 

def _pack_msgpack_snappy(obj):
    # print "pack", obj
    tmp = msgpack.dumps(obj, encoding='utf-8')
    if len(tmp) > 1000:
        return b'S' + snappy.compress(tmp)
    else:
        return b'\0' + tmp 

def compress(self, data):
            return snappy.compress(data) 

def encode(self, obj, container):
        return self._compressor.compress(
            self._format_encoder.encode(obj, container)) 

def compress(self, data):
        buf = io.BytesIO()
        with gzip.GzipFile(fileobj=buf, mode='wb', compresslevel=1) as f:
            f.write(data)
        return buf.getvalue() 

def compress(self, data):
        return data 

def __init__(self, level):
        # Jython zlib.compress doesn't support -1
        if level == -1:
            self.compress = zlib.compress
        # Jython zlib.compress also doesn't support 0
        elif level == 0:
            self.compress = _zlib_no_compress
        else:
            self.compress = lambda data: zlib.compress(data, level) 

def compress(data):
        return snappy.compress(data) 

def _zlib_no_compress(data):
    """Compress data with zlib level 0."""
    cobj = zlib.compressobj(0)
    return b"".join([cobj.compress(data), cobj.flush()]) 

def write_message(self, data: bytes, compress=False):
        if compress:
            data = self.compress(data)
        if len(data) > self._max_message_length:
            raise MessageTooLargeError(
                "Trying to send message larger than max: {message_length} > {max_message_length}".format(
                    message_length=len(data),
                    max_message_length=self._max_message_length,
                )
            )
        self._buffer.append(struct.pack('>?I', compress, len(data)))
        self._buffer.append(data) 

def compress(self, data):
        if self._message_encoding == "gzip" or self._message_encoding == "deflate":
            import zlib
            return zlib.compress(data)
        elif self._message_encoding == "snappy":
            import snappy
            return snappy.compress(data)
        else:
            raise UnsupportedMessageEncodingError(
                "Unsupported compression: {}".format(self._message_encoding)) 

def FlushChunk(self, chunk):
        if len(chunk) == 0:
            return

        bevy_offset = self.bevy_length

        if self.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB:
            compressed_chunk = zlib.compress(chunk)
        elif (snappy and self.compression ==
              lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY):
            compressed_chunk = snappy.compress(chunk)
        elif self.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED:
            compressed_chunk = chunk

        compressedLen = len(compressed_chunk)

        if compressedLen < self.chunk_size - 16:
            self.bevy_index.append((bevy_offset, compressedLen))
            self.bevy.append(compressed_chunk)
            self.bevy_length += compressedLen
        else:
            self.bevy_index.append((bevy_offset, self.chunk_size))
            self.bevy.append(chunk)
            self.bevy_length += self.chunk_size

        #self.bevy_index.append((bevy_offset, len(compressed_chunk)))
        #self.bevy.append(compressed_chunk)
        #self.bevy_length += len(compressed_chunk)
        self.chunk_count_in_bevy += 1

        #self.buffer = chunk[self.chunk_size:]
        if self.chunk_count_in_bevy >= self.chunks_per_segment:
            self._FlushBevy() 

def read(self, _):
        # Stop copying when the bevy is full.
        if self.chunk_count_in_bevy >= self.owner.chunks_per_segment:
            return ""

        chunk = self.stream.read(self.owner.chunk_size)
        if not chunk:
            return ""

        self.size += len(chunk)

        if self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB:
            compressed_chunk = zlib.compress(chunk)
        elif (snappy and self.owner.compression ==
              lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY):
            compressed_chunk = snappy.compress(chunk)
        elif self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED:
            compressed_chunk = chunk

        compressedLen = len(compressed_chunk)
        self.chunk_count_in_bevy += 1

        if compressedLen < self.owner.chunk_size - 16:
            self.bevy_index.append((self.bevy_length, compressedLen))
            self.bevy_length += compressedLen
            return compressed_chunk
        else:
            self.bevy_index.append((self.bevy_length, self.owner.chunk_size))
            self.bevy_length += self.owner.chunk_size
            return chunk 

def _write_block(self):
        if not self._header_written:
            self._write_header()

        if self.block_count > 0:
            # write number of items in block
            self.encoder.write_long(self.block_count)

            # write block contents
            uncompressed_data = self.buffer_writer.getvalue()
            if self.get_meta(CODEC_KEY) == 'null':
                compressed_data = uncompressed_data
                compressed_data_length = len(compressed_data)
            elif self.get_meta(CODEC_KEY) == 'deflate':
                # The first two characters and last character are zlib
                # wrappers around deflate data.
                compressed_data = zlib.compress(uncompressed_data)[2:-1]
                compressed_data_length = len(compressed_data)
            elif self.get_meta(CODEC_KEY) == 'snappy':
                compressed_data = snappy.compress(uncompressed_data)
                compressed_data_length = len(compressed_data) + 4 # crc32
            elif self.get_meta(CODEC_KEY) == 'xz':
                compressed_data = lzma.compress(uncompressed_data, format=lzma.FORMAT_XZ)
                compressed_data_length = len(compressed_data)
            else:
                fail_msg = '"%s" codec is not supported.' % self.get_meta(CODEC_KEY)
                raise DataFileException(fail_msg)

            # Write length of block
            self.encoder.write_long(compressed_data_length)

            # Write block
            self.writer.write(compressed_data)
            
            # Write CRC32 checksum for Snappy
            if self.get_meta(CODEC_KEY) == 'snappy':
                self.encoder.write_crc32(uncompressed_data)

            # write sync marker
            self.writer.write(self.sync_marker)

            # reset buffer
            self.buffer_writer.truncate(0)
            self.buffer_writer.seek(0)
            self.block_count = 0 

def snappy_encode(payload, xerial_compatible=False,
                  xerial_blocksize=32 * 1024):
    """
    Compress the given data with the Snappy algorithm.

    :param bytes payload: Data to compress.
    :param bool xerial_compatible:
        If set then the stream is broken into length-prefixed blocks in
        a fashion compatible with the xerial snappy library.

        The format winds up being::

            +-------------+------------+--------------+------------+--------------+
            |   Header    | Block1_len | Block1 data  | BlockN len | BlockN data  |
            |-------------+------------+--------------+------------+--------------|
            |  16 bytes   |  BE int32  | snappy bytes |  BE int32  | snappy bytes |
            +-------------+------------+--------------+------------+--------------+

    :param int xerial_blocksize:
        Number of bytes per chunk to independently Snappy encode. 32k is the
        default in the xerial library.

    :returns: Compressed bytes.
    :rtype: :class:`bytes`
    """
    if not has_snappy():  # FIXME This should be static, not checked every call.
        raise NotImplementedError("Snappy codec is not available")

    if xerial_compatible:
        def _chunker():
            for i in range(0, len(payload), xerial_blocksize):
                yield payload[i:i+xerial_blocksize]

        out = BytesIO()
        out.write(_XERIAL_HEADER)

        for chunk in _chunker():
            block = snappy.compress(chunk)
            out.write(struct.pack('!i', len(block)))
            out.write(block)

        out.seek(0)
        return out.read()

    else:
        return snappy.compress(payload)