Python snappy.compress() Examples

The following are 26 code examples of snappy.compress(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module snappy , or try the search function .
Example #1
Source File: compression_support.py    From learn_python3_spider with MIT License 5 votes vote down vote up
def _zlib_no_compress(data):
    """Compress data with zlib level 0."""
    cobj = zlib.compressobj(0)
    return b"".join([cobj.compress(data), cobj.flush()]) 
Example #2
Source File: jrpc_py.py    From DataApi with Apache License 2.0 5 votes vote down vote up
def _pack_msgpack_snappy(obj):
    # print "pack", obj
    tmp = msgpack.dumps(obj, encoding='utf-8')
    if len(tmp) > 1000:
        return b'S' + snappy.compress(tmp)
    else:
        return b'\0' + tmp 
Example #3
Source File: commands.py    From trinity with MIT License 5 votes vote down vote up
def encode(self, cmd_id: int, snappy_support: bool) -> MessageAPI:
        raw_payload_data = self.serialization_codec.encode(self.payload)

        if snappy_support:
            payload_data = self.compression_codec.compress(raw_payload_data)
        else:
            payload_data = raw_payload_data

        cmd_id_data = rlp.encode(cmd_id, sedes=rlp.sedes.big_endian_int)
        frame_size = len(cmd_id_data) + len(payload_data)

        if frame_size.bit_length() > 24:
            raise ValueError("Frame size has to fit in a 3-byte integer")

        # Frame-size is a 3-bit integer
        header = frame_size.to_bytes(3, 'big') + RLPX_HEADER_DATA
        body = cmd_id_data + payload_data

        return Message(header, body) 
Example #4
Source File: commands.py    From trinity with MIT License 5 votes vote down vote up
def compress(self, data: bytes) -> bytes:
        return data 
Example #5
Source File: commands.py    From trinity with MIT License 5 votes vote down vote up
def compress(self, data: bytes) -> bytes:
        return snappy.compress(data) 
Example #6
Source File: gossiper.py    From trinity with MIT License 5 votes vote down vote up
def _serialize_gossip(operation: ssz.Serializable) -> bytes:
    encoded_operation = ssz.encode(operation)
    return snappy.compress(encoded_operation) 
Example #7
Source File: jrpc_py.py    From TradeApi with Apache License 2.0 5 votes vote down vote up
def _pack_msgpack_snappy(obj):
    # print "pack", obj
    tmp = msgpack.dumps(obj, encoding='utf-8')
    if len(tmp) > 1000:
        return b'S' + snappy.compress(tmp)
    else:
        return b'\0' + tmp 
Example #8
Source File: base.py    From lahja with MIT License 5 votes vote down vote up
def _compress_event(self, event: BaseEvent) -> Union[BaseEvent, bytes]:
        if self.has_snappy_support:
            import snappy

            return cast(
                bytes,
                snappy.compress(pickle.dumps(event, protocol=pickle.HIGHEST_PROTOCOL)),
            )
        else:
            return event 
Example #9
Source File: snappy.py    From kiel with Apache License 2.0 5 votes vote down vote up
def compress(data):
    """
    Compresses given data via the snappy algorithm.

    The result is preceded with a header containing the string 'SNAPPY' and the
    default and min-compat versions (both ``1``).

    The block size for the compression is hard-coded at 32kb.

    If ``python-snappy`` is not installed a ``RuntimeError`` is raised.
    """
    if not snappy_available:
        raise RuntimeError("Snappy compression unavailable.")

    buff = BytesIO()
    buff.write(raw_header)

    for block_num in range(0, len(data), BLOCK_SIZE):
        block = data[block_num:block_num + BLOCK_SIZE]
        compressed = snappy.compress(block)

        buff.write(struct.pack("!i", len(compressed)))
        buff.write(compressed)

    result = buff.getvalue()

    buff.close()

    return result 
Example #10
Source File: compression_support.py    From learn_python3_spider with MIT License 5 votes vote down vote up
def compress(data):
        # ZstdCompressor is not thread safe.
        # TODO: Use a pool?
        return ZstdCompressor().compress(data) 
Example #11
Source File: compression_support.py    From learn_python3_spider with MIT License 5 votes vote down vote up
def __init__(self, level):
        # Jython zlib.compress doesn't support -1
        if level == -1:
            self.compress = zlib.compress
        # Jython zlib.compress also doesn't support 0
        elif level == 0:
            self.compress = _zlib_no_compress
        else:
            self.compress = lambda data: zlib.compress(data, level) 
Example #12
Source File: compression_support.py    From learn_python3_spider with MIT License 5 votes vote down vote up
def compress(data):
        return snappy.compress(data) 
Example #13
Source File: jrpc_py.py    From TradeSim with Apache License 2.0 5 votes vote down vote up
def _pack_msgpack_snappy(obj):
    # print "pack", obj
    tmp = msgpack.dumps(obj, encoding='utf-8')
    if len(tmp) > 1000:
        return b'S' + snappy.compress(tmp)
    else:
        return b'\0' + tmp 
Example #14
Source File: serializer.py    From intake with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def compress(self, data):
            return snappy.compress(data) 
Example #15
Source File: serializer.py    From intake with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def encode(self, obj, container):
        return self._compressor.compress(
            self._format_encoder.encode(obj, container)) 
Example #16
Source File: serializer.py    From intake with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def compress(self, data):
        buf = io.BytesIO()
        with gzip.GzipFile(fileobj=buf, mode='wb', compresslevel=1) as f:
            f.write(data)
        return buf.getvalue() 
Example #17
Source File: serializer.py    From intake with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def compress(self, data):
        return data 
Example #18
Source File: compression_support.py    From vnpy_crypto with MIT License 5 votes vote down vote up
def __init__(self, level):
        # Jython zlib.compress doesn't support -1
        if level == -1:
            self.compress = zlib.compress
        # Jython zlib.compress also doesn't support 0
        elif level == 0:
            self.compress = _zlib_no_compress
        else:
            self.compress = lambda data: zlib.compress(data, level) 
Example #19
Source File: compression_support.py    From vnpy_crypto with MIT License 5 votes vote down vote up
def compress(data):
        return snappy.compress(data) 
Example #20
Source File: compression_support.py    From vnpy_crypto with MIT License 5 votes vote down vote up
def _zlib_no_compress(data):
    """Compress data with zlib level 0."""
    cobj = zlib.compressobj(0)
    return b"".join([cobj.compress(data), cobj.flush()]) 
Example #21
Source File: buffers.py    From purerpc with Apache License 2.0 5 votes vote down vote up
def write_message(self, data: bytes, compress=False):
        if compress:
            data = self.compress(data)
        if len(data) > self._max_message_length:
            raise MessageTooLargeError(
                "Trying to send message larger than max: {message_length} > {max_message_length}".format(
                    message_length=len(data),
                    max_message_length=self._max_message_length,
                )
            )
        self._buffer.append(struct.pack('>?I', compress, len(data)))
        self._buffer.append(data) 
Example #22
Source File: buffers.py    From purerpc with Apache License 2.0 5 votes vote down vote up
def compress(self, data):
        if self._message_encoding == "gzip" or self._message_encoding == "deflate":
            import zlib
            return zlib.compress(data)
        elif self._message_encoding == "snappy":
            import snappy
            return snappy.compress(data)
        else:
            raise UnsupportedMessageEncodingError(
                "Unsupported compression: {}".format(self._message_encoding)) 
Example #23
Source File: aff4_image.py    From pyaff4 with Apache License 2.0 5 votes vote down vote up
def FlushChunk(self, chunk):
        if len(chunk) == 0:
            return

        bevy_offset = self.bevy_length

        if self.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB:
            compressed_chunk = zlib.compress(chunk)
        elif (snappy and self.compression ==
              lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY):
            compressed_chunk = snappy.compress(chunk)
        elif self.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED:
            compressed_chunk = chunk

        compressedLen = len(compressed_chunk)

        if compressedLen < self.chunk_size - 16:
            self.bevy_index.append((bevy_offset, compressedLen))
            self.bevy.append(compressed_chunk)
            self.bevy_length += compressedLen
        else:
            self.bevy_index.append((bevy_offset, self.chunk_size))
            self.bevy.append(chunk)
            self.bevy_length += self.chunk_size

        #self.bevy_index.append((bevy_offset, len(compressed_chunk)))
        #self.bevy.append(compressed_chunk)
        #self.bevy_length += len(compressed_chunk)
        self.chunk_count_in_bevy += 1

        #self.buffer = chunk[self.chunk_size:]
        if self.chunk_count_in_bevy >= self.chunks_per_segment:
            self._FlushBevy() 
Example #24
Source File: aff4_image.py    From pyaff4 with Apache License 2.0 4 votes vote down vote up
def read(self, _):
        # Stop copying when the bevy is full.
        if self.chunk_count_in_bevy >= self.owner.chunks_per_segment:
            return ""

        chunk = self.stream.read(self.owner.chunk_size)
        if not chunk:
            return ""

        self.size += len(chunk)

        if self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB:
            compressed_chunk = zlib.compress(chunk)
        elif (snappy and self.owner.compression ==
              lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY):
            compressed_chunk = snappy.compress(chunk)
        elif self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED:
            compressed_chunk = chunk

        compressedLen = len(compressed_chunk)
        self.chunk_count_in_bevy += 1

        if compressedLen < self.owner.chunk_size - 16:
            self.bevy_index.append((self.bevy_length, compressedLen))
            self.bevy_length += compressedLen
            return compressed_chunk
        else:
            self.bevy_index.append((self.bevy_length, self.owner.chunk_size))
            self.bevy_length += self.owner.chunk_size
            return chunk 
Example #25
Source File: datafile.py    From spavro with Apache License 2.0 4 votes vote down vote up
def _write_block(self):
        if not self._header_written:
            self._write_header()

        if self.block_count > 0:
            # write number of items in block
            self.encoder.write_long(self.block_count)

            # write block contents
            uncompressed_data = self.buffer_writer.getvalue()
            if self.get_meta(CODEC_KEY) == 'null':
                compressed_data = uncompressed_data
                compressed_data_length = len(compressed_data)
            elif self.get_meta(CODEC_KEY) == 'deflate':
                # The first two characters and last character are zlib
                # wrappers around deflate data.
                compressed_data = zlib.compress(uncompressed_data)[2:-1]
                compressed_data_length = len(compressed_data)
            elif self.get_meta(CODEC_KEY) == 'snappy':
                compressed_data = snappy.compress(uncompressed_data)
                compressed_data_length = len(compressed_data) + 4 # crc32
            elif self.get_meta(CODEC_KEY) == 'xz':
                compressed_data = lzma.compress(uncompressed_data, format=lzma.FORMAT_XZ)
                compressed_data_length = len(compressed_data)
            else:
                fail_msg = '"%s" codec is not supported.' % self.get_meta(CODEC_KEY)
                raise DataFileException(fail_msg)

            # Write length of block
            self.encoder.write_long(compressed_data_length)

            # Write block
            self.writer.write(compressed_data)
            
            # Write CRC32 checksum for Snappy
            if self.get_meta(CODEC_KEY) == 'snappy':
                self.encoder.write_crc32(uncompressed_data)

            # write sync marker
            self.writer.write(self.sync_marker)

            # reset buffer
            self.buffer_writer.truncate(0)
            self.buffer_writer.seek(0)
            self.block_count = 0 
Example #26
Source File: codec.py    From afkak with Apache License 2.0 4 votes vote down vote up
def snappy_encode(payload, xerial_compatible=False,
                  xerial_blocksize=32 * 1024):
    """
    Compress the given data with the Snappy algorithm.

    :param bytes payload: Data to compress.
    :param bool xerial_compatible:
        If set then the stream is broken into length-prefixed blocks in
        a fashion compatible with the xerial snappy library.

        The format winds up being::

            +-------------+------------+--------------+------------+--------------+
            |   Header    | Block1_len | Block1 data  | BlockN len | BlockN data  |
            |-------------+------------+--------------+------------+--------------|
            |  16 bytes   |  BE int32  | snappy bytes |  BE int32  | snappy bytes |
            +-------------+------------+--------------+------------+--------------+

    :param int xerial_blocksize:
        Number of bytes per chunk to independently Snappy encode. 32k is the
        default in the xerial library.

    :returns: Compressed bytes.
    :rtype: :class:`bytes`
    """
    if not has_snappy():  # FIXME This should be static, not checked every call.
        raise NotImplementedError("Snappy codec is not available")

    if xerial_compatible:
        def _chunker():
            for i in range(0, len(payload), xerial_blocksize):
                yield payload[i:i+xerial_blocksize]

        out = BytesIO()
        out.write(_XERIAL_HEADER)

        for chunk in _chunker():
            block = snappy.compress(chunk)
            out.write(struct.pack('!i', len(block)))
            out.write(block)

        out.seek(0)
        return out.read()

    else:
        return snappy.compress(payload)