Python snappy.compress() Examples
The following are 26
code examples of snappy.compress().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
snappy
, or try the search function
.
Example #1
Source File: compression_support.py From learn_python3_spider with MIT License | 5 votes |
def _zlib_no_compress(data): """Compress data with zlib level 0.""" cobj = zlib.compressobj(0) return b"".join([cobj.compress(data), cobj.flush()])
Example #2
Source File: jrpc_py.py From DataApi with Apache License 2.0 | 5 votes |
def _pack_msgpack_snappy(obj): # print "pack", obj tmp = msgpack.dumps(obj, encoding='utf-8') if len(tmp) > 1000: return b'S' + snappy.compress(tmp) else: return b'\0' + tmp
Example #3
Source File: commands.py From trinity with MIT License | 5 votes |
def encode(self, cmd_id: int, snappy_support: bool) -> MessageAPI: raw_payload_data = self.serialization_codec.encode(self.payload) if snappy_support: payload_data = self.compression_codec.compress(raw_payload_data) else: payload_data = raw_payload_data cmd_id_data = rlp.encode(cmd_id, sedes=rlp.sedes.big_endian_int) frame_size = len(cmd_id_data) + len(payload_data) if frame_size.bit_length() > 24: raise ValueError("Frame size has to fit in a 3-byte integer") # Frame-size is a 3-bit integer header = frame_size.to_bytes(3, 'big') + RLPX_HEADER_DATA body = cmd_id_data + payload_data return Message(header, body)
Example #4
Source File: commands.py From trinity with MIT License | 5 votes |
def compress(self, data: bytes) -> bytes: return data
Example #5
Source File: commands.py From trinity with MIT License | 5 votes |
def compress(self, data: bytes) -> bytes: return snappy.compress(data)
Example #6
Source File: gossiper.py From trinity with MIT License | 5 votes |
def _serialize_gossip(operation: ssz.Serializable) -> bytes: encoded_operation = ssz.encode(operation) return snappy.compress(encoded_operation)
Example #7
Source File: jrpc_py.py From TradeApi with Apache License 2.0 | 5 votes |
def _pack_msgpack_snappy(obj): # print "pack", obj tmp = msgpack.dumps(obj, encoding='utf-8') if len(tmp) > 1000: return b'S' + snappy.compress(tmp) else: return b'\0' + tmp
Example #8
Source File: base.py From lahja with MIT License | 5 votes |
def _compress_event(self, event: BaseEvent) -> Union[BaseEvent, bytes]: if self.has_snappy_support: import snappy return cast( bytes, snappy.compress(pickle.dumps(event, protocol=pickle.HIGHEST_PROTOCOL)), ) else: return event
Example #9
Source File: snappy.py From kiel with Apache License 2.0 | 5 votes |
def compress(data): """ Compresses given data via the snappy algorithm. The result is preceded with a header containing the string 'SNAPPY' and the default and min-compat versions (both ``1``). The block size for the compression is hard-coded at 32kb. If ``python-snappy`` is not installed a ``RuntimeError`` is raised. """ if not snappy_available: raise RuntimeError("Snappy compression unavailable.") buff = BytesIO() buff.write(raw_header) for block_num in range(0, len(data), BLOCK_SIZE): block = data[block_num:block_num + BLOCK_SIZE] compressed = snappy.compress(block) buff.write(struct.pack("!i", len(compressed))) buff.write(compressed) result = buff.getvalue() buff.close() return result
Example #10
Source File: compression_support.py From learn_python3_spider with MIT License | 5 votes |
def compress(data): # ZstdCompressor is not thread safe. # TODO: Use a pool? return ZstdCompressor().compress(data)
Example #11
Source File: compression_support.py From learn_python3_spider with MIT License | 5 votes |
def __init__(self, level): # Jython zlib.compress doesn't support -1 if level == -1: self.compress = zlib.compress # Jython zlib.compress also doesn't support 0 elif level == 0: self.compress = _zlib_no_compress else: self.compress = lambda data: zlib.compress(data, level)
Example #12
Source File: compression_support.py From learn_python3_spider with MIT License | 5 votes |
def compress(data): return snappy.compress(data)
Example #13
Source File: jrpc_py.py From TradeSim with Apache License 2.0 | 5 votes |
def _pack_msgpack_snappy(obj): # print "pack", obj tmp = msgpack.dumps(obj, encoding='utf-8') if len(tmp) > 1000: return b'S' + snappy.compress(tmp) else: return b'\0' + tmp
Example #14
Source File: serializer.py From intake with BSD 2-Clause "Simplified" License | 5 votes |
def compress(self, data): return snappy.compress(data)
Example #15
Source File: serializer.py From intake with BSD 2-Clause "Simplified" License | 5 votes |
def encode(self, obj, container): return self._compressor.compress( self._format_encoder.encode(obj, container))
Example #16
Source File: serializer.py From intake with BSD 2-Clause "Simplified" License | 5 votes |
def compress(self, data): buf = io.BytesIO() with gzip.GzipFile(fileobj=buf, mode='wb', compresslevel=1) as f: f.write(data) return buf.getvalue()
Example #17
Source File: serializer.py From intake with BSD 2-Clause "Simplified" License | 5 votes |
def compress(self, data): return data
Example #18
Source File: compression_support.py From vnpy_crypto with MIT License | 5 votes |
def __init__(self, level): # Jython zlib.compress doesn't support -1 if level == -1: self.compress = zlib.compress # Jython zlib.compress also doesn't support 0 elif level == 0: self.compress = _zlib_no_compress else: self.compress = lambda data: zlib.compress(data, level)
Example #19
Source File: compression_support.py From vnpy_crypto with MIT License | 5 votes |
def compress(data): return snappy.compress(data)
Example #20
Source File: compression_support.py From vnpy_crypto with MIT License | 5 votes |
def _zlib_no_compress(data): """Compress data with zlib level 0.""" cobj = zlib.compressobj(0) return b"".join([cobj.compress(data), cobj.flush()])
Example #21
Source File: buffers.py From purerpc with Apache License 2.0 | 5 votes |
def write_message(self, data: bytes, compress=False): if compress: data = self.compress(data) if len(data) > self._max_message_length: raise MessageTooLargeError( "Trying to send message larger than max: {message_length} > {max_message_length}".format( message_length=len(data), max_message_length=self._max_message_length, ) ) self._buffer.append(struct.pack('>?I', compress, len(data))) self._buffer.append(data)
Example #22
Source File: buffers.py From purerpc with Apache License 2.0 | 5 votes |
def compress(self, data): if self._message_encoding == "gzip" or self._message_encoding == "deflate": import zlib return zlib.compress(data) elif self._message_encoding == "snappy": import snappy return snappy.compress(data) else: raise UnsupportedMessageEncodingError( "Unsupported compression: {}".format(self._message_encoding))
Example #23
Source File: aff4_image.py From pyaff4 with Apache License 2.0 | 5 votes |
def FlushChunk(self, chunk): if len(chunk) == 0: return bevy_offset = self.bevy_length if self.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB: compressed_chunk = zlib.compress(chunk) elif (snappy and self.compression == lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY): compressed_chunk = snappy.compress(chunk) elif self.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED: compressed_chunk = chunk compressedLen = len(compressed_chunk) if compressedLen < self.chunk_size - 16: self.bevy_index.append((bevy_offset, compressedLen)) self.bevy.append(compressed_chunk) self.bevy_length += compressedLen else: self.bevy_index.append((bevy_offset, self.chunk_size)) self.bevy.append(chunk) self.bevy_length += self.chunk_size #self.bevy_index.append((bevy_offset, len(compressed_chunk))) #self.bevy.append(compressed_chunk) #self.bevy_length += len(compressed_chunk) self.chunk_count_in_bevy += 1 #self.buffer = chunk[self.chunk_size:] if self.chunk_count_in_bevy >= self.chunks_per_segment: self._FlushBevy()
Example #24
Source File: aff4_image.py From pyaff4 with Apache License 2.0 | 4 votes |
def read(self, _): # Stop copying when the bevy is full. if self.chunk_count_in_bevy >= self.owner.chunks_per_segment: return "" chunk = self.stream.read(self.owner.chunk_size) if not chunk: return "" self.size += len(chunk) if self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_ZLIB: compressed_chunk = zlib.compress(chunk) elif (snappy and self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY): compressed_chunk = snappy.compress(chunk) elif self.owner.compression == lexicon.AFF4_IMAGE_COMPRESSION_STORED: compressed_chunk = chunk compressedLen = len(compressed_chunk) self.chunk_count_in_bevy += 1 if compressedLen < self.owner.chunk_size - 16: self.bevy_index.append((self.bevy_length, compressedLen)) self.bevy_length += compressedLen return compressed_chunk else: self.bevy_index.append((self.bevy_length, self.owner.chunk_size)) self.bevy_length += self.owner.chunk_size return chunk
Example #25
Source File: datafile.py From spavro with Apache License 2.0 | 4 votes |
def _write_block(self): if not self._header_written: self._write_header() if self.block_count > 0: # write number of items in block self.encoder.write_long(self.block_count) # write block contents uncompressed_data = self.buffer_writer.getvalue() if self.get_meta(CODEC_KEY) == 'null': compressed_data = uncompressed_data compressed_data_length = len(compressed_data) elif self.get_meta(CODEC_KEY) == 'deflate': # The first two characters and last character are zlib # wrappers around deflate data. compressed_data = zlib.compress(uncompressed_data)[2:-1] compressed_data_length = len(compressed_data) elif self.get_meta(CODEC_KEY) == 'snappy': compressed_data = snappy.compress(uncompressed_data) compressed_data_length = len(compressed_data) + 4 # crc32 elif self.get_meta(CODEC_KEY) == 'xz': compressed_data = lzma.compress(uncompressed_data, format=lzma.FORMAT_XZ) compressed_data_length = len(compressed_data) else: fail_msg = '"%s" codec is not supported.' % self.get_meta(CODEC_KEY) raise DataFileException(fail_msg) # Write length of block self.encoder.write_long(compressed_data_length) # Write block self.writer.write(compressed_data) # Write CRC32 checksum for Snappy if self.get_meta(CODEC_KEY) == 'snappy': self.encoder.write_crc32(uncompressed_data) # write sync marker self.writer.write(self.sync_marker) # reset buffer self.buffer_writer.truncate(0) self.buffer_writer.seek(0) self.block_count = 0
Example #26
Source File: codec.py From afkak with Apache License 2.0 | 4 votes |
def snappy_encode(payload, xerial_compatible=False, xerial_blocksize=32 * 1024): """ Compress the given data with the Snappy algorithm. :param bytes payload: Data to compress. :param bool xerial_compatible: If set then the stream is broken into length-prefixed blocks in a fashion compatible with the xerial snappy library. The format winds up being:: +-------------+------------+--------------+------------+--------------+ | Header | Block1_len | Block1 data | BlockN len | BlockN data | |-------------+------------+--------------+------------+--------------| | 16 bytes | BE int32 | snappy bytes | BE int32 | snappy bytes | +-------------+------------+--------------+------------+--------------+ :param int xerial_blocksize: Number of bytes per chunk to independently Snappy encode. 32k is the default in the xerial library. :returns: Compressed bytes. :rtype: :class:`bytes` """ if not has_snappy(): # FIXME This should be static, not checked every call. raise NotImplementedError("Snappy codec is not available") if xerial_compatible: def _chunker(): for i in range(0, len(payload), xerial_blocksize): yield payload[i:i+xerial_blocksize] out = BytesIO() out.write(_XERIAL_HEADER) for chunk in _chunker(): block = snappy.compress(chunk) out.write(struct.pack('!i', len(block))) out.write(block) out.seek(0) return out.read() else: return snappy.compress(payload)