Python lz4.block() Examples

def __init__(self, ts=None, block_size=None, back_window=0):
        """A time serie that is limited in size.

        Used to represent the full-resolution buffer of incoming raw
        datapoints associated with a metric.

        The maximum size of this time serie is expressed in a number of block
        size, called the back window.
        When the timeserie is truncated, a whole block is removed.

        You cannot set a value using a timestamp that is prior to the last
        timestamp minus this number of blocks. By default, a back window of 0
        does not allow you to go back in time prior to the current block being
        used.

        """
        super(BoundTimeSerie, self).__init__(ts)
        self.block_size = block_size
        self.back_window = back_window 

def unserialize(cls, data, block_size, back_window):
        uncompressed = lz4.block.decompress(data)
        nb_points = (
            len(uncompressed) // cls._SERIALIZATION_TIMESTAMP_VALUE_LEN
        )

        try:
            timestamps = numpy.frombuffer(uncompressed, dtype='<Q',
                                          count=nb_points)
            values = numpy.frombuffer(
                uncompressed, dtype='<d',
                offset=nb_points * cls._SERIALIZATION_TIMESTAMP_LEN)
        except ValueError:
            raise InvalidData

        return cls.from_data(
            numpy.cumsum(timestamps),
            values,
            block_size=block_size,
            back_window=back_window) 

def decompress(data: bytes, mode: Compression, size: int, inherit=None) -> bytes:
    if mode == Compression.INHERIT:
        mode = inherit
    if mode in (Compression.ON, Compression.LZJB):
        return bytes(lzjb.decompress(data, size))
    elif mode == Compression.LZ4 and lz4:
        length = struct.unpack('>I', data[:4])[0]
        data = data[4:length + 4]
        return lz4.block.decompress(struct.pack('<i', size) + data)
    elif mode.name.startswith('GZIP_'):
        data = zlib.decompress(data)
        return data[:size]
    elif mode == Compression.OFF:
        return data
    else:
        if mode == Compression.LZ4 and not lz4:
            logging.error("Got a block with lz4 compression, but don't have `lz4` available")
        raise ValueError(mode) 

def do_trace(self, line):
		"""t/trace (i/instruction | b/block | f/function | m/memory)
		Toggles tracing of instructions, blocks, functions, or memory."""
		if line.startswith('i'):
			self.flags ^= TRACE_INSTRUCTION
			print 'Instruction tracing', 'on' if self.flags & TRACE_INSTRUCTION else 'off'
		elif line.startswith('b'):
			self.flags ^= TRACE_BLOCK
			print 'Block tracing', 'on' if self.flags & TRACE_BLOCK else 'off'
		elif line.startswith('f'):
			self.flags ^= TRACE_FUNCTION
			print 'Function tracing', 'on' if self.flags & TRACE_FUNCTION else 'off'
		elif line.startswith('m'):
			self.flags ^= TRACE_MEMORY
			print 'Memory tracing', 'on' if self.flags & TRACE_MEMORY else 'off'
		else:
			print 'Unknown trace flag' 

def _compress(payload):
        # FIXME(jd) lz4 > 0.9.2 returns bytearray instead of bytes. But Cradox
        # does not accept bytearray but only bytes, so make sure that we have a
        # byte type returned.
        return memoryview(lz4.block.compress(payload)).tobytes() 

def first_block_timestamp(self):
        """Return the timestamp of the first block."""
        rounded = round_timestamp(self.timestamps[-1], self.block_size)
        return rounded - (self.block_size * self.back_window) 

def _truncate(self):
        """Truncate the timeserie."""
        if self.block_size is not None and len(self.ts) != 0:
            # Change that to remove the amount of block needed to have
            # the size <= max_size. A block is a number of "seconds" (a
            # timespan)
            self.ts = self[self.first_block_timestamp():] 

def _DecompressChunkData(self, chunk_data, data_len):
        '''Decompress an individual compressed chunk (tag=0x600D)'''
        uncompressed = b''
        if chunk_data[0:4] in [b'bv41', b'bv4-']:
            last_uncompressed = b''
            comp_start = 0 # bv** offset
            comp_header = chunk_data[comp_start:comp_start + 4]
            while (data_len > comp_start) and (comp_header != b'bv4$'):
                if comp_header == b'bv41':
                    uncompressed_size, compressed_size = struct.unpack('<II', chunk_data[comp_start + 4:comp_start + 12])
                    last_uncompressed = lz4.block.decompress(chunk_data[comp_start + 12: comp_start + 12 + compressed_size], uncompressed_size, dict=last_uncompressed)
                    comp_start += 12 + compressed_size
                    uncompressed += last_uncompressed
                elif comp_header == b'bv4-':
                    uncompressed_size = struct.unpack('<I', chunk_data[comp_start + 4:comp_start + 8])[0]
                    uncompressed += chunk_data[comp_start + 8:comp_start + 8 + uncompressed_size]
                    comp_start += 8 + uncompressed_size
                else:
                    logger.error('Unknown compression value {} @ 0x{:X} - {}'.format(comp_header.hex(), begin_pos + comp_start, comp_header))
                    break
                comp_header = chunk_data[comp_start:comp_start + 4]
        else:
            logger.error('Unknown compression type {}'.format(chunk_data[16:20].hex()))
        return uncompressed

    # TODO: move this into a TimesyncList class. 

def decompressLZ4(self, file):
        lz4_headers = [ b"mozLz40\0", b"mozLz40p\0", b"mozLz40o\0"]

        for header in lz4_headers:
          value = file.read(len(header))
          if value == header:
              return lz4.block.decompress(file.read())
          file.seek(0)
            
        

        return None 

def loadnso(self, fn, loadbase=0x7100000000, relocate=True):
		data = file(fn, 'rb').read()
		assert data[0:4] == 'NSO0'

		toff, tloc, tsize = struct.unpack('<III', data[0x10:0x1C])
		roff, rloc, rsize = struct.unpack('<III', data[0x20:0x2C])
		doff, dloc, dsize = struct.unpack('<III', data[0x30:0x3C])
		bsssize, = struct.unpack('<I', data[0x3C:0x40])

		text = lz4.block.decompress(data[toff:roff], uncompressed_size=tsize)
		rd = lz4.block.decompress(data[roff:doff], uncompressed_size=rsize)
		data = lz4.block.decompress(data[doff:], uncompressed_size=dsize)

		full = text
		if rloc >= len(full):
			full += '\0' * (rloc - len(full))
			full += rd
		else:
			full = full[:rloc] + rd
		if dloc >= len(full):
			full += '\0' * (dloc - len(full))
			full += data
		else:
			full = full[:dloc] + data

		self.map(loadbase, len(full) + bsssize)
		self.writemem(loadbase, full)
		defineAddressClass(fn.rsplit('/', 1)[-1].split('.', 1)[0].title(), loadbase, len(full))

		if relocate:
			return relocation.relocate(self, loadbase)