Python struct.iter_unpack() Examples

def load( self, mem_image ):

    # Iterate over the sections

    sections = mem_image.get_sections()
    for section in sections:

      # For .mngr2proc sections, copy section into mngr2proc src

      if section.name == ".mngr2proc":
        self.src.msgs.extend(Bits32(bits[0]) for bits in struct.iter_unpack("<I", section.data))

      # For .proc2mngr sections, copy section into proc2mngr_ref src

      elif section.name == ".proc2mngr":
        self.sink.msgs.extend(Bits32(bits[0]) for bits in struct.iter_unpack("<I", section.data))

      # For all other sections, simply copy them into the memory

      else:
        self.mem.write_mem( section.addr, section.data )

  #-----------------------------------------------------------------------
  # done
  #----------------------------------------------------------------------- 

def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 

def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 

def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 

def iter_unpack(raw):
        """Yield successive EVENT_SIZE chunks from message."""
        return struct.iter_unpack(EVENT_FORMAT, raw) 

def test_module_func(self):
        # Sanity check for the global struct.iter_unpack()
        it = struct.iter_unpack('>IB', bytes(range(1, 11)))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_length_hint(self):
        lh = operator.length_hint
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(lh(it), 3)
        next(it)
        self.assertEqual(lh(it), 2)
        next(it)
        self.assertEqual(lh(it), 1)
        next(it)
        self.assertEqual(lh(it), 0)
        self.assertRaises(StopIteration, next, it)
        self.assertEqual(lh(it), 0) 

def test_arbitrary_buffer(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 11))
        it = s.iter_unpack(memoryview(b))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def read_array_data(io, data_struct):
	struct_size = struct.calcsize(data_struct)
	data_struct = "<" + data_struct # force little endianness

	count = struct.unpack("<I", io.read(4))[0]
	data = io.read(count * struct_size)
	unpacked_data = list(struct.iter_unpack(data_struct, data))
	return [tup[0] if len(tup) == 1 else tup for tup in unpacked_data ] 

def decode_chunk_into(chunk, buf, block_size):
    num_channels = chunk.shape[0]
    # Grid size (number of blocks in the chunk)
    gx = ceil_div(chunk.shape[3], block_size[0])
    gy = ceil_div(chunk.shape[2], block_size[1])
    gz = ceil_div(chunk.shape[1], block_size[2])

    if len(buf) < num_channels * (4 + 8 * gx * gy * gz):
        raise InvalidFormatError("compressed_segmentation file too short")

    if sys.version_info < (3,):
        channel_offsets = struct.unpack("<I", buf[:4*num_channels])
        channel_offsets = [ 4 * ret for ret in channel_offsets ]
    else:
        channel_offsets = [
            4 * ret[0] for ret in struct.iter_unpack("<I", buf[:4*num_channels])
        ]

    for channel, (offset, next_offset) in \
        enumerate(zip_longest(channel_offsets, channel_offsets[1:])):

        # next_offset will be None for the last channel
        if offset + 8 * gx * gy * gz > len(buf):
            raise InvalidFormatError("compressed_segmentation channel offset "
                                     "is too large (truncated file?)")
        _decode_channel_into(
            chunk, channel, buf[offset:next_offset], block_size
        )

    return chunk 

def test_module_func(self):
        # Sanity check for the global struct.iter_unpack()
        it = struct.iter_unpack('>IB', bytes(range(1, 11)))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_length_hint(self):
        lh = operator.length_hint
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(lh(it), 3)
        next(it)
        self.assertEqual(lh(it), 2)
        next(it)
        self.assertEqual(lh(it), 1)
        next(it)
        self.assertEqual(lh(it), 0)
        self.assertRaises(StopIteration, next, it)
        self.assertEqual(lh(it), 0) 

def test_arbitrary_buffer(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 11))
        it = s.iter_unpack(memoryview(b))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def load(cls, data: bytes):
        if not data:
            return cls()
        if len(data) % 2 != 0:
            raise ValueError('invalid data, length mismatch')
        month_id_base = struct.unpack('>H', data[:2])[0]
        items = []
        for offset, count in struct.iter_unpack('>2B', data[2:]):
            year, month = month_of_id(month_id_base + offset)
            items.append((year, month, count))
        return cls(items) 

def test_module_func(self):
        # Sanity check for the global struct.iter_unpack()
        it = struct.iter_unpack('>IB', bytes(range(1, 11)))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_length_hint(self):
        lh = operator.length_hint
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(lh(it), 3)
        next(it)
        self.assertEqual(lh(it), 2)
        next(it)
        self.assertEqual(lh(it), 1)
        next(it)
        self.assertEqual(lh(it), 0)
        self.assertRaises(StopIteration, next, it)
        self.assertEqual(lh(it), 0) 

def test_arbitrary_buffer(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 11))
        it = s.iter_unpack(memoryview(b))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def parse_dict(data):
  dct = {}
  mode = 'pad'
  key = []
  value = []

  for c in struct.iter_unpack('c', data):
    c = c[0]

    if mode == 'pad':
      if c in (bytes([0]), bytes([3])):
        continue
      else:
        mode = 'key'

    if mode == 'key':
      if c == bytes([0]):
        mode = 'value'
      else:
        key.append(c.decode())

    elif mode == 'value':
      if c == bytes([0]):
        dct[''.join(key)] = ''.join(value)
        key = []
        value = []
        mode = 'pad'
      else:
        value.append(c.decode())

  return dct 

def _do_iter(self):
        read_size = self._get_total_read_size()
        data = self._get_data(read_size)
        if not data:
            return None
        evdev_objects = iter_unpack(data)
        events = [self._make_event(*event) for event in evdev_objects]
        return events

    # pylint: disable=too-many-arguments 

def iter_unpack(raw):
        """Yield successive EVENT_SIZE chunks from message."""
        return chunks(raw) 

def write_registers(self, packet_data):
        addr_width = self.vmi.get_address_width()
        if addr_width == 4:
            pack_fmt = '@I'
        else:
            pack_fmt = '@Q'

        # for some reason,  GDB has a different parsing for gen registers
        # between 32 and 64 bits
        if addr_width == 4:
            gen_regs_x86 = [
                X86Reg.RAX, X86Reg.RCX, X86Reg.RDX, X86Reg.RBX,
                X86Reg.RSP, X86Reg.RBP, X86Reg.RSI, X86Reg.RDI
            ]
        else:
            gen_regs_x86 = [
                X86Reg.RAX, X86Reg.RBX, X86Reg.RCX, X86Reg.RDX,
                X86Reg.RSI, X86Reg.RDI, X86Reg.RBP, X86Reg.RSP
            ]

        gen_regs_x64 = [
            X86Reg.R8, X86Reg.R9, X86Reg.R10, X86Reg.R11, X86Reg.R12,
            X86Reg.R13, X86Reg.R14, X86Reg.R15
        ]

        # TODO parse the entire buffer
        # regs = Registers()
        regs = self.vmi.get_vcpuregs(0)
        iter = struct.iter_unpack(pack_fmt, unhexlify(packet_data))
        for r in gen_regs_x86:
            value, *rest = next(iter)
            logging.debug('%s: %x', r.name, value)
            regs[r] = value
        # 64 bits ?
        if addr_width == 8:
            for r in gen_regs_x64:
                value, *rest = next(iter)
                logging.debug('%s: %x', r.name, value)
                regs[r] = value
        # RIP ?
        value, *rest = next(iter)
        regs[X86Reg.RIP] = value
        # eflags
        value, *rest = next(iter)
        regs[X86Reg.RFLAGS] = value
        # TODO segment registers
        try:
            self.vmi.set_vcpuregs(regs, 0)
        except LibvmiError:
            return False
        else:
            self.send_packet(GDBPacket(b'OK'))
            return True