Python pickle._Pickler() Examples
The following are 4
code examples of pickle._Pickler().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
pickle
, or try the search function
.
![](https://www.programcreek.com/common/static/images/search.png)
Example #1
Source File: runner.py From doit with MIT License | 5 votes |
def _run_start_processes(self, job_q, result_q): """create and start sub-processes @param job_q: (multiprocessing.Queue) tasks to be executed @param result_q: (multiprocessing.Queue) collect task results @return list of Process """ # #### DEBUG PICKLE ERRORS # class MyPickler (pickle._Pickler): # def save(self, obj): # print('pickling object {} of type {}'.format(obj, type(obj))) # try: # Pickler.save(self, obj) # except: # print('error. skipping...') # from io import BytesIO # pickler = MyPickler(BytesIO()) # pickler.dump(self) # ### END DEBUG proc_list = [] for _ in range(self.num_process): next_job = self.get_next_job(None) if next_job is None: break # do not start more processes than tasks job_q.put(next_job) process = self.Child( target=self.execute_task_subprocess, args=(job_q, result_q, self.reporter.__class__)) process.start() proc_list.append(process) return proc_list
Example #2
Source File: pickletools.py From Fluid-Designer with GNU General Public License v3.0 | 4 votes |
def optimize(p): 'Optimize a pickle string by removing unused PUT opcodes' put = 'PUT' get = 'GET' oldids = set() # set of all PUT ids newids = {} # set of ids used by a GET opcode opcodes = [] # (op, idx) or (pos, end_pos) proto = 0 protoheader = b'' for opcode, arg, pos, end_pos in _genops(p, yield_end_pos=True): if 'PUT' in opcode.name: oldids.add(arg) opcodes.append((put, arg)) elif opcode.name == 'MEMOIZE': idx = len(oldids) oldids.add(idx) opcodes.append((put, idx)) elif 'FRAME' in opcode.name: pass elif 'GET' in opcode.name: if opcode.proto > proto: proto = opcode.proto newids[arg] = None opcodes.append((get, arg)) elif opcode.name == 'PROTO': if arg > proto: proto = arg if pos == 0: protoheader = p[pos: end_pos] else: opcodes.append((pos, end_pos)) else: opcodes.append((pos, end_pos)) del oldids # Copy the opcodes except for PUTS without a corresponding GET out = io.BytesIO() # Write the PROTO header before any framing out.write(protoheader) pickler = pickle._Pickler(out, proto) if proto >= 4: pickler.framer.start_framing() idx = 0 for op, arg in opcodes: if op is put: if arg not in newids: continue data = pickler.put(idx) newids[arg] = idx idx += 1 elif op is get: data = pickler.get(newids[arg]) else: data = p[op:arg] pickler.framer.commit_frame() pickler.write(data) pickler.framer.end_framing() return out.getvalue() ############################################################################## # A symbolic pickle disassembler.
Example #3
Source File: pickletools.py From ironpython3 with Apache License 2.0 | 4 votes |
def optimize(p): 'Optimize a pickle string by removing unused PUT opcodes' put = 'PUT' get = 'GET' oldids = set() # set of all PUT ids newids = {} # set of ids used by a GET opcode opcodes = [] # (op, idx) or (pos, end_pos) proto = 0 protoheader = b'' for opcode, arg, pos, end_pos in _genops(p, yield_end_pos=True): if 'PUT' in opcode.name: oldids.add(arg) opcodes.append((put, arg)) elif opcode.name == 'MEMOIZE': idx = len(oldids) oldids.add(idx) opcodes.append((put, idx)) elif 'FRAME' in opcode.name: pass elif 'GET' in opcode.name: if opcode.proto > proto: proto = opcode.proto newids[arg] = None opcodes.append((get, arg)) elif opcode.name == 'PROTO': if arg > proto: proto = arg if pos == 0: protoheader = p[pos: end_pos] else: opcodes.append((pos, end_pos)) else: opcodes.append((pos, end_pos)) del oldids # Copy the opcodes except for PUTS without a corresponding GET out = io.BytesIO() # Write the PROTO header before any framing out.write(protoheader) pickler = pickle._Pickler(out, proto) if proto >= 4: pickler.framer.start_framing() idx = 0 for op, arg in opcodes: if op is put: if arg not in newids: continue data = pickler.put(idx) newids[arg] = idx idx += 1 elif op is get: data = pickler.get(newids[arg]) else: data = p[op:arg] pickler.framer.commit_frame() pickler.write(data) pickler.framer.end_framing() return out.getvalue() ############################################################################## # A symbolic pickle disassembler.
Example #4
Source File: pickletools.py From Project-New-Reign---Nemesis-Main with GNU General Public License v3.0 | 4 votes |
def optimize(p): 'Optimize a pickle string by removing unused PUT opcodes' put = 'PUT' get = 'GET' oldids = set() # set of all PUT ids newids = {} # set of ids used by a GET opcode opcodes = [] # (op, idx) or (pos, end_pos) proto = 0 protoheader = b'' for opcode, arg, pos, end_pos in _genops(p, yield_end_pos=True): if 'PUT' in opcode.name: oldids.add(arg) opcodes.append((put, arg)) elif opcode.name == 'MEMOIZE': idx = len(oldids) oldids.add(idx) opcodes.append((put, idx)) elif 'FRAME' in opcode.name: pass elif 'GET' in opcode.name: if opcode.proto > proto: proto = opcode.proto newids[arg] = None opcodes.append((get, arg)) elif opcode.name == 'PROTO': if arg > proto: proto = arg if pos == 0: protoheader = p[pos: end_pos] else: opcodes.append((pos, end_pos)) else: opcodes.append((pos, end_pos)) del oldids # Copy the opcodes except for PUTS without a corresponding GET out = io.BytesIO() # Write the PROTO header before any framing out.write(protoheader) pickler = pickle._Pickler(out, proto) if proto >= 4: pickler.framer.start_framing() idx = 0 for op, arg in opcodes: if op is put: if arg not in newids: continue data = pickler.put(idx) newids[arg] = idx idx += 1 elif op is get: data = pickler.get(newids[arg]) else: data = p[op:arg] pickler.framer.commit_frame() pickler.write(data) pickler.framer.end_framing() return out.getvalue() ############################################################################## # A symbolic pickle disassembler.