Python cPickle.Pickler() Examples

def getClient(
        cls, servers, debug=0, pickleProtocol=0,
        pickler=pickle.Pickler, unpickler=pickle.Unpickler,
        pload=None, pid=None
    ):

        if cls.allowTestCache:
            return TestClient(
                servers, debug=debug,
                pickleProtocol=pickleProtocol, pickler=pickler,
                unpickler=unpickler, pload=pload, pid=pid)
        elif config.Memcached.Pools.Default.ClientEnabled:
            return Client(
                servers, debug=debug, pickleProtocol=pickleProtocol,
                pickler=pickler, unpickler=unpickler, pload=pload, pid=pid)
        else:
            return None 

def __init__(self, request, servers, debug=0, pickleProtocol=0,
                 pickler=pickle.Pickler, unpickler=pickle.Unpickler,
                 pload=None, pid=None,
                 default_time_expire = DEFAULT_TIME_EXPIRE):
        self.request=request
        self.default_time_expire = default_time_expire
        if request:
            app = request.application
        else:
            app = ''
        Client.__init__(self, servers, debug, pickleProtocol,
                        pickler, unpickler, pload, pid)
        if not app in self.meta_storage:
            self.storage = self.meta_storage[app] = {
                CacheAbstract.cache_stats_name: {
                    'hit_total': 0,
                    'misses': 0,
                    }}
        else:
            self.storage = self.meta_storage[app] 

def dumps(self, arg, proto=0):
        f = StringIO()
        p = cPickle.Pickler(f, proto)
        p.dump(arg)
        f.seek(0)
        return f.read() 

def __init__(self,
               servers=None,
               debug=0,
               pickleProtocol=cPickle.HIGHEST_PROTOCOL,
               pickler=cPickle.Pickler,
               unpickler=cPickle.Unpickler,
               pload=None,
               pid=None,
               make_sync_call=None,
               _app_id=None):
    """Create a new Client object.

    No parameters are required.

    Arguments:
      servers: Ignored; only for compatibility.
      debug: Ignored; only for compatibility.
      pickleProtocol: Pickle protocol to use for pickling the object.
      pickler: pickle.Pickler sub-class to use for pickling.
      unpickler: pickle.Unpickler sub-class to use for unpickling.
      pload: Callable to use for retrieving objects by persistent id.
      pid: Callable to use for determine the persistent id for objects, if any.
      make_sync_call: Ignored; only for compatibility with an earlier version.
    """





    self._pickler_factory = pickler
    self._unpickler_factory = unpickler
    self._pickle_protocol = pickleProtocol
    self._persistent_id = pid
    self._persistent_load = pload
    self._app_id = _app_id
    self._cas_ids = {} 

def compile(self):
        self.log.info("Compiling quark code")
        for root in self.roots.sorted():
            if getattr(root, "_compiled", False):
                continue
            for use in root.uses:
                dep = self.roots[use]
                assert getattr(dep, "_compiled", False)
                self.merge(root.env, dep.env)
            self.icompile(root)
            root._compiled = True

        modified = []
        for url, file in self.entries.items():
            if not os.path.exists(url): continue
            urlc = compiled_quark(url)
            trans_roots = tuple(self.trans_roots(file.root))
            deps = tuple(self.deps(trans_roots))
            if not is_newer(urlc, __file__, *deps):
                self.log.info("Writing %s" % urlc)
                with open(urlc, "w") as fd:
                    pickler = pickle.Pickler(fd, -1)
                    pickler.dump(deps)
                    pickler.dump(trans_roots)
                    # Write out an object to mark the archive end so
                    # we can detect partial writes.
                    pickler.dump(ARCHIVE_END)
                modified.append(file.root)
        # We compute the modified flag here so it never gets saved to disk.
        for r in modified:
            r._modified = True 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def dumps(self, arg, proto=0):
        f = StringIO()
        p = cPickle.Pickler(f, proto)
        p.dump(arg)
        f.seek(0)
        return f.read() 

def dumps(self, arg, proto=0):
        p = cPickle.Pickler(proto)
        p.dump(arg)
        return p.getvalue() 

def dumps(self, arg, proto=0):
        f = StringIO()
        p = cPickle.Pickler(f, proto)
        p.fast = 1
        p.dump(arg)
        f.seek(0)
        return f.read() 

def loadData (self, filename, verbose=True, replace_missing=True):
		''' Get the data from a text file in one of 3 formats: matrix, sparse, binary_sparse'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join(self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'format' not in self.info.keys():
			self.getFormatData(filename)
		if 'feat_num' not in self.info.keys():
			self.getNbrFeatures(filename)
			
		data_func = {'dense':data_io.data, 'sparse':data_io.data_sparse, 'sparse_binary':data_io.data_binary_sparse}
		
		data = data_func[self.info['format']](filename, self.info['feat_num'])
  
		# INPORTANT: when we replace missing values we double the number of variables
  
		if self.info['format']=='dense' and replace_missing and np.any(map(np.isnan,data)):
			vprint (verbose, "Replace missing values by 0 (slow, sorry)")
			data = data_converter.replace_missing(data)
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(data)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return data 

def loadLabel (self, filename, verbose=True):
		''' Get the solution/truth values'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'task' not in self.info.keys():
			self.getTypeProblem(filename)
	
           # IG: Here change to accommodate the new multiclass label format
		if self.info['task'] == 'multilabel.classification':
			label = data_io.data(filename)
		elif self.info['task'] == 'multiclass.classification':
			label = data_converter.convert_to_num(data_io.data(filename))              
		else:
			label = np.ravel(data_io.data(filename)) # get a column vector
			#label = np.array([np.ravel(data_io.data(filename))]).transpose() # get a column vector
   
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(label)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return label 

def loadData (self, filename, verbose=True, replace_missing=True):
		''' Get the data from a text file in one of 3 formats: matrix, sparse, binary_sparse'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join(self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'format' not in self.info.keys():
			self.getFormatData(filename)
		if 'feat_num' not in self.info.keys():
			self.getNbrFeatures(filename)
			
		data_func = {'dense':data_io.data, 'sparse':data_io.data_sparse, 'sparse_binary':data_io.data_binary_sparse}
		
		data = data_func[self.info['format']](filename, self.info['feat_num'])
  
		# INPORTANT: when we replace missing values we double the number of variables
  
		if self.info['format']=='dense' and replace_missing and np.any(map(np.isnan,data)):
			vprint (verbose, "Replace missing values by 0 (slow, sorry)")
			data = data_converter.replace_missing(data)
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(data)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return data 

def loadLabel (self, filename, verbose=True):
		''' Get the solution/truth values'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'task' not in self.info.keys():
			self.getTypeProblem(filename)
	
           # IG: Here change to accommodate the new multiclass label format
		if self.info['task'] == 'multilabel.classification':
			label = data_io.data(filename)
		elif self.info['task'] == 'multiclass.classification':
			label = data_converter.convert_to_num(data_io.data(filename))              
		else:
			label = np.ravel(data_io.data(filename)) # get a column vector
			#label = np.array([np.ravel(data_io.data(filename))]).transpose() # get a column vector
   
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(label)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return label 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def Write(self):
    """Write search indexes to the index file.

    This method is a no-op if index_file is set to None.
    """
    if not self.__index_file:
      return





    descriptor, tmp_filename = tempfile.mkstemp(
        dir=os.path.dirname(self.__index_file))
    tmpfile = os.fdopen(descriptor, 'wb')

    pickler = pickle.Pickler(tmpfile, protocol=1)
    pickler.fast = True
    pickler.dump((self._VERSION, self.__indexes))

    tmpfile.close()

    self.__index_file_lock.acquire()
    try:
      try:

        os.rename(tmp_filename, self.__index_file)
      except OSError:


        os.remove(self.__index_file)
        os.rename(tmp_filename, self.__index_file)
    finally:
      self.__index_file_lock.release() 

def dumps(self, arg, proto=0):
        p = cPickle.Pickler(proto)
        p.dump(arg)
        return p.getvalue() 

def dumps(self, arg, proto=0):
        f = StringIO()
        p = cPickle.Pickler(f, proto)
        p.fast = 1
        p.dump(arg)
        f.seek(0)
        return f.read() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 

def dumps(self, arg, proto=0):
        f = StringIO()
        p = cPickle.Pickler(f, proto)
        p.dump(arg)
        f.seek(0)
        return f.read() 

def dumps(self, arg, proto=0):
        p = cPickle.Pickler(proto)
        p.dump(arg)
        return p.getvalue() 

def dumps(self, arg, proto=0):
        f = StringIO()
        p = cPickle.Pickler(f, proto)
        p.fast = 1
        p.dump(arg)
        f.seek(0)
        return f.read() 

def dumps(self, arg, proto=0):
        f = self.output()
        try:
            p = cPickle.Pickler(f, proto)
            p.dump(arg)
            f.seek(0)
            return f.read()
        finally:
            self.close(f) 

def _SaveDicts():
	if is_readonly:
		raise RuntimeError("Trying to write to a readonly gencache ('%s')!" \
		                    % win32com.__gen_path__)
	f = open(os.path.join(GetGeneratePath(), "dicts.dat"), "wb")
	try:
		p = pickle.Pickler(f)
		p.dump(pickleVersion)
		p.dump(clsidToTypelib)
	finally:
		f.close() 

def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue()