Python networkx.write_gpickle() Examples

def write_graph(self, G=None, subgraph_file=None):
        if G is None:
            G = self.context_graph
        if subgraph_file is None:
            subgraph_file = self.context_graph_file
        logging.info("Writing graph.")
        # write the graph out
        file_format = subgraph_file.split(".")[-1]
        if file_format == "graphml":
            nx.write_graphml(G, subgraph_file)
        elif file_format == "gml":
            nx.write_gml(G, subgraph_file)
        elif file_format == "gexf":
            nx.write_gexf(G, subgraph_file)
        elif file_format == "net":
            nx.write_pajek(G, subgraph_file)
        elif file_format == "yaml":
            nx.write_yaml(G, subgraph_file)
        elif file_format == "gpickle":
            nx.write_gpickle(G, subgraph_file)
        else:
            print "File format not found, writing graphml."
            nx.write_graphml(G, subgraph_file) 

def serialize_network(network, path):
    """
    Serialize the meta schema index
    :param network:
    :param path:
    :return:
    """
    G = network._get_underlying_repr_graph()
    id_to_field_info = network._get_underlying_repr_id_to_field_info()
    table_to_ids = network._get_underlying_repr_table_to_ids()

    # Make sure we create directory if this does not exist
    path = path + '/'  # force separator
    os.makedirs(os.path.dirname(path), exist_ok=True)

    nx.write_gpickle(G, path + "graph.pickle")
    nx.write_gpickle(id_to_field_info, path + "id_info.pickle")
    nx.write_gpickle(table_to_ids, path + "table_ids.pickle") 

def saveDynamicSBmGraph(file_perfix, dynamic_graphs):
    length = len(dynamic_graphs)
    graph_files = ['%s_%d_graph.gpickle' % (file_perfix, i) for i in range(length)]
    info_files = ['%s_%d_node.pkl' % (file_perfix, i) for i in range(length)]

    for i in xrange(length):
        # save graph
        nx.write_gpickle(dynamic_graphs[i][0], graph_files[i])
        # save additional node info
        with open(info_files[i], 'wb') as fp:
            node_infos = {}
            node_infos['community'] = dynamic_graphs[i][1]
            node_infos['perturbation'] = dynamic_graphs[i][2]
            pickle.dump(node_infos, fp) 

def test_protocol(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            with tempfile.TemporaryFile() as f:
                nx.write_gpickle(G, f, 0)
                f.seek(0)
                Gin = nx.read_gpickle(f)
                assert_nodes_equal(list(G.nodes(data=True)),
                                   list(Gin.nodes(data=True)))
                assert_edges_equal(list(G.edges(data=True)),
                                   list(Gin.edges(data=True)))
                assert_graphs_equal(G, Gin) 

def test_gpickle(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            (fd,fname)=tempfile.mkstemp()
            nx.write_gpickle(G,fname)
            Gin=nx.read_gpickle(fname)
            assert_nodes_equal(list(G.nodes(data=True)),
                               list(Gin.nodes(data=True)))
            assert_edges_equal(list(G.edges(data=True)),
                               list(Gin.edges(data=True)))
            assert_graphs_equal(G, Gin)
            os.close(fd)
            os.unlink(fname) 

def read_gpickle(path):
    """Read graph object in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be uncompressed.

    Returns
    -------
    G : graph
       A NetworkX graph

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")
    >>> G = nx.read_gpickle("test.gpickle")

    References
    ----------
    .. [1] https://docs.python.org/2/library/pickle.html
    """
    return pickle.load(path)

# fixture for nose tests 

def write_gpickle(G, path, protocol=pickle.HIGHEST_PROTOCOL):
    """Write graph in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    G : graph
       A NetworkX graph

    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be compressed.

    protocol : integer
        Pickling protocol to use. Default value: ``pickle.HIGHEST_PROTOCOL``.

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")

    References
    ----------
    .. [1] https://docs.python.org/2/library/pickle.html
    """
    pickle.dump(G, path, protocol) 

def test_protocol(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            with tempfile.TemporaryFile() as f:
                nx.write_gpickle(G, f, 0)
                f.seek(0)
                Gin = nx.read_gpickle(f)
                assert_nodes_equal(list(G.nodes(data=True)),
                                   list(Gin.nodes(data=True)))
                assert_edges_equal(list(G.edges(data=True)),
                                   list(Gin.edges(data=True)))
                assert_graphs_equal(G, Gin) 

def test_gpickle(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            (fd, fname) = tempfile.mkstemp()
            nx.write_gpickle(G, fname)
            Gin = nx.read_gpickle(fname)
            assert_nodes_equal(list(G.nodes(data=True)),
                               list(Gin.nodes(data=True)))
            assert_edges_equal(list(G.edges(data=True)),
                               list(Gin.edges(data=True)))
            assert_graphs_equal(G, Gin)
            os.close(fd)
            os.unlink(fname) 

def read_gpickle(path):
    """Read graph object in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be uncompressed.

    Returns
    -------
    G : graph
       A NetworkX graph

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")
    >>> G = nx.read_gpickle("test.gpickle")

    References
    ----------
    .. [1] https://docs.python.org/2/library/pickle.html
    """
    return pickle.load(path)

# fixture for nose tests 

def write_gpickle(G, path, protocol=pickle.HIGHEST_PROTOCOL):
    """Write graph in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    G : graph
       A NetworkX graph

    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be compressed.

    protocol : integer
        Pickling protocol to use. Default value: ``pickle.HIGHEST_PROTOCOL``.

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")

    References
    ----------
    .. [1] https://docs.python.org/2/library/pickle.html
    """
    pickle.dump(G, path, protocol) 

def test_protocol(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            with tempfile.TemporaryFile() as f:
                nx.write_gpickle(G, f, 0)
                f.seek(0)
                Gin = nx.read_gpickle(f)
                assert_nodes_equal(G.nodes(data=True),
                                   Gin.nodes(data=True))
                assert_edges_equal(G.edges(data=True),
                                   Gin.edges(data=True))
                assert_graphs_equal(G, Gin) 

def test_gpickle(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            (fd,fname)=tempfile.mkstemp()
            nx.write_gpickle(G,fname)
            Gin=nx.read_gpickle(fname)
            assert_nodes_equal(G.nodes(data=True),
                                Gin.nodes(data=True))
            assert_edges_equal(G.edges(data=True),
                                Gin.edges(data=True))
            assert_graphs_equal(G, Gin)
            os.close(fd)
            os.unlink(fname) 

def read_gpickle(path):
    """Read graph object in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be uncompressed.

    Returns
    -------
    G : graph
       A NetworkX graph

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")
    >>> G = nx.read_gpickle("test.gpickle")

    References
    ----------
    .. [1] http://docs.python.org/library/pickle.html
    """
    return pickle.load(path)

# fixture for nose tests 

def write_gpickle(G, path, protocol=pickle.HIGHEST_PROTOCOL):
    """Write graph in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    G : graph
       A NetworkX graph

    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be compressed.

    protocol : integer
        Pickling protocol to use. Default value: ``pickle.HIGHEST_PROTOCOL``.

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")

    References
    ----------
    .. [1] http://docs.python.org/library/pickle.html
    """
    pickle.dump(G, path, protocol) 

def saveRealGraphSeries(G, file_prefix='graphs/day_'):
    for idx in range(len(G)):
        f_name = file_prefix + str(idx) + "_graph.gpickle"
        # cPickle.dump(G[idx], open(f_name, 'wb'))
        nx.write_gpickle(G[idx], f_name) 

def nx_save_to_file(self, graph, file_path='output-amazon.pkl'):
        nx.write_gpickle(graph, file_path) 

def save_cpnet():
    global concept2id, relation2id, id2relation, id2concept, blacklist
    load_resources()
    graph = nx.MultiDiGraph()
    with open(config["paths"]["conceptnet_en"], "r", encoding="utf8") as f:
        lines = f.readlines()

        def not_save(cpt):
            if cpt in blacklist:
                return True
            for t in cpt.split("_"):
                if t in nltk_stopwords:
                    return True
            return False

        for line in tqdm(lines, desc="saving to graph"):
            ls = line.strip().split('\t')
            rel = relation2id[ls[0]]
            subj = concept2id[ls[1]]
            obj = concept2id[ls[2]]
            weight = float(ls[3])
            if id2relation[rel] == "hascontext":
                continue
            if not_save(ls[1]) or not_save(ls[2]):
                continue
            if id2relation[rel] == "relatedto" or id2relation[rel] == "antonym":
                weight -= 0.3
                # continue
            if subj == obj: # delete loops
                continue
            weight = 1+float(math.exp(1-weight))
            graph.add_edge(subj, obj, rel=rel, weight=weight)
            graph.add_edge(obj, subj, rel=rel+len(relation2id), weight=weight)


    nx.write_gpickle(graph, config["paths"]["conceptnet_en_graph"])
    # with open(config["paths"]["conceptnet_en_graph"], 'w') as f:
    #     f.write(json.dumps(nx.node_link_data(graph))) 

def cached_reading_graph_edges(self):
        """
        Checks if networkx and edges dictionary pickles are present. If they are older than
        CACHING_RETENTION_MINUTES, make fresh pickles, else read them from the files.
        """
        cache_dir = os.path.join(settings.home_dir, 'cache')
        if not os.path.exists(cache_dir):
            os.mkdir(cache_dir)
        cache_edges_filename = os.path.join(cache_dir, 'graph.gpickle')
        cache_graph_filename = os.path.join(cache_dir, 'edges.gpickle')

        try:
            timestamp_graph = os.path.getmtime(cache_graph_filename)
        except FileNotFoundError:
            timestamp_graph = 0  # set very old timestamp

        if timestamp_graph < time.time() - settings.CACHING_RETENTION_MINUTES * 60:  # old graph in file
            logger.info(f"Saved graph is too old. Fetching new one.")
            self.set_graph_and_edges()
            nx.write_gpickle(self.graph, cache_graph_filename)
            with open(cache_edges_filename, 'wb') as file:
                pickle.dump(self.edges, file)
        else:  # recent graph in file
            logger.info("Reading graph from file.")
            self.graph = nx.read_gpickle(cache_graph_filename)
            with open(cache_edges_filename, 'rb') as file:
                self.edges = pickle.load(file) 

def save_network(self, **kwargs):
        if 'output_file' in kwargs:
            output_file = kwargs['output_file']
        else:
            output_file = os.path.join(os.getcwd(), self.path_to_network_file + '.gpickle')

        nx.write_gpickle(self.graph, output_file) 

def saveDynamicSBmGraph(file_perfix, dynamic_graphs):
    length = len(dynamic_graphs)
    graph_files = ['%s_%d_graph.gpickle' % (file_perfix, i) for i in xrange(length)]
    info_files = ['%s_%d_node.pkl' % (file_perfix, i) for i in xrange(length)]

    for i in xrange(length):
        # save graph
        nx.write_gpickle(dynamic_graphs[i][0], graph_files[i])
        # save additional node info
        with open(info_files[i], 'wb') as fp:
            node_infos = {}
            node_infos['community'] = dynamic_graphs[i][1]
            node_infos['perturbation'] = dynamic_graphs[i][2]
            pickle.dump(node_infos, fp) 

def saveRealGraphSeries(G, file_prefix='graphs/day_'):
    for idx in range(len(G)):
        f_name = file_prefix + str(idx) + "_graph.gpickle"
        # cPickle.dump(G[idx], open(f_name, 'wb'))
        nx.write_gpickle(G[idx], f_name) 

def to_pickle(graph: BELGraph, path: Union[str, BinaryIO], protocol: int = HIGHEST_PROTOCOL) -> None:
    """Write this graph to a pickle object with :func:`networkx.write_gpickle`.

    Note that the pickle module has some incompatibilities between Python 2 and 3. To export a universally importable
    pickle, choose 0, 1, or 2.

    :param graph: A BEL graph
    :param path: A path or file-like
    :param protocol: Pickling protocol to use. Defaults to ``HIGHEST_PROTOCOL``.

    .. seealso:: https://docs.python.org/3.6/library/pickle.html#data-stream-format
    """
    raise_for_not_bel(graph)
    nx.write_gpickle(graph, path, protocol=protocol) 

def save_cache(self, mg, src, dst, ntid, etid, ntypes, etypes):
        nx.write_gpickle(mg, os.path.join(self._dir, 'cached_mg.gpickle'))
        np.save(os.path.join(self._dir, 'cached_src.npy'), src)
        np.save(os.path.join(self._dir, 'cached_dst.npy'), dst)
        np.save(os.path.join(self._dir, 'cached_ntid.npy'), ntid)
        np.save(os.path.join(self._dir, 'cached_etid.npy'), etid)
        save_strlist(os.path.join(self._dir, 'cached_ntypes.txt'), ntypes)
        save_strlist(os.path.join(self._dir, 'cached_etypes.txt'), etypes)
        np.save(os.path.join(self._dir, 'cached_train_idx.npy'), F.asnumpy(self.train_idx))
        np.save(os.path.join(self._dir, 'cached_test_idx.npy'), F.asnumpy(self.test_idx))
        np.save(os.path.join(self._dir, 'cached_labels.npy'), F.asnumpy(self.labels)) 

def prepare_data(G, df_osm_built, df_osm_pois, df_indices, num_processes, kw_arguments):
	""" 
	Pickles data to a temporary folder in order to achieve parallel accessibility calculation
	A new subprocess will be created in order to minimize memory requirements

	Parameters
	----------
	G : networkx multidigraph
		input graph to calculate accessibility
	df_osm_built : geopandas.GeoDataFrame
		buildings data
	df_osm_pois : geopandas.GeoDataFrame
		buildings data
	df_indices : geopandas.GeoDataFrame
		data frame where indices will be calculated
	num_processes : int
		number of data chunks to create
	kw_arguments : pandas.Series
		additional keyword arguments

	Returns
	----------

	"""
	# Divide long edges
	divide_long_edges_graph(G, kw_arguments.max_edge_length )
	log("Graph long edges shortened")

	# Get activities
	df_built_activ = df_osm_built[ df_osm_built.classification.isin(["activity","mixed"]) ]
	df_pois_activ = df_osm_pois[ df_osm_pois.classification.isin(["activity","mixed"]) ]
	
	# Associate them to its closest node in the graph
	associate_activities_closest_node(G, df_built_activ, df_pois_activ )
	log("Activities associated to graph nodes")

	# Nodes dict
	for n, data in G.nodes.data(data=True):
		# Remove useless keys
		keys_ = list( data.keys() )
		[ data.pop(k) for k in keys_ if k not in ["x","y","num_activities"] ]
	
	# Edges dict
	for u, v, data in G.edges.data(data=True, keys=False):
		# Remove useless keys
		keys_ = list( data.keys() )
		[ data.pop(k) for k in keys_ if k not in ["length","key"] ]

	try:
		G.graph.pop("streets_per_node")
	except:
		pass
	# Pickle graph
	nx.write_gpickle(G, "temp/graph.gpickle")

	# Prepare input indices points
	data_split = np.array_split(df_indices, num_processes)
	for i in range(num_processes):
		data_split[i].to_pickle("temp/points_"+str(i)+".pkl")
	# Pickle arguments
	kw_arguments.to_pickle("temp/arguments.pkl") 

def androcg_main(verbose,
                 APK,
                 classname,
                 methodname,
                 descriptor,
                 accessflag,
                 no_isolated,
                 show,
                 output):
    from androguard.core.androconf import show_logging
    from androguard.core.bytecode import FormatClassToJava
    from androguard.misc import AnalyzeAPK
    import networkx as nx
    import logging
    log = logging.getLogger("androcfg")
    if verbose:
        show_logging(logging.INFO)

    a, d, dx = AnalyzeAPK(APK)

    entry_points = map(FormatClassToJava,
                       a.get_activities() + a.get_providers() +
                       a.get_services() + a.get_receivers())
    entry_points = list(entry_points)

    log.info("Found The following entry points by search AndroidManifest.xml: "
             "{}".format(entry_points))

    CG = dx.get_call_graph(classname,
                           methodname,
                           descriptor,
                           accessflag,
                           no_isolated,
                           entry_points,
                           )

    write_methods = dict(gml=_write_gml,
                         gexf=nx.write_gexf,
                         gpickle=nx.write_gpickle,
                         graphml=nx.write_graphml,
                         yaml=nx.write_yaml,
                         net=nx.write_pajek,
                         )

    if show:
        plot(CG)
    else:
        writer = output.rsplit(".", 1)[1]
        if writer in ["bz2", "gz"]:
            writer = output.rsplit(".", 2)[1]
        if writer not in write_methods:
            print("Could not find a method to export files to {}!"
                  .format(writer))
            sys.exit(1)

        write_methods[writer](CG, output)