Python networkx.read_gpickle() Examples

def call_exps(params, data_set):
    print('Dataset: %s' % data_set)
    model_hyp = json.load(
        open('gem/experiments/config/%s.conf' % data_set, 'r')
    )
    if bool(params["node_labels"]):
        node_labels = cPickle.load(
            open('gem/data/%s/node_labels.pickle' % data_set, 'rb')
        )
    else:
        node_labels = None
    di_graph = nx.read_gpickle('gem/data/%s/graph.gpickle' % data_set)
    for d, meth in itertools.product(params["dimensions"], params["methods"]):
        dim = int(d)
        MethClass = getattr(
            importlib.import_module("gem.embedding.%s" % meth),
            methClassMap[meth]
        )
        hyp = {"d": dim}
        hyp.update(model_hyp[meth])
        MethObj = MethClass(hyp)
        run_exps(MethObj, di_graph, data_set, node_labels, params) 

def dbt_dag(start_date, schedule_interval, default_args):
    temp_dag = DAG('gospel_.dbt_sub_dag', start_date=start_date, schedule_interval=schedule_interval, default_args=default_args)
    G = nx.read_gpickle('/home/airflowuser/project/graph.gpickle')

    def make_dbt_task(model_name):
        simple_model_name = model_name.split('.')[-1]
        dbt_task = BashOperator(
                    task_id=model_name,
                    bash_command='cd ~/gospel && dbt run  --profile=warehouse --target=prod --non-destructive --models {simple_model_name}'.format(simple_model_name=simple_model_name),
                    dag=temp_dag
                    )
        return dbt_task


    dbt_tasks = {}
    for node_name in set(G.nodes()):
        dbt_task = make_dbt_task(node_name)
        dbt_tasks[node_name] = dbt_task

    for edge in G.edges():
        dbt_tasks[edge[0]].set_downstream(dbt_tasks[edge[1]])
    return temp_dag 

def test_protocol(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            with tempfile.TemporaryFile() as f:
                nx.write_gpickle(G, f, 0)
                f.seek(0)
                Gin = nx.read_gpickle(f)
                assert_nodes_equal(list(G.nodes(data=True)),
                                   list(Gin.nodes(data=True)))
                assert_edges_equal(list(G.edges(data=True)),
                                   list(Gin.edges(data=True)))
                assert_graphs_equal(G, Gin) 

def loadDynamicSBmGraph(file_perfix, length):
    graph_files = ['%s_%d_graph.gpickle' % (file_perfix, i) for i in range(length)]
    info_files = ['%s_%d_node.pkl' % (file_perfix, i) for i in range(length)]

    graphs = [nx.read_gpickle(graph_file) for graph_file in graph_files]

    nodes_comunities = []
    perturbations = []
    for info_file in info_files:
        with open(info_file, 'rb') as fp:
            node_infos = pickle.load(fp)
            nodes_comunities.append(node_infos['community'])
            perturbations.append(node_infos['perturbation'])

    return zip(graphs, nodes_comunities, perturbations) 

def read_gpickle(path):
    """Read graph object in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be uncompressed.

    Returns
    -------
    G : graph
       A NetworkX graph

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")
    >>> G = nx.read_gpickle("test.gpickle")

    References
    ----------
    .. [1] http://docs.python.org/library/pickle.html
    """
    return pickle.load(path)

# fixture for nose tests 

def test_gpickle(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            (fd,fname)=tempfile.mkstemp()
            nx.write_gpickle(G,fname)
            Gin=nx.read_gpickle(fname)
            assert_nodes_equal(G.nodes(data=True),
                                Gin.nodes(data=True))
            assert_edges_equal(G.edges(data=True),
                                Gin.edges(data=True))
            assert_graphs_equal(G, Gin)
            os.close(fd)
            os.unlink(fname) 

def test_protocol(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            with tempfile.TemporaryFile() as f:
                nx.write_gpickle(G, f, 0)
                f.seek(0)
                Gin = nx.read_gpickle(f)
                assert_nodes_equal(G.nodes(data=True),
                                   Gin.nodes(data=True))
                assert_edges_equal(G.edges(data=True),
                                   Gin.edges(data=True))
                assert_graphs_equal(G, Gin) 

def read_graph(name):
    dirname = os.path.dirname(__file__)
    path = os.path.join(dirname, name + '.gpickle.bz2')
    return nx.read_gpickle(path) 

def test_large(self):
        fname = os.path.join(os.path.dirname(__file__), 'netgen-2.gpickle.bz2')
        G = nx.read_gpickle(fname)
        flowCost, flowDict = nx.network_simplex(G)
        assert_equal(6749969302, flowCost)
        assert_equal(6749969302, nx.cost_of_flow(G, flowDict))
        flowCost, flowDict = nx.capacity_scaling(G)
        assert_equal(6749969302, flowCost)
        assert_equal(6749969302, nx.cost_of_flow(G, flowDict)) 

def read_gpickle(path):
    """Read graph object in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be uncompressed.

    Returns
    -------
    G : graph
       A NetworkX graph

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")
    >>> G = nx.read_gpickle("test.gpickle")

    References
    ----------
    .. [1] https://docs.python.org/2/library/pickle.html
    """
    return pickle.load(path)

# fixture for nose tests 

def test_gpickle(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            (fd, fname) = tempfile.mkstemp()
            nx.write_gpickle(G, fname)
            Gin = nx.read_gpickle(fname)
            assert_nodes_equal(list(G.nodes(data=True)),
                               list(Gin.nodes(data=True)))
            assert_edges_equal(list(G.edges(data=True)),
                               list(Gin.edges(data=True)))
            assert_graphs_equal(G, Gin)
            os.close(fd)
            os.unlink(fname) 

def loadRealGraphSeries(file_prefix, startId, endId):
    graphs = []
    for file_id in range(startId, endId + 1):
        graph_file = file_prefix + str(file_id) + '_graph.gpickle'
        graphs.append(nx.read_gpickle(graph_file))
    return graphs 

def read_graph(name):
    dirname = os.path.dirname(__file__)
    path = os.path.join(dirname, name + '.gpickle.bz2')
    return nx.read_gpickle(path) 

def test_large(self):
        fname = os.path.join(os.path.dirname(__file__), 'netgen-2.gpickle.bz2')
        G = nx.read_gpickle(fname)
        flowCost, flowDict = nx.network_simplex(G)
        assert_equal(6749969302, flowCost)
        assert_equal(6749969302, nx.cost_of_flow(G, flowDict))
        flowCost, flowDict = nx.capacity_scaling(G)
        assert_equal(6749969302, flowCost)
        assert_equal(6749969302, nx.cost_of_flow(G, flowDict)) 

def read_gpickle(path):
    """Read graph object in Python pickle format.

    Pickles are a serialized byte stream of a Python object [1]_.
    This format will preserve Python objects used as nodes or edges.

    Parameters
    ----------
    path : file or string
       File or filename to write.
       Filenames ending in .gz or .bz2 will be uncompressed.

    Returns
    -------
    G : graph
       A NetworkX graph

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> nx.write_gpickle(G, "test.gpickle")
    >>> G = nx.read_gpickle("test.gpickle")

    References
    ----------
    .. [1] https://docs.python.org/2/library/pickle.html
    """
    return pickle.load(path)

# fixture for nose tests 

def test_gpickle(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            (fd,fname)=tempfile.mkstemp()
            nx.write_gpickle(G,fname)
            Gin=nx.read_gpickle(fname)
            assert_nodes_equal(list(G.nodes(data=True)),
                               list(Gin.nodes(data=True)))
            assert_edges_equal(list(G.edges(data=True)),
                               list(Gin.edges(data=True)))
            assert_graphs_equal(G, Gin)
            os.close(fd)
            os.unlink(fname) 

def test_protocol(self):
        for G in [self.G, self.DG, self.MG, self.MDG,
                  self.fG, self.fDG, self.fMG, self.fMDG]:
            with tempfile.TemporaryFile() as f:
                nx.write_gpickle(G, f, 0)
                f.seek(0)
                Gin = nx.read_gpickle(f)
                assert_nodes_equal(list(G.nodes(data=True)),
                                   list(Gin.nodes(data=True)))
                assert_edges_equal(list(G.edges(data=True)),
                                   list(Gin.edges(data=True)))
                assert_graphs_equal(G, Gin) 

def read_graph(name):
    dirname = os.path.dirname(__file__)
    path = os.path.join(dirname, name + '.gpickle.bz2')
    return nx.read_gpickle(path) 

def test_large(self):
        fname = os.path.join(os.path.dirname(__file__), 'netgen-2.gpickle.bz2')
        G = nx.read_gpickle(fname)
        flowCost, flowDict = nx.network_simplex(G)
        assert_equal(6749969302, flowCost)
        assert_equal(6749969302, nx.cost_of_flow(G, flowDict))
        flowCost, flowDict = nx.capacity_scaling(G)
        assert_equal(6749969302, flowCost)
        assert_equal(6749969302, nx.cost_of_flow(G, flowDict)) 

def load_network_from_gpickle(filename, verbose=True):

    filename = re.sub('~', expanduser('~'), filename)
    G = nx.read_gpickle(filename)

    return G 

def process(self):
        G = nx.read_gpickle(self.raw_paths[0])
        data = [[v, w, d['e_label'].item()] for v, w, d in G.edges(data=True)]

        edge_index = torch.tensor(data)[:, :2].t().contiguous()
        edge_type = torch.tensor(data)[:, 2]

        data = Data(edge_index=edge_index, edge_type=edge_type,
                    num_nodes=G.number_of_nodes())

        if self.pre_transform is not None:
            data = self.pre_filter(data)

        torch.save(self.collate([data]), self.processed_paths[0]) 

def load_cache(self):
        mg = nx.read_gpickle(os.path.join(self._dir, 'cached_mg.gpickle'))
        src = np.load(os.path.join(self._dir, 'cached_src.npy'))
        dst = np.load(os.path.join(self._dir, 'cached_dst.npy'))
        ntid = np.load(os.path.join(self._dir, 'cached_ntid.npy'))
        etid = np.load(os.path.join(self._dir, 'cached_etid.npy'))
        ntypes = load_strlist(os.path.join(self._dir, 'cached_ntypes.txt'))
        etypes = load_strlist(os.path.join(self._dir, 'cached_etypes.txt'))
        self.train_idx = F.tensor(np.load(os.path.join(self._dir, 'cached_train_idx.npy')))
        self.test_idx = F.tensor(np.load(os.path.join(self._dir, 'cached_test_idx.npy')))
        labels = np.load(os.path.join(self._dir, 'cached_labels.npy'))
        self.num_classes = labels.max() + 1
        self.labels = F.tensor(labels)

        self.build_graph(mg, src, dst, ntid, etid, ntypes, etypes) 

def from_pickle(path: Union[str, BinaryIO], check_version: bool = True) -> BELGraph:
    """Read a graph from a pickle file.

    :param path: File or filename to read. Filenames ending in .gz or .bz2 will be uncompressed.
    :param bool check_version: Checks if the graph was produced by this version of PyBEL
    """
    graph = nx.read_gpickle(path)

    raise_for_not_bel(graph)
    if check_version:
        raise_for_old_graph(graph)

    return graph 

def read_graph(self, subgraph_file=None):
        if subgraph_file is None:
            subraph_file = self.context_graph_file
        logging.info("Writing graph.")
        # write the graph out
        file_format = subgraph_file.split(".")[-1]
        if file_format == "graphml":
            return nx.read_graphml(subgraph_file)
        elif file_format == "gml":
            return nx.read_gml(subgraph_file)
        elif file_format == "gexf":
            return nx.read_gexf(subgraph_file)
        elif file_format == "net":
            return nx.read_pajek(subgraph_file)
        elif file_format == "yaml":
            return nx.read_yaml(subgraph_file)
        elif file_format == "gpickle":
            return nx.read_gpickle(subgraph_file)
        else:
            logging.warning("File format not found, returning empty graph.")
        return nx.MultiDiGraph() 

def loadSBMGraph(file_prefix):
    graph_file = file_prefix + '_graph.gpickle'
    G = nx.read_gpickle(graph_file)
    node_file = file_prefix + '_node.pkl'
    with open(node_file, 'rb') as fp:
        node_community = pickle.load(fp)
    return (G, node_community) 

def loadRealGraphSeries(file_prefix, startId, endId):
    graphs = []
    for file_id in range(startId, endId + 1):
        graph_file = file_prefix + str(file_id) + '_graph.gpickle'
        graphs.append(nx.read_gpickle(graph_file))
    return graphs 

def loadDynamicSBmGraph(file_perfix, length):
    graph_files = ['%s_%d_graph.gpickle' % (file_perfix, i) for i in xrange(length)]
    info_files = ['%s_%d_node.pkl' % (file_perfix, i) for i in xrange(length)]

    graphs = [nx.read_gpickle(graph_file) for graph_file in graph_files]

    nodes_comunities = []
    perturbations = []
    for info_file in info_files:
        with open(info_file, 'rb') as fp:
            node_infos = pickle.load(fp)
            nodes_comunities.append(node_infos['community'])
            perturbations.append(node_infos['perturbation'])

    return zip(graphs, nodes_comunities, perturbations) 

def deserialize_network(path):
    G = nx.read_gpickle(path + "graph.pickle")
    id_to_info = nx.read_gpickle(path + "id_info.pickle")
    table_to_ids = nx.read_gpickle(path + "table_ids.pickle")
    network = FieldNetwork(G, id_to_info, table_to_ids)
    return network 

def cached_reading_graph_edges(self):
        """
        Checks if networkx and edges dictionary pickles are present. If they are older than
        CACHING_RETENTION_MINUTES, make fresh pickles, else read them from the files.
        """
        cache_dir = os.path.join(settings.home_dir, 'cache')
        if not os.path.exists(cache_dir):
            os.mkdir(cache_dir)
        cache_edges_filename = os.path.join(cache_dir, 'graph.gpickle')
        cache_graph_filename = os.path.join(cache_dir, 'edges.gpickle')

        try:
            timestamp_graph = os.path.getmtime(cache_graph_filename)
        except FileNotFoundError:
            timestamp_graph = 0  # set very old timestamp

        if timestamp_graph < time.time() - settings.CACHING_RETENTION_MINUTES * 60:  # old graph in file
            logger.info(f"Saved graph is too old. Fetching new one.")
            self.set_graph_and_edges()
            nx.write_gpickle(self.graph, cache_graph_filename)
            with open(cache_edges_filename, 'wb') as file:
                pickle.dump(self.edges, file)
        else:  # recent graph in file
            logger.info("Reading graph from file.")
            self.graph = nx.read_gpickle(cache_graph_filename)
            with open(cache_edges_filename, 'rb') as file:
                self.edges = pickle.load(file) 

def load_cpnet():
    global cpnet,concept2id, relation2id, id2relation, id2concept, cpnet_simple
    print("loading cpnet....")
    cpnet = nx.read_gpickle(config["paths"]["conceptnet_en_graph"])
    print("Done")

    cpnet_simple = nx.Graph()
    for u, v, data in cpnet.edges(data=True):
        w = data['weight'] if 'weight' in data else 1.0
        if cpnet_simple.has_edge(u, v):
            cpnet_simple[u][v]['weight'] += w
        else:
            cpnet_simple.add_edge(u, v, weight=w)