it.unimi.dsi.webgraph.ImmutableGraph Java Examples

@Override
protected int[][] parseSet() throws IOException
{
	
	ImmutableGraph in_graph = WikiGraphs.get(lang, IndexType.IN_GRAPH);
	int[][] arrayGraph = new int[in_graph.numNodes()][];
	for(int i=0; i<arrayGraph.length; i++)
	{
		int i_deg = in_graph.outdegree(i);
		if (i_deg > 0){
			arrayGraph[i] = new int[i_deg];
			LazyIntIterator iter = in_graph.successors(i);
			for (int j=0; j<arrayGraph[i].length; j++)
				arrayGraph[i][j] = iter.nextInt();
		}
	}

	return arrayGraph;
}
 

public static int reachable(final ImmutableGraph graph, final int startingNode) {
	final int n = graph.numNodes();
	final boolean[] known = new boolean[n];
	final IntArrayFIFOQueue queue = new IntArrayFIFOQueue();

	queue.enqueue(startingNode);
	known[startingNode] = true;
	int visited = 0;

	while (!queue.isEmpty()) {
		final int currentNode = queue.dequeueInt();
		visited++;
		final LazyIntIterator iterator = graph.successors(currentNode);
		for (int succ; (succ = iterator.nextInt()) != -1;) {
			if (!known[succ]) {
				known[succ] = true;
				queue.enqueue(succ);
			}
		}
	}

	return visited;
}
 

/** Given <code>g</code>, returns a Fenwick tree for the nodes of <code>g</code> whose counter number <code>i+1</code> corresponds to node <code>i</code>
 *  and it is initialized to <code>d</code> or to <code>D-d+1</code> (the latter iff <code>reverse</code> is true),
 *  where <code>D</code> is the maximum indegree in <code>g</code>, and <code>d</code> is <code>i</code>'s indegree.
 *
 * @param g a graph.
 * @param reverse if true, we generate the <em>reverse</em> distribution.
 * @return a Fenwick tree as above.
 */
public static FenwickTree getPreferentialDistribution(final ImmutableGraph g, final boolean reverse) {
	final int n = g.numNodes();
	final int[] indegree = new int[n];
	int maxIndegree = 0;
	final NodeIterator nodeIterator = g.nodeIterator();
	while (nodeIterator.hasNext()) {
		nodeIterator.nextInt();
		final LazyIntIterator successors = nodeIterator.successors();
		int target;
		while ((target = successors.nextInt()) >= 0) {
			indegree[target]++;
			if (indegree[target] > maxIndegree) maxIndegree = indegree[target];
		}
	}
	final FenwickTree t = new FenwickTree(n);
	for (int i = 0; i < indegree.length; i++)
		t.incrementCount(i + 1, reverse? maxIndegree - indegree[i] + 1 : indegree[i]);

	return t;
}
 

public static int reachable(final ImmutableGraph graph, final int startingNode) {
	final int n = graph.numNodes();
	final boolean[] known = new boolean[n];
	final IntArrayFIFOQueue queue = new IntArrayFIFOQueue();

	queue.enqueue(startingNode);
	known[startingNode] = true;
	int visited = 0;

	while (!queue.isEmpty()) {
		final int currentNode = queue.dequeueInt();
		visited++;
		final LazyIntIterator iterator = graph.successors(currentNode);
		for (int succ; (succ = iterator.nextInt()) != -1;) {
			if (!known[succ]) {
				known[succ] = true;
				queue.enqueue(succ);
			}
		}
	}

	return visited;
}
 

/** Creates a new instance using a specific initial permutation and specified number of threads.
 *
 * <p>If <code>exact</code> is true, the final permutation is
 * <em>exactly</em> the same as if you first permute the graph with <code>startPerm</code> and
 * then apply LLP with an {@code null} starting permutation.
 *
 * @param symGraph a symmetric, loopless graph.
 * @param startPerm an initial permutation of the graph, or {@code null} for no permutation.
 * @param numberOfThreads the number of threads to be used (0 for automatic sizing).
 * @param seed a random seed.
 * @param exact a boolean flag that forces the algorithm to run exactly.
 */
public LayeredLabelPropagation(final ImmutableGraph symGraph, final int[] startPerm, final int numberOfThreads, final long seed, final boolean exact) throws IOException {
	this.symGraph = symGraph;
	this.n = symGraph.numNodes();
	this.startPerm = startPerm;
	this.seed = seed;
	this.r = new XoRoShiRo128PlusRandom(seed);
	this.exact = exact;
	this.label = new AtomicIntegerArray(n);
	this.volume = new AtomicIntegerArray(n);
	cumulativeOutdegrees = new EliasFanoCumulativeOutdegreeList(symGraph, symGraph.numArcs(), 1);

	this.gapCost = new MutableDouble();
	this.updateList = Util.identity(n);
	simpleUncaughtExceptionHandler = new SimpleUncaughtExceptionHandler();
	labelling = File.createTempFile(this.getClass().getName(), "labelling");
	labelling.deleteOnExit();

	this.numberOfThreads = numberOfThreads != 0 ? numberOfThreads : Runtime.getRuntime().availableProcessors();
	this.canChange = new boolean[n];
	this.modified = new AtomicInteger(0);
	this.objectiveFunction = new double[this.numberOfThreads];
}
 

public CallGraphData(final ImmutableGraph graph, final ImmutableGraph transpose, final Properties graphProperties, final Properties transposeProperties, final long[] LID2GID, final Long2IntOpenHashMap GID2LID, final int nInternal, final int size) {
	super();
	this.graph = graph;
	this.transpose = transpose;
	this.graphProperties = graphProperties;
	this.transposeProperties = transposeProperties;
	this.LID2GID = LID2GID;
	this.GID2LID = GID2LID;
	this.externalNodes = new LongOpenHashSet(Arrays.copyOfRange(LID2GID, nInternal, LID2GID.length));
	this.size = size;
}
 

/** Builds the server.
 *
 * @param graph the graph.
 * @param map a string map representing the URLs of the graph.
 */
public ImmutableGraphNamedGraphServer(final ImmutableGraph graph, StringMap<? extends CharSequence> map) {
	if (graph.numNodes() != map.size()) throw new IllegalArgumentException("The graph has " + graph.numNodes() + " nodes, but the map has " + map.size() + " keys.");
	this.graph = graph;
	this.map = map;
	this.list = map.list();
}
 

public CallGraphData(final ImmutableGraph graph, final ImmutableGraph transpose, final Properties graphProperties, final Properties transposeProperties, final long[] LID2GID, final GOV3LongFunction GID2LID, final int nInternal, final int size) {
	super();
	this.graph = graph;
	this.transpose = transpose;
	this.graphProperties = graphProperties;
	this.transposeProperties = transposeProperties;
	this.LID2GID = LID2GID;
	this.GID2LID = GID2LID;
	this.externalNodes = new LongOpenHashSet(Arrays.copyOfRange(LID2GID, nInternal, LID2GID.length));
	this.size = size;
}
 

public static ImmutableGraph get(String lang, IndexType type)
{
	String key = buildKey(lang, type);
	ImmutableGraph g = graphs.get(key);
	if (g == null)
	{
		synchronized(WikiGraphs.class)
		{
			if ((g=graphs.get(key)) == null)
			{
				log.debug("["+key+"] Loading ...");
				try {
					switch(type){
					case GRAPH:
						String path = RepositoryDirs.GRAPH.getPath(lang)+"/graph";//"/graph";
						g = ImmutableGraph.load(path);
						break;
					case IN_GRAPH:
						String inpath =RepositoryDirs.IN_GRAPH.getPath(lang)+"/in_graph";
						g = ImmutableGraph.load(inpath);
						break;
					default:
						throw new RestoringException(key+": Unknown graph type!");
					}
					log.debug("["+key+"] Graph loaded. Memory: "/*+Utils.memSize(false)*/);
					graphs.put(key, g);
				} catch (IOException ioe){
					throw new RestoringException(key,ioe);
				}
			}
		}
	}
	return g;

}
 

public ImmutableGraph rawGraph() {
	return graph;
}
 

public static void main(String[] args) throws IOException {

		TagmeConfig.init();
		String lang = "en";

		RelatednessMeasure rel = RelatednessMeasure.create(lang);

		TopicSearcher searcher = new TopicSearcher(lang);

		int wid1 = 8485; // Wikipedia ID of the page "Diego Maradona"
		int wid2 = 808402; // Wikipedia ID of the page
							// "Mexico national football team"
		System.out.println("\n\nThe relatedness between "
				+ searcher.getTitle(wid1) + " and " + searcher.getTitle(wid2)
				+ " is " + rel.rel(wid1, wid2));

		int[][] in_graph = DatasetLoader.get(new InGraphArray(lang));

		System.out.println("\n\nThe number of pages in Wikipedia is "
				+ in_graph.length);
		System.out.println("The number of pages linking to "
				+ searcher.getTitle(8485) + " is " + in_graph[8485].length);

		int count = 0;
		for (int i = 0; i < in_graph.length; i++)
			if (searcher.getTitle(i) != null) {
				count++;
				// System.out.println(searcher.getTitle(i));
			}
		System.out.println("The number of actual unique pages in Wikipedia is "
				+ count);

		ImmutableGraph in_webgraph = WikiGraphs.get(lang, IndexType.IN_GRAPH);
		System.out.println("\n\nThe number of pages in Wikipedia is "
				+ in_webgraph.numNodes());
		System.out.println("The number of pages linking to "
				+ searcher.getTitle(wid1) + " is "
				+ in_webgraph.outdegree(wid1));

		ImmutableGraph webgraph = WikiGraphs.get(lang, IndexType.GRAPH);
		System.out.println("The number of pages in Wikipedia is "
				+ webgraph.numNodes());
		System.out.println("The number of pages linked by "
				+ searcher.getTitle(wid1) + " is " + webgraph.outdegree(wid1));

		System.out.println("\n\n\nThe pages linking to "
				+ searcher.getTitle(wid1) + " are the following:");
		int[] linkingTo = in_graph[wid1];
		for (int wid : linkingTo) {
			System.out.println("Page: " + searcher.getTitle(wid) + "\trel: "
					+ rel.rel(wid1, wid));
		}

	}
 

private IterationThread(final ImmutableGraph symGraph, final double gamma, final int index, final ProgressLogger pl) {
	this.symGraph = symGraph;
	this.gamma = gamma;
	this.index = index;
	this.pl = pl;
}
 

@Override
public void run() {
	final ImmutableGraph symGraph = this.symGraph;
	final int numNodes = LayeredLabelPropagation.this.n;
	final long numArcs = LayeredLabelPropagation.this.symGraph.numArcs();
	final int[] perm = this.perm;
	int[] permutedSuccessors = new int[32];
	int[] successors;
	final long granularity = Math.max(1024, numArcs >>> 9);
	int start, end;

	double gapCost = 0;
	for (;;) {

		// Try to get another piece of work.
		synchronized(LayeredLabelPropagation.this.cumulativeOutdegrees) {
			if (nextNode == numNodes) {
				LayeredLabelPropagation.this.gapCost.add(gapCost);
				break;
			}
			start = nextNode;
			final long target = nextArcs + granularity;
			if (target >= numArcs) nextNode = numNodes;
			else {
				nextArcs = cumulativeOutdegrees.skipTo(target);
				nextNode = cumulativeOutdegrees.currentIndex();
			}
			end = nextNode;
		}

		final NodeIterator nodeIterator = symGraph.nodeIterator(start);
		for (int i = start; i < end; i++) {
			nodeIterator.nextInt();
			final int node = perm[i];
			final int outdegree = nodeIterator.outdegree();
			if (outdegree > 0) {
				successors = nodeIterator.successorArray();
				permutedSuccessors = IntArrays.grow(permutedSuccessors, outdegree);
				for (int j = outdegree; j-- != 0;)
					permutedSuccessors[j] = perm[successors[j]];
				IntArrays.quickSort(permutedSuccessors, 0, outdegree);
				int prev = node;
				for (int j = 0; j < outdegree; j++) {
					gapCost += Fast.ceilLog2(Math.abs(prev - permutedSuccessors[j]));
					prev = permutedSuccessors[j];
				}
			}
		}
	}
}
 

private GapCostThread(final ImmutableGraph symGraph, final int[] perm) {
	this.symGraph = symGraph;
	this.perm = perm;
}
 

public ImmutableGraph rawTranspose() {
	return transpose;
}
 

public ImmutableGraph rawGraph() {
	return graph;
}
 

/**
 * Return the permutation induced by the visit order of a breadth-first visit.
 *
 * @param graph a graph.
 * @param startingNode the only starting node of the visit, or -1 for a complete visit.
 * @param internalNodes number of internal nodes in the graph
 * @return the permutation induced by the visit order of a breadth-first visit.
 */
public static int[] bfsperm(final ImmutableGraph graph, final int startingNode, final int internalNodes) {
	final int n = graph.numNodes();

	final int[] visitOrder = new int[n];
	Arrays.fill(visitOrder, -1);
	final IntArrayFIFOQueue queue = new IntArrayFIFOQueue();
	final LongArrayBitVector visited = LongArrayBitVector.ofLength(n);
	final ProgressLogger pl = new ProgressLogger(LOGGER);
	pl.expectedUpdates = n;
	pl.itemsName = "nodes";
	pl.start("Starting breadth-first visit...");

	int internalPos = 0, externalPos = internalNodes;

	for (int i = 0; i < n; i++) {
		final int start = i == 0 && startingNode != -1 ? startingNode : i;
		if (visited.getBoolean(start)) continue;
		queue.enqueue(start);
		visited.set(start);

		int currentNode;
		final IntArrayList successors = new IntArrayList();

		while (!queue.isEmpty()) {
			currentNode = queue.dequeueInt();
			if (currentNode < internalNodes) visitOrder[internalPos++] = currentNode;
			else visitOrder[externalPos++] = currentNode;
			int degree = graph.outdegree(currentNode);
			final LazyIntIterator iterator = graph.successors(currentNode);

			successors.clear();
			while (degree-- != 0) {
				final int succ = iterator.nextInt();
				if (!visited.getBoolean(succ)) {
					successors.add(succ);
					visited.set(succ);
				}
			}

			final int[] randomSuccessors = successors.elements();
			IntArrays.quickSort(randomSuccessors, 0, successors.size(), (x, y) -> x - y);

			for (int j = successors.size(); j-- != 0;)
				queue.enqueue(randomSuccessors[j]);
			pl.update();
		}

		if (startingNode != -1) break;
	}

	pl.done();
	for (int i = 0; i < visitOrder.length; i++)
		assert (i < internalNodes) == (visitOrder[i] < internalNodes);
	return visitOrder;
}
 

public ImmutableGraph rawTranspose() {
	return transpose;
}
 

/** Creates a new instance using a specific initial permutation.
 *
 * <p>If <code>exact</code> is true, the final permutation is
 * <em>exactly</em> the same as if you first permute the graph with <code>startPerm</code> and
 * then apply LLP with an {@code null} starting permutation.
 *
 * @param symGraph a symmetric, loopless graph.
 * @param startPerm an initial permutation of the graph, or {@code null} for no permutation.
 * @param seed a random seed.
 * @param exact a boolean flag that forces the algorithm to run exactly.
 */
public LayeredLabelPropagation(final ImmutableGraph symGraph, final int[] startPerm, final long seed, final boolean exact) throws IOException {
	this(symGraph, startPerm, 0, seed, exact);
}
 

/** Creates a new instance using a specific initial permutation.
 *
 * @param symGraph a symmetric, loopless graph.
 * @param startPerm an initial permutation of the graph, or {@code null} for no permutation.
 * @param seed a random seed.
 */
public LayeredLabelPropagation(final ImmutableGraph symGraph, final int[] startPerm, final long seed) throws IOException {
	this(symGraph, startPerm, seed, false);
}
 

/** Creates a new instance.
 *
 * @param symGraph a symmetric, loopless graph.
 * @param seed a random seed.
 */
public LayeredLabelPropagation(final ImmutableGraph symGraph, final long seed) throws IOException {
	this(symGraph, null, seed, false);
}