it.unimi.dsi.Util Java Examples

/** Creates a new instance using a specific initial permutation and specified number of threads.
 *
 * <p>If <code>exact</code> is true, the final permutation is
 * <em>exactly</em> the same as if you first permute the graph with <code>startPerm</code> and
 * then apply LLP with an {@code null} starting permutation.
 *
 * @param symGraph a symmetric, loopless graph.
 * @param startPerm an initial permutation of the graph, or {@code null} for no permutation.
 * @param numberOfThreads the number of threads to be used (0 for automatic sizing).
 * @param seed a random seed.
 * @param exact a boolean flag that forces the algorithm to run exactly.
 */
public LayeredLabelPropagation(final ImmutableGraph symGraph, final int[] startPerm, final int numberOfThreads, final long seed, final boolean exact) throws IOException {
	this.symGraph = symGraph;
	this.n = symGraph.numNodes();
	this.startPerm = startPerm;
	this.seed = seed;
	this.r = new XoRoShiRo128PlusRandom(seed);
	this.exact = exact;
	this.label = new AtomicIntegerArray(n);
	this.volume = new AtomicIntegerArray(n);
	cumulativeOutdegrees = new EliasFanoCumulativeOutdegreeList(symGraph, symGraph.numArcs(), 1);

	this.gapCost = new MutableDouble();
	this.updateList = Util.identity(n);
	simpleUncaughtExceptionHandler = new SimpleUncaughtExceptionHandler();
	labelling = File.createTempFile(this.getClass().getName(), "labelling");
	labelling.deleteOnExit();

	this.numberOfThreads = numberOfThreads != 0 ? numberOfThreads : Runtime.getRuntime().availableProcessors();
	this.canChange = new boolean[n];
	this.modified = new AtomicInteger(0);
	this.objectiveFunction = new double[this.numberOfThreads];
}
 

/** Creates a new Mercator-like sieve.
 *
 * @param sieveIsNew whether we are creating a new sieve or opening an old one.
 * @param sieveDir a directory for storing the sieve files.
 * @param sieveSize the size of the size in longs.
 * @param storeIOBufferSize the size in bytes of the buffer used to read and write the hash store during flushes (allocated twice during flushes).
 * @param auxFileIOBufferSize the size in bytes of the buffer used to read and write the auxiliary file (always allocated; another allocation happens during flushes).
 * @param newFlowReceiver a receiver for the flow of new keys.
 * @param keySerDeser a serializer/deserializer for keys.
 * @param valueSerDeser a serializer/deserializer for values.
 * @param hashingStrategy a hashing strategy for keys.
 * @param updateStrategy the strategy used to update the values associated to duplicate keys.
 */
public MercatorSieve(final boolean sieveIsNew, final File sieveDir, final int sieveSize, final int storeIOBufferSize, final int auxFileIOBufferSize, final NewFlowReceiver<K> newFlowReceiver, final ByteSerializerDeserializer<K> keySerDeser, final ByteSerializerDeserializer<V> valueSerDeser,
		final AbstractHashFunction<K> hashingStrategy, final UpdateStrategy<K, V> updateStrategy)
		throws IOException {
	super(keySerDeser, valueSerDeser, hashingStrategy, updateStrategy);

	LOGGER.info("Creating Mercator sieve of size " + sieveSize + " (" + Util.formatSize2(sieveSize * 12L) + " bytes), store I/O buffer size " + storeIOBufferSize + " and aux-file I/O buffer size " + auxFileIOBufferSize);

	setNewFlowRecevier(newFlowReceiver);

	if ((storeIOBufferSize & 0x7) != 0) throw new IllegalArgumentException("Store I/O buffer size length must be a multiple of 8");

	bucket = new Bucket<>(sieveSize, auxFileIOBufferSize, sieveDir, keySerDeser);
	store = new Store(sieveIsNew, sieveDir, "store", storeIOBufferSize);
	position = new int[sieveSize];
}
 

@Test
public void testBlocking() throws InterruptedException {
	for(final int size: new int[] { 10, 100, 128, 256 }) {
		for(final int d: new int[] { 1, 2, 3, 4 }) {
			final ReorderingBlockingQueue<Integer> q = new ReorderingBlockingQueue<>(size / d);
			final int[] perm = Util.identity(size);
			IntArrays.shuffle(perm, new XoRoShiRo128PlusRandom());
			for(int i = perm.length; i-- != 0;) {
				final int t = perm[i];
				new Thread() {
					@Override
					public void run() {
						try {
							q.put(Integer.valueOf(t), t);
						}
						catch (final InterruptedException e) {
							throw new RuntimeException(e.getMessage(), e);
						}
					}
				}.start();
			}
			for(int i = 0; i < perm.length; i++) assertEquals(i, q.take().intValue());
			assertEquals(0, q.size());
		}
	}
}
 

public void testFormatBinarySize() {
	assertEquals( "1", Util.formatBinarySize( 1 ) );
	assertEquals( "2", Util.formatBinarySize( 2 ) );
	boolean ok = false;
	try {
		Util.formatBinarySize( 6 );
	}
	catch( IllegalArgumentException e ) {
		ok = true;
	}
	assertTrue( ok );
	assertEquals( "128", Util.formatBinarySize( 128 ) );
	assertEquals( "1Ki", Util.formatBinarySize( 1024 ) );
	assertEquals( "2Ki", Util.formatBinarySize( 2048 ) );
	assertEquals( "1Mi", Util.formatBinarySize( 1024 * 1024 ) );
	assertEquals( "2Mi", Util.formatBinarySize( 2 * 1024 * 1024 ) );
	assertEquals( "1Gi", Util.formatBinarySize( 1024 * 1024 * 1024 ) );
	assertEquals( "2Gi", Util.formatBinarySize( 2L * 1024 * 1024 * 1024 ) );
	assertEquals( "1Ti", Util.formatBinarySize( 1024L * 1024 * 1024 * 1024 ) );
	assertEquals( "2Ti", Util.formatBinarySize( 2L * 1024 * 1024 * 1024 * 1024 ) );
}
 

private void update(final double gamma) {
	final int n = this.n;
	final int[] updateList = this.updateList;
	modified.set(0);
	nextArcs = nextNode = 0;

	if (exact) {
		if (startPerm == null) Util.identity(updateList);
		else Util.invertPermutation(startPerm, updateList);
	}

	// Local shuffle
	for(int i = 0; i < n;) IntArrays.shuffle(updateList, i, Math.min(i += SHUFFLE_GRANULARITY, n), r);

	final ProgressLogger pl = new ProgressLogger(LOGGER);
	pl.expectedUpdates = n;
	pl.logInterval = ProgressLogger.TEN_SECONDS;
	pl.itemsName = "nodes";
	pl.start("Starting update " + update + "...");

	final Thread[] thread = new Thread[numberOfThreads];

	nextArcs = nextNode =  0;
	for (int i = 0; i < numberOfThreads; i++) {
		thread[i] = new IterationThread(symGraph.copy(), gamma, i, pl);
		thread[i].setUncaughtExceptionHandler(simpleUncaughtExceptionHandler);
		thread[i].start();
	}

	for (int i = 0; i < numberOfThreads; i++)
		try {
			thread[i].join();
		}
		catch (final InterruptedException e) {
			throw new RuntimeException(e);
		}

	if (threadException != null) throw new RuntimeException(threadException);
	pl.done();
}
 

/** Emits the statistics. */
public void emit() {
	requestLogger.setAndDisplay(frontier.fetchedResources.get() + frontier.fetchedRobots.get());
	final long duplicates = frontier.duplicates.get();
	final long archetypes = frontier.archetypes();
	resourceLogger.setAndDisplay(archetypes + duplicates);
	transferredBytesLogger.setAndDisplay(frontier.transferredBytes.get());
	receivedURLsLogger.setAndDisplay(frontier.numberOfReceivedURLs.get());

	LOGGER.info("Duplicates: " + Util.format(duplicates) + " (" + Util.format(100.0 * duplicates / (duplicates + archetypes)) + "%)");
	LOGGER.info("Archetypes 1XX/2XX/3XX/4XX/5XX/Other: "
			+ Util.format(frontier.archetypesStatus[1].get()) + "/"
			+ Util.format(frontier.archetypesStatus[2].get()) + "/"
			+ Util.format(frontier.archetypesStatus[3].get()) + "/"
			+ Util.format(frontier.archetypesStatus[4].get()) + "/"
			+ Util.format(frontier.archetypesStatus[5].get()) + "/"
			+ Util.format(frontier.archetypesStatus[0].get()));

	LOGGER.info("Outdegree stats: " + frontier.outdegree.toString());
	LOGGER.info("External outdegree stats: " + frontier.externalOutdegree.toString());
	LOGGER.info("Archetype content-length stats: " + frontier.contentLength.toString());
	LOGGER.info("Archetypes text/image/application/other: "
			+ Util.format(frontier.contentTypeText.get()) + "/"
			+ Util.format(frontier.contentTypeImage.get()) + "/"
			+ Util.format(frontier.contentTypeApplication.get()) + "/"
			+ Util.format(frontier.contentTypeOthers.get()));

	LOGGER.info("Ready URLs: " + Util.format(frontier.readyURLs.size64()));

	LOGGER.info("FetchingThread waits: " + frontier.fetchingThreadWaits.get() + "; total wait time: " + frontier.fetchingThreadWaitingTimeSum.get());
	frontier.resetFetchingThreadsWaitingStats();
}
 

@Test
public void testNoBlocking() throws InterruptedException {
	for(final int size: new int[] { 1, 10, 100, 128, 256 }) {
		final ReorderingBlockingQueue<Integer> q = new ReorderingBlockingQueue<>(size);
		final int[] perm = Util.identity(size);
		IntArrays.shuffle(perm, new XoRoShiRo128PlusRandom());
		for(int i = perm.length; i-- != 0;) q.put(Integer.valueOf(perm[i]), perm[i]);
		for(int i = 0; i < perm.length; i++) assertEquals(i, q.take().intValue());
		assertEquals(0, q.size());
	}
}
 

private String freeMemory() {
	return ( displayFreeMemory ? "; used/avail/free/total/max mem: " 
			+ Util.formatSize( RUNTIME.totalMemory() - RUNTIME.freeMemory() ) + "/" 
			+ Util.formatSize( RUNTIME.freeMemory() + ( RUNTIME.maxMemory() - RUNTIME.totalMemory() ) ) + "/" 
			+ Util.formatSize( RUNTIME.freeMemory() ) + "/" 
			+ Util.formatSize( RUNTIME.totalMemory() ) + "/" 
			+ Util.formatSize( RUNTIME.maxMemory() ) : "" );
}
 

private void updateInternal() {
	final long currentTime = System.currentTimeMillis();
	final long millisToEnd = Math.round( ( expectedUpdates - count ) * ( ( currentTime - start ) / ( count + 1.0 ) ) );
	// Formatting is expensive, so we check for actual logging.
	if ( logger.isEnabledFor( priority ) ) 
		logger.log( priority, Util.format( count ) + " " + itemsName + ", " + 
				millis2hms( millis() ) + ", " + Util.format( ( count * 1000.0 ) / ( currentTime - start ) ) + " " + itemsName + 
				"/s"  + ( expectedUpdates > 0 ? "; " + Util.format( ( 100 * count ) / expectedUpdates ) + "% done, " + 
				millis2hms( millisToEnd ) + " to end" : "" ) + freeMemory() + ( info != null ? "; " + info : "" ) );

	lastLog = currentTime;
	return;
}
 

public void testFormatSize() {
	assertEquals( "1", Util.formatSize( 1 ) );
	assertEquals( "2", Util.formatSize( 2 ) );
	assertEquals( "128", Util.formatSize( 128 ) );
	assertEquals( "1.00K", Util.formatSize( 1000 ) );
	assertEquals( "2.00K", Util.formatSize( 2000 ) );
	assertEquals( "2.50K", Util.formatSize( 2500 ) );
	assertEquals( "1.00M", Util.formatSize( 1000 * 1000 ) );
	assertEquals( "2.00M", Util.formatSize( 2 * 1000 * 1000 ) );
	assertEquals( "1.00G", Util.formatSize( 1000 * 1000 * 1000 ) );
	assertEquals( "2.00G", Util.formatSize( 2L * 1000 * 1000 * 1000 ) );
	assertEquals( "1.00T", Util.formatSize( 1000L * 1000 * 1000 * 1000 ) );
	assertEquals( "2.00T", Util.formatSize( 2L * 1000 * 1000 * 1000 * 1000 ) );
}
 

/** Generates <code>np</code> call graphs. Each call graph is obtained using {@link #preferentialAttachmentDAG(int, int, IntegerDistribution, RandomGenerator)} (with
 *  specified initial graph size (<code>initialGraphSizeDistribution</code>), graph size (<code>graphSizeDistribution</code>), outdegree distribution (<code>outdegreeDistribution</code>).
 *  Then a dependency DAG is generated between the call graphs, once more using {@link #preferentialAttachmentDAG(int, int, IntegerDistribution, RandomGenerator)} (this
 *  time the initial graph size is 1, whereas the outdegree distribution is <code>outdegreeDistribution</code>).
 *  Then to each node of each call graph a new set of outgoing arcs is generated (their number is chosen using <code>externalOutdegreeDistribution</code>): the target
 *  call graph is generated using the indegree distribution of the dependency DAG; the target node is chosen according to the reverse indegree distribution within the revision call graph.
 *
 * @param np number of revision call graphs to be generated.
 * @param graphSizeDistribution the distribution of the graph sizes (number of functions per call graph).
 * @param initialGraphSizeDistribution the distribution of the initial graph sizes (the initial independent set from which the preferential attachment starts).
 * @param outdegreeDistribution the distribution of internal outdegrees (number of internal calls per function).
 * @param externalOutdegreeDistribution the distribution of external outdegrees (number of external calls per function).
 * @param depExponent exponent of the Zipf distribution used to establish the dependencies between call graphs.
 * @param random the random object used for the generation.
 */
public void generate(final int np, final IntegerDistribution graphSizeDistribution, final IntegerDistribution initialGraphSizeDistribution,
		final IntegerDistribution outdegreeDistribution, final IntegerDistribution externalOutdegreeDistribution, final IntegerDistribution dependencyOutdegreeDistribution, final RandomGenerator random) {
	rcgs = new ArrayListMutableGraph[np];
	nodePermutation = new int[np][];
	final FenwickTree[] td = new FenwickTree[np];
	deps = new IntOpenHashSet[np];
	source2Targets = new ObjectOpenCustomHashSet[np];

	// Generate rcg of the np revisions, and the corresponding reverse preferential distribution; cumsize[i] is the sum of all nodes in packages <i
	for ( int i = 0; i < np; i++) {
		deps[i] = new IntOpenHashSet();
		final int n = graphSizeDistribution.sample();
		final int n0 = Math.min(initialGraphSizeDistribution.sample(), n);
		rcgs[i] = preferentialAttachmentDAG(n, n0, outdegreeDistribution, random);
		td[i] = getPreferentialDistribution(rcgs[i].immutableView(), true);
		nodePermutation[i] = Util.identity(n);
		Collections.shuffle(IntArrayList.wrap(nodePermutation[i]), new Random(random.nextLong()));
	}

	// Generate the dependency DAG between revisions using preferential attachment starting from 1 node
	final ArrayListMutableGraph depDAG = preferentialAttachmentDAG(np, 1, dependencyOutdegreeDistribution, random);

	// For each source package, generate function calls so to cover all dependencies
	for (int sourcePackage = 0; sourcePackage < np; sourcePackage++) {
		source2Targets[sourcePackage] = new ObjectOpenCustomHashSet<>(IntArrays.HASH_STRATEGY);
		final int outdegree = depDAG.outdegree(sourcePackage);
		if (outdegree == 0) continue; // No calls needed (I'm kinda busy)

		final int numFuncs = rcgs[sourcePackage].numNodes();
		final int[] externalArcs = new int[numFuncs];
		int allExternalArcs = 0;
		// We decide how many calls to dispatch from each function
		for (int sourceNode = 0; sourceNode < numFuncs; sourceNode++) allExternalArcs += (externalArcs[sourceNode] = externalOutdegreeDistribution.sample());
		// We create a global list of external successors by shuffling
		final int[] targetPackage = new int[allExternalArcs];
		final int[] succ = depDAG.successorArray(sourcePackage);
		for(int i = 0; i < outdegree; i++) deps[sourcePackage].add(succ[i]);
		for(int i = 0; i < allExternalArcs; i++) targetPackage[i] = succ[i % outdegree];
		MathArrays.shuffle(targetPackage, random);

		for (int sourceNode = allExternalArcs = 0; sourceNode < numFuncs; sourceNode++) {
			final int externalOutdegree = externalArcs[sourceNode];
			for (int t = 0; t < externalOutdegree; t++) {
				final int targetNode = td[targetPackage[allExternalArcs + t]].sample(random) - 1;
				source2Targets[sourcePackage].add(new int[] { sourceNode, targetPackage[allExternalArcs + t], targetNode });
			}
			allExternalArcs += externalOutdegree;
		}
	}
}
 

@Override
public String toString() {
	return "[size: " + Util.format(size) + " min: " + min + " max: " + max + " \u03BC: " + mean() + " \u03C3: " + sampleStandardDeviation() + " ("
			+ Util.format(100 * sampleRelativeStandardDeviation()) + " %)]";
}
 

/** Converts the data stored in this progress logger to a string. 
 * 
 * @return the data in this progress logger in a printable form.
 */

public String toString() {
	final long t = stop - start + 1 ;

	if ( t <= 0 ) return "Illegal progress logger state";

	return "Elapsed: " + millis2hms( t ) + ( count != 0 ? " [" + Util.format( count ) + " " + itemsName + ", " + Util.format( count / ( t / 1000.0 ) ) + " " + itemsName + "/s]" : "" );
}
 

/** Creates a Warc parallel output stream using 2×{@link Runtime#availableProcessors()} buffers.
 *
 * @param outputStream the final output stream.
 * @param compress whether to write compressed records.
 */
public ParallelBufferedWarcWriter(final OutputStream outputStream, final boolean compress) {
	this(outputStream, compress, 2 * Util.RUNTIME.availableProcessors());
}