it.unimi.dsi.logging.ProgressLogger Java Examples

/**
 * Computes the labels of a graph for a given value of γ.
 *
 * @param gamma the gamma parameter.
 * @param maxUpdates the maximum number of updates performed.
 * @return the labels.
 */
public AtomicIntegerArray computeLabels(final double gamma, final int maxUpdates) {
	init();
	final String gammaFormatted = GAMMA_FORMAT.format(gamma);
	double prevObjFun = 0;
	double gain = 0;
	final ProgressLogger pl = new ProgressLogger(LOGGER, "updates");
	pl.logger().info("Running " + this.numberOfThreads + " threads");
	pl.start("Starting iterations with gamma=" + gammaFormatted + "...");

	update = 0;

	do {
		prevObjFun = objectiveFunction();
		update(gamma);
		pl.updateAndDisplay();
		gain = 1 - (prevObjFun / objectiveFunction());
		LOGGER.info("Gain: " + gain);
		LOGGER.info("Modified: " + modified.get());
		update++;
	} while (modified.get() > 0 && gain > GAIN_TRESHOLD && update < maxUpdates);

	pl.done();

	return label;
}
 

public static void main(String[] arg) throws IOException, JSAPException {
	final SimpleJSAP jsap = new SimpleJSAP(GZIPIndexer.class.getName(), "Computes and stores a quasi-succinct index for a compressed archive.",
			new Parameter[] {
				new UnflaggedOption("archive", JSAP.STRING_PARSER, JSAP.REQUIRED, "The name a GZIP's archive."),
				new UnflaggedOption("index", JSAP.STRING_PARSER, JSAP.REQUIRED, "The output (a serialized LongBigList of pointers to the records in the archive) filename."),
			}
	);

	final JSAPResult jsapResult = jsap.parse(arg);
	if (jsap.messagePrinted()) return;

	final FastBufferedInputStream input = new FastBufferedInputStream(new FileInputStream(jsapResult.getString("archive")));
	ProgressLogger pl = new ProgressLogger(LOGGER, 1, TimeUnit.MINUTES, "records");
	pl.start("Scanning...");
	final EliasFanoMonotoneLongBigList list = new EliasFanoMonotoneLongBigList(index(input, pl));
	pl.done();
	BinIO.storeObject(list, jsapResult.getString("index"));
}
 

/** Creates the thread.
 *
 * @param frontier the frontier instantiating the thread.
 * @param distributor the distributor used.
 */
public StatsThread(final Frontier frontier, final Distributor distributor) {
	this.frontier = frontier;
	this.distributor = distributor;

	requestLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "requests");
	requestLogger.displayFreeMemory = requestLogger.displayLocalSpeed = true;
	requestLogger.speedTimeUnit = TimeUnit.SECONDS;
	requestLogger.itemTimeUnit = TimeUnit.MILLISECONDS;

	resourceLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "resources");
	resourceLogger.displayLocalSpeed = true;
	resourceLogger.speedTimeUnit = TimeUnit.SECONDS;
	resourceLogger.itemTimeUnit = TimeUnit.MILLISECONDS;

	transferredBytesLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "bytes");
	transferredBytesLogger.displayLocalSpeed = true;
	transferredBytesLogger.speedTimeUnit = TimeUnit.SECONDS;
	transferredBytesLogger.itemTimeUnit = TimeUnit.NANOSECONDS;

	receivedURLsLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "receivedURLs");
	receivedURLsLogger.displayLocalSpeed = true;
	receivedURLsLogger.speedTimeUnit = TimeUnit.SECONDS;
}
 

/**
 * Returns a list of pointers to a GZIP archive entries positions (including the end of file).
 *
 * @param in the stream from which to read the GZIP archive.
 * @param pl a progress logger.
 * @return a list of longs where the <em>i</em>-th long is the offset in the stream of the first byte of the <em>i</em>-th archive entry.
 */
public static LongBigArrayBigList index(final InputStream in, final ProgressLogger pl) throws IOException {
	final LongBigArrayBigList pointers = new LongBigArrayBigList();
	long current = 0;
	final GZIPArchiveReader gzar = new GZIPArchiveReader(in);
	GZIPArchive.ReadEntry re;
	for (;;) {
		re = gzar.skipEntry();
		if (re == null) break;
		pointers.add(current);
		current += re.compressedSkipLength;
		if (pl != null) pl.lightUpdate();
	}
	in.close();
	return pointers;
}
 

@Override
public Void call() throws Exception {
	final ProgressLogger pl = new ProgressLogger(LOGGER, 1, TimeUnit.MINUTES, "records");

	pl.itemsName = "records";
	pl.displayFreeMemory = true;
	pl.displayLocalSpeed = true;
	pl.start("Scanning...");

	try {
		for(;;) {
			final Result<?>[] result = queue.take();
			if (result == Result.END_OF_RESULTS) {
				pl.done();
				return null;
			}
			for (final Result<?> r : result) if (r != null) r.write();
			pl.lightUpdate();
		}
	}
	catch(final Exception e) {
		LOGGER.error("Exception in flushing thread", e);
		throw e;
	}
}
 

public void testLineIterator( ProgressLogger pl ) {
	final LineIterator lineIterator = new LineIterator( new FastBufferedReader( new StringReader( TEXT ) ), pl );
	int i = 0;
	while( lineIterator.hasNext() )
		assertEquals( LINES[ i++ ].toString(), lineIterator.next().toString() );

	assertEquals( i, LINES.length );
}
 

public static void main( final String[] arg ) throws IOException, JSAPException, NoSuchMethodException {
	
	final SimpleJSAP jsap = new SimpleJSAP( BloomFilter.class.getName(), "Creates a Bloom filter reading from standard input a newline-separated list of terms.",
			new Parameter[] {
				new FlaggedOption( "bufferSize", IntSizeStringParser.getParser(), "64Ki", JSAP.NOT_REQUIRED, 'b',  "buffer-size", "The size of the I/O buffer used to read terms." ),
				new FlaggedOption( "encoding", ForNameStringParser.getParser( Charset.class ), "UTF-8", JSAP.NOT_REQUIRED, 'e', "encoding", "The term file encoding." ),
				new UnflaggedOption( "bloomFilter", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The filename for the serialised front-coded list." ),
				new UnflaggedOption( "size", JSAP.INTSIZE_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The size of the filter (i.e., the expected number of elements in the filter; usually, the number of terms)." ),
				new UnflaggedOption( "precision", JSAP.INTEGER_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The precision of the filter." )
	});
	
	JSAPResult jsapResult = jsap.parse( arg );
	if ( jsap.messagePrinted() ) return;
	
	final int bufferSize = jsapResult.getInt( "bufferSize" );
	final String filterName = jsapResult.getString( "bloomFilter" );
	final Charset encoding = (Charset)jsapResult.getObject( "encoding" );

	BloomFilter filter = new BloomFilter( jsapResult.getInt( "size" ), jsapResult.getInt( "precision" ) );
	final ProgressLogger pl = new ProgressLogger();
	pl.itemsName = "terms";
	pl.start( "Reading terms..." );
	MutableString s = new MutableString();
	FastBufferedReader reader = new FastBufferedReader( new InputStreamReader( System.in, encoding ), bufferSize );
	while( reader.readLine( s ) != null ) { 
		filter.add( s );
		pl.lightUpdate();
	}
	pl.done();

	BinIO.storeObject( filter, filterName );
}
 

public static void main( final String[] arg ) throws IOException, JSAPException, NoSuchMethodException {

		final SimpleJSAP jsap = new SimpleJSAP( TernaryIntervalSearchTree.class.getName(), "Builds a ternary interval search tree reading from standard input a newline-separated list of terms.",
			new Parameter[] {
				new FlaggedOption( "bufferSize", JSAP.INTSIZE_PARSER, "64Ki", JSAP.NOT_REQUIRED, 'b',  "buffer-size", "The size of the I/O buffer used to read terms." ),
				new FlaggedOption( "encoding", ForNameStringParser.getParser( Charset.class ), "UTF-8", JSAP.NOT_REQUIRED, 'e', "encoding", "The term file encoding." ),
				new UnflaggedOption( "tree", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The filename for the serialised tree." )
		});

		JSAPResult jsapResult = jsap.parse( arg );
		if ( jsap.messagePrinted() ) return;

		final TernaryIntervalSearchTree tree = new TernaryIntervalSearchTree();
		
		MutableString term = new MutableString();
		final ProgressLogger pl = new ProgressLogger();
		pl.itemsName = "terms";
		final FastBufferedReader terms = new FastBufferedReader( new InputStreamReader( System.in, (Charset)jsapResult.getObject( "encoding" ) ), jsapResult.getInt( "bufferSize" ) );
				
		pl.start( "Reading terms..." );

		while( terms.readLine( term ) != null ) {
			pl.update();
			tree.add( term );
		}

		pl.done();

		BinIO.storeObject( tree, jsapResult.getString( "tree" ) );
	}
 

public static void main( final String[] arg ) throws IOException, JSAPException, NoSuchMethodException {
	
	final SimpleJSAP jsap = new SimpleJSAP( FrontCodedStringList.class.getName(), "Builds a front-coded string list reading from standard input a newline-separated ordered list of terms.",
			new Parameter[] {
				new FlaggedOption( "bufferSize", IntSizeStringParser.getParser(), "64Ki", JSAP.NOT_REQUIRED, 'b',  "buffer-size", "The size of the I/O buffer used to read terms." ),
				new FlaggedOption( "encoding", ForNameStringParser.getParser( Charset.class ), "UTF-8", JSAP.NOT_REQUIRED, 'e', "encoding", "The term file encoding." ),
				new FlaggedOption( "ratio", IntSizeStringParser.getParser(), "4", JSAP.NOT_REQUIRED, 'r',  "ratio", "The compression ratio." ),
				new Switch( "utf8", 'u', "utf8", "Store the strings as UTF-8 byte arrays." ),
				new Switch( "zipped", 'z', "zipped", "The term list is compressed in gzip format." ),
				new UnflaggedOption( "frontCodedList", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The filename for the serialised front-coded list." )
	});
	
	JSAPResult jsapResult = jsap.parse( arg );
	if ( jsap.messagePrinted() ) return;
	
	final int bufferSize = jsapResult.getInt( "bufferSize" );
	final int ratio = jsapResult.getInt( "ratio" );
	final boolean utf8 = jsapResult.getBoolean( "utf8" );
	final boolean zipped = jsapResult.getBoolean( "zipped" );
	final String listName = jsapResult.getString( "frontCodedList" );
	final Charset encoding = (Charset)jsapResult.getObject( "encoding" );
	
	final ProgressLogger pl = new ProgressLogger();
	pl.itemsName = "words";
	pl.start( "Reading words..." );
	final FrontCodedStringList frontCodedStringList = new FrontCodedStringList( new LineIterator( new FastBufferedReader( 
			new InputStreamReader( zipped ? new GZIPInputStream( System.in ) : System.in, encoding ), bufferSize ), pl ), ratio, utf8 );
	pl.done();

	System.err.print( "Writing to file..." );
	BinIO.storeObject( frontCodedStringList, listName );
	System.err.println( " done." );
}
 

@SuppressWarnings("resource")
public static int[] writeRecords(final String path, final int numRecords, final WarcRecord[] randomRecords, final int parallel) throws IOException, InterruptedException {
	final ProgressLogger pl = new ProgressLogger(LOGGER, "records");
	if (parallel <= 1) pl.expectedUpdates = numRecords;
	final ProgressLogger plb = new ProgressLogger(LOGGER, "KB");
	final CountingOutputStream cos = new CountingOutputStream(new FastBufferedOutputStream(new FileOutputStream (path)));
	final WarcWriter ww;
	if (parallel == 0) {
		ww = new UncompressedWarcWriter(cos);
		pl.start("Writing records…");
	} else if (parallel == 1) {
		ww = new CompressedWarcWriter(cos);
		pl.start("Writing records (compressed)…");
	} else {
		ww = null;
		pl.start("SHOULD NOT HAPPEN");
		throw new IllegalStateException();
	}
	plb.start();
	long written = 0;
	final int[] position = new int[numRecords];
	for (int i = 0; i < numRecords; i++) {
		final int pos = RandomTestMocks.RNG.nextInt(randomRecords.length);
		position[i] = pos;
		ww.write(randomRecords[pos]);
		if (parallel <= 0) {
			pl.lightUpdate();
			plb.update((cos.getCount() - written) / 1024);
		}
		written = cos.getCount();
	}
	ww.close();
	pl.done(numRecords);
	plb.done(cos.getCount());
	return position;
}
 

public static void main(String[] arg) throws IOException {
	if (arg.length == 0) {
		System.err.println("Usage: " + BuildRepetitionSet.class.getSimpleName() + " REPETITIONSET");
		System.exit(1);
	}

	final FastBufferedReader fastBufferedReader = new FastBufferedReader(new InputStreamReader(System.in, Charsets.US_ASCII));
	final MutableString s = new MutableString();
	final LongOpenHashSet repeatedSet = new LongOpenHashSet();
	final String outputFilename = arg[0];
	final ProgressLogger pl = new ProgressLogger();

	MutableString lastUrl = new MutableString();
	pl.itemsName = "lines";
	pl.start("Reading... ");
	while(fastBufferedReader.readLine(s) != null) {
		final int firstTab = s.indexOf('\t');
		final int secondTab = s.indexOf('\t', firstTab + 1);
		MutableString url = s.substring(secondTab + 1);
		if (url.equals(lastUrl)) {
			final int storeIndex = Integer.parseInt(new String(s.array(), 0, firstTab));
			final long storePosition = Long.parseLong(new String(s.array(), firstTab + 1, secondTab - firstTab - 1));
			repeatedSet.add((long)storeIndex << 48 | storePosition);
			System.out.print(storeIndex);
			System.out.print('\t');
			System.out.print(storePosition);
			System.out.print('\t');
			System.out.println(url);
		}

		lastUrl = url;
		pl.lightUpdate();
	}

	pl.done();

	fastBufferedReader.close();
	BinIO.storeObject(repeatedSet, outputFilename);
}
 

private void update(final double gamma) {
	final int n = this.n;
	final int[] updateList = this.updateList;
	modified.set(0);
	nextArcs = nextNode = 0;

	if (exact) {
		if (startPerm == null) Util.identity(updateList);
		else Util.invertPermutation(startPerm, updateList);
	}

	// Local shuffle
	for(int i = 0; i < n;) IntArrays.shuffle(updateList, i, Math.min(i += SHUFFLE_GRANULARITY, n), r);

	final ProgressLogger pl = new ProgressLogger(LOGGER);
	pl.expectedUpdates = n;
	pl.logInterval = ProgressLogger.TEN_SECONDS;
	pl.itemsName = "nodes";
	pl.start("Starting update " + update + "...");

	final Thread[] thread = new Thread[numberOfThreads];

	nextArcs = nextNode =  0;
	for (int i = 0; i < numberOfThreads; i++) {
		thread[i] = new IterationThread(symGraph.copy(), gamma, i, pl);
		thread[i].setUncaughtExceptionHandler(simpleUncaughtExceptionHandler);
		thread[i].start();
	}

	for (int i = 0; i < numberOfThreads; i++)
		try {
			thread[i].join();
		}
		catch (final InterruptedException e) {
			throw new RuntimeException(e);
		}

	if (threadException != null) throw new RuntimeException(threadException);
	pl.done();
}
 

public static void main(String arg[]) throws IOException, InterruptedException, JSAPException {

	SimpleJSAP jsap = new SimpleJSAP(WarcCompressor.class.getName(),
		"Given a store uncompressed, write a compressed store.",
		new Parameter[] { new FlaggedOption("output", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'o', "output", "The output filename  (- for stdout)."),
		new UnflaggedOption("store", JSAP.STRING_PARSER, JSAP.NOT_REQUIRED, "The name of the store (if omitted, stdin)."),
	});

	JSAPResult jsapResult = jsap.parse(arg);
	if (jsap.messagePrinted()) return;

	final InputStream in = jsapResult.userSpecified("store") ? new FastBufferedInputStream(new FileInputStream(jsapResult.getString("store"))) : System.in;

	final WarcReader reader = new UncompressedWarcReader(in);
	final ProgressLogger pl = new ProgressLogger(LOGGER, 1, TimeUnit.MINUTES, "records");
	final String output = jsapResult.getString("output");

	PrintStream out = "-".equals(output) ? System.out : new PrintStream(new FastBufferedOutputStream(new FileOutputStream(output)), false, "UTF-8");
	final WarcWriter writer = new CompressedWarcWriter(out);

	pl.itemsName = "records";
	pl.displayFreeMemory = true;
	pl.displayLocalSpeed = true;
	pl.start("Scanning...");

	for (long storePosition = 0;; storePosition++) {
	    LOGGER.trace("STOREPOSITION " + storePosition);
	    WarcRecord record = null;
	    try {
		record = reader.read();
	    } catch (Exception e) {
		LOGGER.error("Exception while reading record " + storePosition + " ");
		LOGGER.error(e.getMessage());
		e.printStackTrace();
		continue;
	    }
	    if (record == null)
		break;
	    writer.write(record);
	    pl.lightUpdate();
	}
	pl.done();
	writer.close();
    }
 

@Override
public void run() {
    ProgressLogger pl = new ProgressLogger();
    final int nwords = vectors.size();
    final int d = embeddingsSize;
    try {
        pl.count = count / nbThreads;
        pl.itemsName = "entities";
        final PrintWriter pw = new PrintWriter(new BufferedWriter(new FileWriter(output+"."+rank, false)));
        /*if (rank == 0)
            pw.println(count + " " + d);*/

        float alpha = 10;
        EntityEmbeddings eb = new EntityEmbeddings();
        final BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(input)) , "UTF-8"));
        pl.start();
        String line;
        int nb = 0;
        int nbWritten = 0;
        while((line = br.readLine()) != null) {
            if ((nb < (count/nbThreads)*rank)) {
                nb++; 
                continue;
            }

            if ((rank != nbThreads-1) && (nb > (count/nbThreads)*(rank+1)))
                break;

            String[] parts = line.split("\t");
            if(parts.length > 1) {
                TrainingExamples ex = eb.getVectors(parts[1], vectors, rho, nwords, maxWords);
                if (ex.y.length == 0) {
                    nb++;
                    continue;
                }
                float[] w = eb.trainLR2(ex.x, d, ex.y, alpha);
                pw.print(parts[0] + " ");
                for(int i = 0; i < d; i++) {
                    pw.print(w[i] + " ");
                }
                pw.println();
                pl.lightUpdate();
                nbWritten++;
                if (nbWritten == 1000) {
                    pw.flush();
                    nbWritten = 0;
                }

                for(int i = 0; i < ex.y.length; i++) {
                    if(ex.y[i] > 0) {
                        eb.scoreLR(ex.x[i], w);
                        //double v = eb.scoreLR(ex.x[i], w);
                    }
                }
            }
            nb++;
        }
        br.close();
        pw.close();
        pl.stop();
    } catch(Exception e) {
        e.printStackTrace();
    }
}
 

private IterationThread(final ImmutableGraph symGraph, final double gamma, final int index, final ProgressLogger pl) {
	this.symGraph = symGraph;
	this.gamma = gamma;
	this.index = index;
	this.pl = pl;
}
 

public static void main(final String[] args) throws JSAPException, ClassNotFoundException, RocksDBException, IOException {
	final SimpleJSAP jsap = new SimpleJSAP(ExtractProperties.class.getName(),
			"Extract properties files from a knowledge base.",
			new Parameter[] {
					new FlaggedOption("min", JSAP.INTEGER_PARSER, "0", JSAP.NOT_REQUIRED, 'm', "min", "Consider only graphs with at least this number of internal nodes."),
					new FlaggedOption("n", JSAP.LONG_PARSER, Long.toString(Long.MAX_VALUE), JSAP.NOT_REQUIRED, 'n', "n", "Analyze just this number of graphs."),
					new UnflaggedOption("kb", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The directory of the RocksDB instance containing the knowledge base." ),
					new UnflaggedOption("kbmeta", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The file containing the knowledge base metadata." ),
	});

	final JSAPResult jsapResult = jsap.parse(args);
	if ( jsap.messagePrinted() ) return;

	final int minNodes = jsapResult.getInt("min");
	final long n = jsapResult.getLong("n");
	final String kbDir = jsapResult.getString("kb");
	if (!new File(kbDir).exists()) throw new IllegalArgumentException("No such directory: " + kbDir);
	final String kbMetadataFilename = jsapResult.getString("kbmeta");
	if (!new File(kbMetadataFilename).exists()) throw new IllegalArgumentException("No such file: " + kbMetadataFilename);
	LOGGER.info("Loading KnowledgeBase metadata");
	final KnowledgeBase kb = KnowledgeBase.getInstance(kbDir, kbMetadataFilename, true);

	final ProgressLogger pl = new ProgressLogger();

	pl.count = kb.callGraphs.size();
	pl.itemsName = "graphs";
	pl.start("Enumerating graphs");

	long i = 0;
	for(final CallGraph callGraph: kb.callGraphs.values()) {
		if (i++ >= n) break;
		pl.update();
		if (callGraph.nInternal < minNodes) continue;
		final CallGraphData callGraphData = callGraph.callGraphData();
		System.out.print(callGraph.index);
		System.out.print('\t');
		System.out.print(callGraph.product);
		System.out.print('\t');
		System.out.print(callGraph.version);
		System.out.print('\t');
		System.out.print(callGraphData.graphProperties);
		System.out.print('\t');
		System.out.print(callGraphData.transposeProperties);
		System.out.println();
	}

	LOGGER.info("Closing KnowledgeBase");
	kb.close();
}
 

public static Result coreaches(final KnowledgeBase kb, final long startSig, final int maxRevs, final ProgressLogger pl) {
	final LongOpenHashSet result = new LongOpenHashSet();
	final Object2ObjectOpenHashMap<String, IntOpenHashSet> product2Revs = new Object2ObjectOpenHashMap<>();
	final MutableLong totRevs = new MutableLong();

	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	String p = kb.callGraphs.get(index(startSig)).product;
	IntOpenHashSet revs = new IntOpenHashSet();
	revs.add(index(startSig));
	product2Revs.put(p, revs);
	totRevs.increment();


	pl.itemsName = "nodes";
	pl.info = new Object() {
		@Override
		public String toString() {
			return "[nodes: " + result.size() + " products: " + product2Revs.size() + " revisions: " + totRevs.getValue() + "]";
		}
	};
	pl.start("Visiting coreachable nodes...");
	while (!queue.isEmpty()) {
		final long node = queue.dequeueLong();

		for (final long s : kb.predecessors(node)) if (!result.contains(s)) {
			p = kb.callGraphs.get(index(s)).product;
			final String targetNameSpace = kb.new Node(gid(s), index(s)).toFastenURI().getRawNamespace();
			if (targetNameSpace.startsWith("java.") || targetNameSpace.startsWith("javax.") || targetNameSpace.startsWith("jdk.")) continue;
			revs = product2Revs.get(p);
			if (revs == null) product2Revs.put(p, revs = new IntOpenHashSet());
			if (revs.contains(index(s)) || revs.size() < maxRevs) {
				queue.enqueue(s);
				result.add(s);
				//System.out.println(kb.new Node(gid(node), index(node)).toFastenURI() + " -> " + kb.new Node(gid(s), index(s)).toFastenURI());
				if (revs.add(index(s))) totRevs.increment();
			}
		}
		pl.lightUpdate();
	}

	pl.done();
	return new Result(result, product2Revs.size(), totRevs.getValue().longValue());
}
 

public void testLineIteratorProgressMeter() {
	testLineIterator( new ProgressLogger() );
}
 

public static Result reaches(final KnowledgeBase kb, final long startSig, final int maxRevs, final ProgressLogger pl) {
	final LongOpenHashSet result = new LongOpenHashSet();
	final Object2ObjectOpenHashMap<String, IntOpenHashSet> product2Revs = new Object2ObjectOpenHashMap<>();
	final MutableLong totRevs = new MutableLong();

	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	String p = kb.callGraphs.get(index(startSig)).product;
	IntOpenHashSet revs = new IntOpenHashSet();
	revs.add(index(startSig));
	product2Revs.put(p, revs);
	totRevs.increment();


	pl.itemsName = "nodes";
	pl.info = new Object() {
		@Override
		public String toString() {
			return "[nodes: " + result.size() + " products: " + product2Revs.size() + " revisions: " + totRevs.getValue() + "]";
		}
	};

	pl.start("Visiting reachable nodes...");

	while (!queue.isEmpty()) {
		final long node = queue.dequeueLong();

		for (final long s : kb.successors(node)) if (!result.contains(s)) {
			p = kb.callGraphs.get(index(s)).product;
			final long gid = gid(s);
			if (badGIDs.contains(gid)) continue;
			final String targetNameSpace = kb.new Node(gid, index(s)).toFastenURI().getRawNamespace();
			if (targetNameSpace.startsWith("java.") || targetNameSpace.startsWith("javax.") || targetNameSpace.startsWith("jdk.")) {
				badGIDs.add(gid);
				continue;
			}
			revs = product2Revs.get(p);
			if (revs == null) product2Revs.put(p, revs = new IntOpenHashSet());
			if (revs.contains(index(s)) || revs.size() < maxRevs) {
				queue.enqueue(s);
				result.add(s);
				//System.out.println(kb.new Node(gid(node), index(node)).toFastenURI() + " -> " + kb.new Node(gid(s), index(s)).toFastenURI());
				if (revs.add(index(s))) totRevs.increment();
			}
		}
		pl.lightUpdate();
	}

	pl.done();
	return new Result(result, product2Revs.size(), totRevs.getValue().longValue());
}
 

/** Creates a new line iterator over a specified fast buffered reader. 
 * 
 * @param fastBufferedReader the underlying buffered reader.
 * @param pl an optional progress logger, or <code>null</code>.
 * */
public LineIterator( final FastBufferedReader fastBufferedReader, final ProgressLogger pl ) {
	this.fastBufferedReader = fastBufferedReader;
	this.pl = pl;
}