Java Code Examples for it.unimi.dsi.fastutil.longs.LongOpenHashSet#add()

/**
 * The set of all node signatures that are reachable from the signature <code>startSig</code>.
 *
 * @param start the starting node.
 * @return the set of all node signatures for which there is a directed path from
 *         <code>startSig</code> to that node.
 */
public synchronized LongSet reaches(final long startSig) {
	final LongOpenHashSet result = new LongOpenHashSet();
	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	while (!queue.isEmpty()) {
		final long nodeSig = queue.dequeueLong();
		for (final long s : successors(nodeSig)) if (!result.contains(s)) {
			queue.enqueue(s);
			result.add(s);
		}
	}

	return result;
}
 

/**
 * The set of all nodes signatures that are coreachable from <code>startSig</code>.
 *
 * @param start the starting node signature.
 * @return the set of all node signatures for which there is a directed path from that node to
 *         <code>startSig</code>.
 */
public synchronized LongSet coreaches(final long startSig) {
	final LongOpenHashSet result = new LongOpenHashSet();
	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	while (!queue.isEmpty()) {
		final long nodeSig = queue.dequeueLong();
		for (final long s : predecessors(nodeSig)) if (!result.contains(s)) {
			queue.enqueue(s);
			result.add(s);
		}
	}

	return result;
}
 

@Test
public void testRunReturningPositive() {
  HigherBitsEdgeTypeMask higherBitsEdgeTypeMask = new HigherBitsEdgeTypeMask();
  OutIndexedPowerLawMultiSegmentDirectedGraph powerLawMultiSegmentDirectedGraph =
          new OutIndexedPowerLawMultiSegmentDirectedGraph(1249,
                  1249,
                  1249,
                  1249,
                  1249,
                  higherBitsEdgeTypeMask,
                  new NullStatsReceiver());
  LongOpenHashSet longOpenHashSet = new LongOpenHashSet();
  powerLawMultiSegmentDirectedGraph.addEdge(0L, 1170L, (byte) (-126));
  longOpenHashSet.add(0L);
  PageRank pageRank =
          new PageRank(powerLawMultiSegmentDirectedGraph, longOpenHashSet, 1249, 1249, 1249, 1249);
  int resultInt = pageRank.run();

  assertEquals(0.0, pageRank.getL1Norm(), 0.01);
  assertEquals(3, resultInt);
}
 

public void addArc(final long x, final long y) {
	if (!graph.containsKey(x)) throw new IllegalArgumentException("Node " + x + " is not in the node set");
	if (!graph.containsKey(y)) throw new IllegalArgumentException("Node " + y + " is not in the node set");
	final LongOpenHashSet successors = graph.get(x);
	if (!successors.add(y)) throw new IllegalArgumentException("Duplicate arc " + x + " -> " + y);
	if (numArcs * 2 + graph.size() >= Integer.MAX_VALUE - 8) throw new IllegalStateException("Graph is too large");
	numArcs++;
}
 

public static void main(String[] arg) throws IOException {
	if (arg.length == 0) {
		System.err.println("Usage: " + BuildRepetitionSet.class.getSimpleName() + " REPETITIONSET");
		System.exit(1);
	}

	final FastBufferedReader fastBufferedReader = new FastBufferedReader(new InputStreamReader(System.in, Charsets.US_ASCII));
	final MutableString s = new MutableString();
	final LongOpenHashSet repeatedSet = new LongOpenHashSet();
	final String outputFilename = arg[0];
	final ProgressLogger pl = new ProgressLogger();

	MutableString lastUrl = new MutableString();
	pl.itemsName = "lines";
	pl.start("Reading... ");
	while(fastBufferedReader.readLine(s) != null) {
		final int firstTab = s.indexOf('\t');
		final int secondTab = s.indexOf('\t', firstTab + 1);
		MutableString url = s.substring(secondTab + 1);
		if (url.equals(lastUrl)) {
			final int storeIndex = Integer.parseInt(new String(s.array(), 0, firstTab));
			final long storePosition = Long.parseLong(new String(s.array(), firstTab + 1, secondTab - firstTab - 1));
			repeatedSet.add((long)storeIndex << 48 | storePosition);
			System.out.print(storeIndex);
			System.out.print('\t');
			System.out.print(storePosition);
			System.out.print('\t');
			System.out.println(url);
		}

		lastUrl = url;
		pl.lightUpdate();
	}

	pl.done();

	fastBufferedReader.close();
	BinIO.storeObject(repeatedSet, outputFilename);
}
 

public void testConflicts() {
	BloomFilter bloomFilter = new BloomFilter( 1000, 11 ); // Low precision
	LongOpenHashSet longs = new LongOpenHashSet();
	Random random = new Random( 0 );
	
	for( int i = 1000; i-- != 0; ) {
		final long l = random.nextLong();
		longs.add( l );
		bloomFilter.add( Long.toBinaryString( l ) );
	}
	
	assertEquals( longs.size(), bloomFilter.size() );
}
 

LongRawValueBasedNotInPredicateEvaluator(NotInPredicate notInPredicate) {
  List<String> values = notInPredicate.getValues();
  _nonMatchingValues = new LongOpenHashSet(values.size());
  for (String value : values) {
    _nonMatchingValues.add(Long.parseLong(value));
  }
}
 

LongRawValueBasedInPredicateEvaluator(InPredicate inPredicate) {
  List<String> values = inPredicate.getValues();
  _matchingValues = new LongOpenHashSet(HashUtil.getMinHashSetSize(values.size()));
  for (String value : values) {
    _matchingValues.add(Long.parseLong(value));
  }
}
 

public static Result reaches(final KnowledgeBase kb, final long startSig, final int maxRevs, final ProgressLogger pl) {
	final LongOpenHashSet result = new LongOpenHashSet();
	final Object2ObjectOpenHashMap<String, IntOpenHashSet> product2Revs = new Object2ObjectOpenHashMap<>();
	final MutableLong totRevs = new MutableLong();

	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	String p = kb.callGraphs.get(index(startSig)).product;
	IntOpenHashSet revs = new IntOpenHashSet();
	revs.add(index(startSig));
	product2Revs.put(p, revs);
	totRevs.increment();


	pl.itemsName = "nodes";
	pl.info = new Object() {
		@Override
		public String toString() {
			return "[nodes: " + result.size() + " products: " + product2Revs.size() + " revisions: " + totRevs.getValue() + "]";
		}
	};

	pl.start("Visiting reachable nodes...");

	while (!queue.isEmpty()) {
		final long node = queue.dequeueLong();

		for (final long s : kb.successors(node)) if (!result.contains(s)) {
			p = kb.callGraphs.get(index(s)).product;
			final long gid = gid(s);
			if (badGIDs.contains(gid)) continue;
			final String targetNameSpace = kb.new Node(gid, index(s)).toFastenURI().getRawNamespace();
			if (targetNameSpace.startsWith("java.") || targetNameSpace.startsWith("javax.") || targetNameSpace.startsWith("jdk.")) {
				badGIDs.add(gid);
				continue;
			}
			revs = product2Revs.get(p);
			if (revs == null) product2Revs.put(p, revs = new IntOpenHashSet());
			if (revs.contains(index(s)) || revs.size() < maxRevs) {
				queue.enqueue(s);
				result.add(s);
				//System.out.println(kb.new Node(gid(node), index(node)).toFastenURI() + " -> " + kb.new Node(gid(s), index(s)).toFastenURI());
				if (revs.add(index(s))) totRevs.increment();
			}
		}
		pl.lightUpdate();
	}

	pl.done();
	return new Result(result, product2Revs.size(), totRevs.getValue().longValue());
}
 

public static Result coreaches(final KnowledgeBase kb, final long startSig, final int maxRevs, final ProgressLogger pl) {
	final LongOpenHashSet result = new LongOpenHashSet();
	final Object2ObjectOpenHashMap<String, IntOpenHashSet> product2Revs = new Object2ObjectOpenHashMap<>();
	final MutableLong totRevs = new MutableLong();

	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	String p = kb.callGraphs.get(index(startSig)).product;
	IntOpenHashSet revs = new IntOpenHashSet();
	revs.add(index(startSig));
	product2Revs.put(p, revs);
	totRevs.increment();


	pl.itemsName = "nodes";
	pl.info = new Object() {
		@Override
		public String toString() {
			return "[nodes: " + result.size() + " products: " + product2Revs.size() + " revisions: " + totRevs.getValue() + "]";
		}
	};
	pl.start("Visiting coreachable nodes...");
	while (!queue.isEmpty()) {
		final long node = queue.dequeueLong();

		for (final long s : kb.predecessors(node)) if (!result.contains(s)) {
			p = kb.callGraphs.get(index(s)).product;
			final String targetNameSpace = kb.new Node(gid(s), index(s)).toFastenURI().getRawNamespace();
			if (targetNameSpace.startsWith("java.") || targetNameSpace.startsWith("javax.") || targetNameSpace.startsWith("jdk.")) continue;
			revs = product2Revs.get(p);
			if (revs == null) product2Revs.put(p, revs = new IntOpenHashSet());
			if (revs.contains(index(s)) || revs.size() < maxRevs) {
				queue.enqueue(s);
				result.add(s);
				//System.out.println(kb.new Node(gid(node), index(node)).toFastenURI() + " -> " + kb.new Node(gid(s), index(s)).toFastenURI());
				if (revs.add(index(s))) totRevs.increment();
			}
		}
		pl.lightUpdate();
	}

	pl.done();
	return new Result(result, product2Revs.size(), totRevs.getValue().longValue());
}
 

public void evaluate(Transaction transaction, LongArrayList recommendations, LongOpenHashSet userTransactions) {
	//if there is no ground truth, there is nothing to evaluate
	if (userTransactions == null || userTransactions.isEmpty()) {
		return;
	}
	// if the algorithm does not return any recommendations, count it as 0
	if (recommendations.isEmpty()) {
		results.add(0);
		return;
	}

	// if the algorithm retrieves less than k recommendations, we calculate
	// the real k value for this case
	int realK = Math.min(k, recommendations.size());

	// check duplicates
	LongOpenHashSet uniqueRecs = new LongOpenHashSet();
	for (int i = 0; i < realK; i++) {
		if (!uniqueRecs.add(recommendations.getLong(i))) {
			throw new RuntimeException("Duplicate recommendation.");
		}
	}

	// calculate the precision
	double result = 0;
	// iterate over relevant items and recommendations to calculate the
	// intersection
	for (LongIterator iterator = userTransactions.iterator(); iterator.hasNext();) {
		long itemID = iterator.nextLong();
		for (int i = 0; i < realK; i++) {
			if (itemID == recommendations.getLong(i)) {
				result++;
			}
		}
	}

	//determine the divider of the fraction (different for precision and recall)
	double divider;
	if(type == Type.Precision){
		divider = realK;
	}else if(type == Type.Recall){
		divider = userTransactions.size();
	}else{
		throw new RuntimeException("Neither precision nor recall defined.");
	}
	// store the precision/Recall
	results.add(result / divider);
}
 

@Test
public void testLesMisGraph() throws Exception {
  OutIndexedPowerLawMultiSegmentDirectedGraph graph =
      new OutIndexedPowerLawMultiSegmentDirectedGraph(1, 1000, 100, 10, 2,
          new IdentityEdgeTypeMask(), new NullStatsReceiver());

  for (int i=0; i<LES_MIS_GRAPH.length; i++) {
    graph.addEdge(LES_MIS_GRAPH[i][0], LES_MIS_GRAPH[i][1], (byte) 0);
  }

  // Spot check the graph to make sure it's been loaded correctly.
  assertEquals(7, graph.getOutDegree(76));
  assertEquals(new LongArrayList(new long[]{64, 65, 66, 63, 62, 48, 58}), new LongArrayList(graph.getOutEdges(76)));

  assertEquals(1, graph.getOutDegree(30));
  assertEquals(new LongArrayList(new long[]{23}), new LongArrayList(graph.getOutEdges(30)));

  assertEquals(4, graph.getOutDegree(11));
  assertEquals(new LongArrayList(new long[]{10, 3, 2, 0}), new LongArrayList(graph.getOutEdges(11)));

  LongOpenHashSet nodes = new LongOpenHashSet();
  long maxNodeId = 0;
  for (int i=0; i<LES_MIS_GRAPH.length; i++) {
    if ( !nodes.contains(LES_MIS_GRAPH[i][0])) nodes.add(LES_MIS_GRAPH[i][0]);
    if ( !nodes.contains(LES_MIS_GRAPH[i][1])) nodes.add(LES_MIS_GRAPH[i][1]);
    if ( LES_MIS_GRAPH[i][0] > maxNodeId ) maxNodeId = LES_MIS_GRAPH[i][0];
    if ( LES_MIS_GRAPH[i][1] > maxNodeId ) maxNodeId = LES_MIS_GRAPH[i][1];
  }

  assertEquals(76, maxNodeId);
  PageRank pr = new PageRank(graph, nodes, maxNodeId, 0.85, 10, 1e-15);
  int numIterations = pr.run();
  double normL1 = pr.getL1Norm();
  double[] pagerank = pr.getPageRankVector();
  assertEquals(10, numIterations);
  assertEquals(0.00108, normL1, 10e-4);

  List<Map.Entry<Long, Double>> scores = new ArrayList<>();
  for (int i=0; i<maxNodeId+1; i++) {
    scores.add(new AbstractMap.SimpleEntry<>((long) i, pagerank[i]));
  }

  // Sort by score.
  scores.sort((e1, e2) -> e2.getValue() > e1.getValue() ? 1 : e2.getKey().compareTo(e1.getKey()));

  // We're going to verify that the ranking and score are both correct. These rankings have been verified against an
  // external implementation (JUNG).
  assertEquals(11, (long) scores.get(0).getKey());
  assertEquals(0.1088995, scores.get(0).getValue(), 10e-4);
  assertEquals(0, (long) scores.get(1).getKey());
  assertEquals(0.09538347, scores.get(1).getValue(), 10e-4);
  assertEquals(16, (long) scores.get(2).getKey());
  assertEquals(0.05104386, scores.get(2).getValue(), 10e-4);
  assertEquals(23, (long) scores.get(3).getKey());
  assertEquals(0.04389916, scores.get(3).getValue(), 10e-4);
  assertEquals(25, (long) scores.get(4).getKey());
  assertEquals(0.04095956, scores.get(4).getValue(), 10e-4);
  assertEquals(2, (long) scores.get(5).getKey());
  assertEquals(0.03868165, scores.get(5).getValue(), 10e-4);
  assertEquals(24, (long) scores.get(6).getKey());
  assertEquals(0.03617344, scores.get(6).getValue(), 10e-4);
  assertEquals(48, (long) scores.get(7).getKey());
  assertEquals(0.0290502, scores.get(7).getValue(), 10e-4);
  assertEquals(10, (long) scores.get(8).getKey());
  assertEquals(0.02714507, scores.get(8).getValue(), 10e-4);
  assertEquals(3, (long) scores.get(9).getKey());
  assertEquals(0.02714507, scores.get(9).getValue(), 10e-4);

  double totalMass = 0.0;
  for (int i=0; i<maxNodeId+1; i++) {
    totalMass += scores.get(i).getValue();
  }
  // Total mass should still be 1.0.
  assertEquals(1.0, totalMass, 10e-10);
}
 

@Test
public void testLesMisGraph() throws Exception {
  OutIndexedPowerLawMultiSegmentDirectedGraph graph =
      new OutIndexedPowerLawMultiSegmentDirectedGraph(1, 1000, 100, 10, 2,
          new IdentityEdgeTypeMask(), new NullStatsReceiver());

  for (int i=0; i<LES_MIS_GRAPH.length; i++) {
    graph.addEdge(LES_MIS_GRAPH[i][0], LES_MIS_GRAPH[i][1], (byte) 0);
  }

  // Spot check the graph to make sure it's been loaded correctly.
  assertEquals(7, graph.getOutDegree(76));
  assertEquals(new LongArrayList(new long[]{64, 65, 66, 63, 62, 48, 58}), new LongArrayList(graph.getOutEdges(76)));

  assertEquals(1, graph.getOutDegree(30));
  assertEquals(new LongArrayList(new long[]{23}), new LongArrayList(graph.getOutEdges(30)));

  assertEquals(4, graph.getOutDegree(11));
  assertEquals(new LongArrayList(new long[]{10, 3, 2, 0}), new LongArrayList(graph.getOutEdges(11)));

  LongOpenHashSet nodes = new LongOpenHashSet();
  long maxNodeId = 0;
  for (int i=0; i<LES_MIS_GRAPH.length; i++) {
    if ( !nodes.contains(LES_MIS_GRAPH[i][0])) nodes.add(LES_MIS_GRAPH[i][0]);
    if ( !nodes.contains(LES_MIS_GRAPH[i][1])) nodes.add(LES_MIS_GRAPH[i][1]);
    if ( LES_MIS_GRAPH[i][0] > maxNodeId ) maxNodeId = LES_MIS_GRAPH[i][0];
    if ( LES_MIS_GRAPH[i][1] > maxNodeId ) maxNodeId = LES_MIS_GRAPH[i][1];
  }

  assertEquals(76, maxNodeId);
  MultiThreadedPageRank pr = new MultiThreadedPageRank(graph, new LongArrayList(nodes), maxNodeId, 0.85, 10, 1e-15, 3);
  int numIterations = pr.run();
  double normL1 = pr.getL1Norm();
  AtomicDoubleArray pagerank = pr.getPageRankVector();
  assertEquals(10, numIterations);
  assertEquals(0.00108, normL1, 10e-4);

  List<Map.Entry<Long, Double>> scores = new ArrayList<>();
  for (int i=0; i<maxNodeId+1; i++) {
    scores.add(new AbstractMap.SimpleEntry<>((long) i, pagerank.get(i)));
  }

  // Sort by score.
  scores.sort((e1, e2) -> e2.getValue() > e1.getValue() ? 1 : e2.getKey().compareTo(e1.getKey()));

  // We're going to verify that the ranking and score are both correct. These rankings have been verified against an
  // external implementation (JUNG).
  assertEquals(11, (long) scores.get(0).getKey());
  assertEquals(0.1088995, scores.get(0).getValue(), 10e-4);
  assertEquals(0, (long) scores.get(1).getKey());
  assertEquals(0.09538347, scores.get(1).getValue(), 10e-4);
  assertEquals(16, (long) scores.get(2).getKey());
  assertEquals(0.05104386, scores.get(2).getValue(), 10e-4);
  assertEquals(23, (long) scores.get(3).getKey());
  assertEquals(0.04389916, scores.get(3).getValue(), 10e-4);
  assertEquals(25, (long) scores.get(4).getKey());
  assertEquals(0.04095956, scores.get(4).getValue(), 10e-4);
  assertEquals(2, (long) scores.get(5).getKey());
  assertEquals(0.03868165, scores.get(5).getValue(), 10e-4);
  assertEquals(24, (long) scores.get(6).getKey());
  assertEquals(0.03617344, scores.get(6).getValue(), 10e-4);
  assertEquals(48, (long) scores.get(7).getKey());
  assertEquals(0.0290502, scores.get(7).getValue(), 10e-4);
  assertEquals(10, (long) scores.get(8).getKey());
  assertEquals(0.02714507, scores.get(8).getValue(), 10e-4);
  assertEquals(3, (long) scores.get(9).getKey());
  assertEquals(0.02714507, scores.get(9).getValue(), 10e-4);

  double totalMass = 0.0;
  for (int i=0; i<maxNodeId+1; i++) {
    totalMass += scores.get(i).getValue();
  }
  // Total mass should still be 1.0.
  assertEquals(1.0, totalMass, 10e-10);
}
 

@BeforeClass
public void setUp()
    throws Exception {
  FileUtils.deleteQuietly(TEMP_DIR);

  IntOpenHashSet intSet = new IntOpenHashSet();
  while (intSet.size() < NUM_VALUES) {
    intSet.add(RANDOM.nextInt());
  }
  _intValues = intSet.toIntArray();
  Arrays.sort(_intValues);

  LongOpenHashSet longSet = new LongOpenHashSet();
  while (longSet.size() < NUM_VALUES) {
    longSet.add(RANDOM.nextLong());
  }
  _longValues = longSet.toLongArray();
  Arrays.sort(_longValues);

  FloatOpenHashSet floatSet = new FloatOpenHashSet();
  while (floatSet.size() < NUM_VALUES) {
    floatSet.add(RANDOM.nextFloat());
  }
  _floatValues = floatSet.toFloatArray();
  Arrays.sort(_floatValues);

  DoubleOpenHashSet doubleSet = new DoubleOpenHashSet();
  while (doubleSet.size() < NUM_VALUES) {
    doubleSet.add(RANDOM.nextDouble());
  }
  _doubleValues = doubleSet.toDoubleArray();
  Arrays.sort(_doubleValues);

  Set<String> stringSet = new HashSet<>();
  while (stringSet.size() < NUM_VALUES) {
    stringSet.add(RandomStringUtils.random(RANDOM.nextInt(MAX_STRING_LENGTH)).replace('\0', ' '));
  }
  _stringValues = stringSet.toArray(new String[NUM_VALUES]);
  Arrays.sort(_stringValues);

  Set<ByteArray> bytesSet = new HashSet<>();
  while (bytesSet.size() < NUM_VALUES) {
    byte[] bytes = new byte[BYTES_LENGTH];
    RANDOM.nextBytes(bytes);
    bytesSet.add(new ByteArray(bytes));
  }
  _bytesValues = bytesSet.toArray(new ByteArray[NUM_VALUES]);
  Arrays.sort(_bytesValues);

  try (SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(_intValues,
      new DimensionFieldSpec(INT_COLUMN_NAME, FieldSpec.DataType.INT, true), TEMP_DIR)) {
    dictionaryCreator.build();
  }

  try (SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(_longValues,
      new DimensionFieldSpec(LONG_COLUMN_NAME, FieldSpec.DataType.LONG, true), TEMP_DIR)) {
    dictionaryCreator.build();
  }

  try (SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(_floatValues,
      new DimensionFieldSpec(FLOAT_COLUMN_NAME, FieldSpec.DataType.FLOAT, true), TEMP_DIR)) {
    dictionaryCreator.build();
  }

  try (SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(_doubleValues,
      new DimensionFieldSpec(DOUBLE_COLUMN_NAME, FieldSpec.DataType.DOUBLE, true), TEMP_DIR)) {
    dictionaryCreator.build();
  }

  try (SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(_stringValues,
      new DimensionFieldSpec(STRING_COLUMN_NAME, FieldSpec.DataType.STRING, true), TEMP_DIR)) {
    dictionaryCreator.build();
    _numBytesPerStringValue = dictionaryCreator.getNumBytesPerEntry();
  }

  try (SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(_bytesValues,
      new DimensionFieldSpec(BYTES_COLUMN_NAME, FieldSpec.DataType.BYTES, true), TEMP_DIR)) {
    dictionaryCreator.build();
    assertEquals(dictionaryCreator.getNumBytesPerEntry(), BYTES_LENGTH);
  }
}