it.unimi.dsi.fastutil.longs.LongOpenHashSet Java Examples

@Override
public void evaluate(Transaction transaction, LongArrayList recommendations, LongOpenHashSet userTransactions) {
	if(precision==null){
		//if the delegation objects are not yet created, create them
		precision = new PrecisionOrRecall();
		precision.setType(Type.Precision);
		precision.setK(k);
		
		recall = new PrecisionOrRecall();
		recall.setType(Type.Recall);
		recall.setK(k);
	}
	
	//delegate the work to Precision and Recall instances
	precision.evaluate(transaction, recommendations, userTransactions);
	recall.evaluate(transaction, recommendations, userTransactions);
}
 

@Test
public void databaseTest2() throws IOException, RocksDBException {
    var json = new JSONObject("{" +
            "\"index\": 2," +
            "\"product\": \"test\"," +
            "\"version\": \"0.0.1\"," +
            "\"nodes\": [255, 256, 257, 258]," +
            "\"numInternalNodes\": 3," +
            "\"edges\": [[255, 256], [255, 258], [256, 257], [257, 258]]" +
            "}");
    var graph = GidGraph.getGraph(json);
    rocksDao.saveToRocksDb(graph.getIndex(), graph.getNodes(), graph.getNumInternalNodes(), graph.getEdges());
    var graphData = rocksDao.getGraphData(graph.getIndex());
    assertEquals(graph.getNumInternalNodes(), graphData.nodes().size() - graphData.externalNodes().size());
    assertEquals(graph.getNodes().size(), graphData.nodes().size());
    assertEquals(new LongOpenHashSet(graph.getNodes()), graphData.nodes());
    assertEquals(new LongArrayList(List.of(256L, 258L)), graphData.successors(255L));
    assertEquals(new LongArrayList(List.of(257L)), graphData.successors(256L));
    assertEquals(new LongArrayList(List.of(258L)), graphData.successors(257L));
    assertEquals(new LongArrayList(), graphData.predecessors(255L));
    assertEquals(new LongArrayList(List.of(255L)), graphData.predecessors(256L));
    assertEquals(new LongArrayList(List.of(256L)), graphData.predecessors(257L));
    assertEquals(new LongArrayList(List.of(255L, 257L)), graphData.predecessors(258L));
    assertEquals(graph.getEdges().size(), graphData.numArcs());
    assertEquals(new LongOpenHashSet(List.of(258L)), graphData.externalNodes());
}
 

@Test
public void databaseTest1() throws IOException, RocksDBException {
    var json = new JSONObject("{" +
            "\"index\": 1," +
            "\"product\": \"test\"," +
            "\"version\": \"0.0.1\"," +
            "\"nodes\": [0, 1, 2]," +
            "\"numInternalNodes\": 2," +
            "\"edges\": [[0, 1], [1, 2]]" +
            "}");
    var graph = GidGraph.getGraph(json);
    rocksDao.saveToRocksDb(graph.getIndex(), graph.getNodes(), graph.getNumInternalNodes(), graph.getEdges());
    var graphData = rocksDao.getGraphData(graph.getIndex());
    assertEquals(graph.getNumInternalNodes(), graphData.nodes().size() - graphData.externalNodes().size());
    assertEquals(graph.getNodes().size(), graphData.nodes().size());
    assertEquals(new LongOpenHashSet(graph.getNodes()), graphData.nodes());
    assertEquals(new LongArrayList(List.of(1L)), graphData.successors(0L));
    assertEquals(new LongArrayList(List.of(2L)), graphData.successors(1L));
    assertEquals(new LongArrayList(List.of(0L)), graphData.predecessors(1L));
    assertEquals(new LongArrayList(List.of(1L)), graphData.predecessors(2L));
    assertEquals(graph.getEdges().size(), graphData.numArcs());
    assertEquals(new LongOpenHashSet(List.of(2L)), graphData.externalNodes());
}
 

@Test
public void databaseTest3() throws IOException, RocksDBException {
    var json = new JSONObject("{" +
            "\"index\": 3," +
            "\"product\": \"test\"," +
            "\"version\": \"0.0.1\"," +
            "\"nodes\": [255, 256, 257, 258]," +
            "\"numInternalNodes\": 4," +
            "\"edges\": [[255, 256], [255, 258], [256, 257], [257, 258]]" +
            "}");
    var graph = GidGraph.getGraph(json);
    rocksDao.saveToRocksDb(graph.getIndex(), graph.getNodes(), graph.getNumInternalNodes(), graph.getEdges());
    var graphData = rocksDao.getGraphData(graph.getIndex());
    assertEquals(graph.getNumInternalNodes(), graphData.nodes().size() - graphData.externalNodes().size());
    assertEquals(graph.getNodes().size(), graphData.nodes().size());
    assertEquals(new LongOpenHashSet((graph.getNodes())), new LongOpenHashSet(graphData.nodes()));
    assertEquals(new LongArrayList(List.of(256L, 258L)), graphData.successors(255L));
    assertEquals(new LongArrayList(List.of(257L)), graphData.successors(256L));
    assertEquals(new LongArrayList(List.of(258L)), graphData.successors(257L));
    assertEquals(new LongArrayList(List.of()), graphData.predecessors(255L));
    assertEquals(new LongArrayList(List.of(255L)), graphData.predecessors(256L));
    assertEquals(new LongArrayList(List.of(256L)), graphData.predecessors(257L));
    assertEquals(new LongArrayList(List.of(255L, 257L)), graphData.predecessors(258L));
    assertEquals(graph.getEdges().size(), graphData.numArcs());
    assertEquals(new LongOpenHashSet(), graphData.externalNodes());
}
 

@Test
public void databaseTest4() throws IOException, RocksDBException {
    var json = new JSONObject("{" +
            "\"index\": 4," +
            "\"product\": \"test\"," +
            "\"version\": \"0.0.1\"," +
            "\"nodes\": [255, 256, 257, 258]," +
            "\"numInternalNodes\": 0," +
            "\"edges\": [[255, 256], [255, 258], [256, 257], [257, 258]]" +
            "}");
    var graph = GidGraph.getGraph(json);
    rocksDao.saveToRocksDb(graph.getIndex(), graph.getNodes(), graph.getNumInternalNodes(), graph.getEdges());
    var graphData = rocksDao.getGraphData(graph.getIndex());
    assertEquals(graph.getNumInternalNodes(), graphData.nodes().size() - graphData.externalNodes().size());
    assertEquals(graph.getNodes().size(), graphData.nodes().size());
    assertEquals(new LongOpenHashSet((graph.getNodes())), new LongOpenHashSet(graphData.nodes()));
    assertEquals(new LongArrayList(List.of(256L, 258L)), graphData.successors(255L));
    assertEquals(new LongArrayList(List.of(257L)), graphData.successors(256L));
    assertEquals(new LongArrayList(List.of(258L)), graphData.successors(257L));
    assertEquals(new LongArrayList(List.of()), graphData.predecessors(255L));
    assertEquals(new LongArrayList(List.of(255L)), graphData.predecessors(256L));
    assertEquals(new LongArrayList(List.of(256L)), graphData.predecessors(257L));
    assertEquals(new LongArrayList(List.of(255L, 257L)), graphData.predecessors(258L));
    assertEquals(graph.getEdges().size(), graphData.numArcs());
    assertEquals(new LongOpenHashSet(graph.getNodes()), graphData.externalNodes());
}
 

public TweetAuthorFilter(
    LeftIndexedMultiSegmentBipartiteGraph leftIndexedBipartiteGraph,
    LongSet whitelistTweetAuthors,
    LongSet blacklistTweetAuthors,
    StatsReceiver statsReceiver) {
  super(statsReceiver);
  this.isIgnoreWhitelist = whitelistTweetAuthors.isEmpty();
  if (this.isIgnoreWhitelist) {
    this.whitelistedTweets = new LongOpenHashSet();
    this.blacklistedTweets = getTweetsByAuthors(leftIndexedBipartiteGraph, blacklistTweetAuthors);
  } else {
    // Performance hack. Remove blacklisted authors from the whitelist, and only check whitelist
    LongSet dedupedWhitelistAuthors = dedupWhitelistAuthors(whitelistTweetAuthors, blacklistTweetAuthors);
    this.whitelistedTweets = getTweetsByAuthors(leftIndexedBipartiteGraph, dedupedWhitelistAuthors);
    this.blacklistedTweets = new LongOpenHashSet();
  }
}
 

/**
 * Return the list of tweets authored by the input list of users
 */
private LongSet getTweetsByAuthors(
    LeftIndexedMultiSegmentBipartiteGraph leftIndexedBipartiteGraph,
    LongSet tweetAuthors) {
  LongSet authoredTweets = new LongOpenHashSet();
  for (long authorId: tweetAuthors) {
    EdgeIterator edgeIterator = leftIndexedBipartiteGraph.getLeftNodeEdges(authorId);
    if (edgeIterator == null) {
      continue;
    }

    // Sequentially iterating through the latest MAX_EDGES_PER_NODE edges per node
    int numEdgesPerNode = 0;
    while (edgeIterator.hasNext() && numEdgesPerNode++ < RecommendationRequest.MAX_EDGES_PER_NODE) {
      long rightNode = edgeIterator.nextLong();
      byte edgeType = edgeIterator.currentEdgeType();
      if (edgeType == RecommendationRequest.AUTHOR_SOCIAL_PROOF_TYPE) {
        authoredTweets.add(rightNode);
      }
    }
  }
  return authoredTweets;
}
 

/**
 * The set of all nodes signatures that are coreachable from <code>startSig</code>.
 *
 * @param start the starting node signature.
 * @return the set of all node signatures for which there is a directed path from that node to
 *         <code>startSig</code>.
 */
public synchronized LongSet coreaches(final long startSig) {
	final LongOpenHashSet result = new LongOpenHashSet();
	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	while (!queue.isEmpty()) {
		final long nodeSig = queue.dequeueLong();
		for (final long s : predecessors(nodeSig)) if (!result.contains(s)) {
			queue.enqueue(s);
			result.add(s);
		}
	}

	return result;
}
 

/**
 * The set of all node signatures that are reachable from the signature <code>startSig</code>.
 *
 * @param start the starting node.
 * @return the set of all node signatures for which there is a directed path from
 *         <code>startSig</code> to that node.
 */
public synchronized LongSet reaches(final long startSig) {
	final LongOpenHashSet result = new LongOpenHashSet();
	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	while (!queue.isEmpty()) {
		final long nodeSig = queue.dequeueLong();
		for (final long s : successors(nodeSig)) if (!result.contains(s)) {
			queue.enqueue(s);
			result.add(s);
		}
	}

	return result;
}
 

@Override
protected void trainInternal(List<Item> items, List<ClickData> transactions) {
	//iterate over all newly published items
	for (Item item : items) {
		//if the keywords of the item are not null, create a keyword map an save it
		if(item.keywords!=null){
			//create an empty bitmap of keyword IDs
			EWAHCompressedBitmap itemKeywords = new EWAHCompressedBitmap();
			itemKeywordMap.put(item.id, itemKeywords);
			//iterate over the item's keywords
			for (String keyword : item.keywords.keySet()) {				
				//if the keyword is unknown, generate a new INT id for it
				if(!keywordMap.containsKey(keyword)){
					keywordMap.put(keyword, ++currentKeywordCounter);
					keywordItemMap.put(currentKeywordCounter, new LongOpenHashSet());
				}
				//extract the id of the keyword
				int keywordInt = keywordMap.get(keyword);
				//set the bit for this keyword in the item's bitmap
				itemKeywords.set(keywordInt);
				//also set the item in the keyword's reverse lookup map
				keywordItemMap.get(keywordInt).add(item.id);
			}
		}
	}
}
 

@Override
public void evaluate(Transaction transaction, LongArrayList recommendations, LongOpenHashSet userTransactions) {
	if(precision==null){
		//if the delegation objects are not yet created, create them
		precision = new PrecisionOrRecall();
		precision.setType(Type.Precision);
		precision.setK(k);
		
		recall = new PrecisionOrRecall();
		recall.setType(Type.Recall);
		recall.setK(k);
	}
	
	//delegate the work to Precision and Recall instances
	precision.evaluate(transaction, recommendations, userTransactions);
	recall.evaluate(transaction, recommendations, userTransactions);
}
 

@Override
public LongColumn unique() {
  final LongSet values = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    values.add(getLong(i));
  }
  final LongColumn column = LongColumn.create(name() + " Unique values");
  for (long value : values) {
    column.append(value);
  }
  return column;
}
 

@Override
public int countUnique() {
  LongSet uniqueElements = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    uniqueElements.add(getLong(i));
  }
  return uniqueElements.size();
}
 

@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 

@Override
public InstantColumn unique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  InstantColumn column = emptyCopy(ints.size());
  column.setName(name() + " Unique values");
  column.data = LongArrayList.wrap(ints.toLongArray());
  return column;
}
 

@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 

@Override
public DateTimeColumn unique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  DateTimeColumn column = emptyCopy(ints.size());
  column.setName(name() + " Unique values");
  column.data = LongArrayList.wrap(ints.toLongArray());
  return column;
}
 

public void testConflicts() {
	BloomFilter bloomFilter = new BloomFilter( 1000, 11 ); // Low precision
	LongOpenHashSet longs = new LongOpenHashSet();
	Random random = new Random( 0 );
	
	for( int i = 1000; i-- != 0; ) {
		final long l = random.nextLong();
		longs.add( l );
		bloomFilter.add( Long.toBinaryString( l ) );
	}
	
	assertEquals( longs.size(), bloomFilter.size() );
}
 

/**
 * Get a new instance of a fresh internal state.
 *
 * @param salsaStats          is the stats object to use
 * @param expectedNodesToHit  is the number of nodes the random walk is expected to hit
 */
public CommonInternalState(
    SalsaStats salsaStats,
    int expectedNodesToHit) {
  this.salsaStats = salsaStats;
  this.currentLeftNodes = new Long2IntOpenHashMap(expectedNodesToHit);
  this.currentRightNodes = new Long2IntOpenHashMap(expectedNodesToHit);
  this.visitedRightNodes = new Long2ObjectOpenHashMap<NodeInfo>(expectedNodesToHit);
  this.nonZeroSeedSet = new LongOpenHashSet(expectedNodesToHit);
}
 

private static boolean isLessThanMinUserSocialProofSizeCombined(
  SmallArrayBasedLongToDoubleMap[] socialProofs,
  int minUserSocialProofSize,
  Set<byte[]> socialProofTypeUnions) {
  if (socialProofTypeUnions.isEmpty() ||
    // check if the size of any social proof union is greater than minUserSocialProofSize before dedupping
    isSocialProofUnionSizeLessThanMin(socialProofs, minUserSocialProofSize, socialProofTypeUnions)) {
    return true;
  }

  LongSet uniqueNodes = new LongOpenHashSet(minUserSocialProofSize);

  for (byte[] socialProofTypeUnion: socialProofTypeUnions) {
    // Clear removes all elements, but does not change the size of the set.
    // Thus, we only use one LongOpenHashSet with at most a size of 2*minUserSocialProofSize
    uniqueNodes.clear();
    for (byte socialProofType: socialProofTypeUnion) {
      if (socialProofs[socialProofType] != null) {
        for (int i = 0; i < socialProofs[socialProofType].size(); i++) {
          uniqueNodes.add(socialProofs[socialProofType].keys()[i]);
          if (uniqueNodes.size() >= minUserSocialProofSize) {
            return false;
          }
        }
      }
    }
  }
  return true;
}
 

/**
 * Constructs this object for running PageRank over a directed graph.
 *
 * @param graph          the directed graph
 * @param nodes          nodes in the graph
 * @param maxNodeId      maximum node id
 * @param dampingFactor  damping factor
 * @param maxIterations  maximum number of iterations to run
 * @param tolerance      L1 norm threshold for convergence
 */
public PageRank(OutIndexedDirectedGraph graph, LongOpenHashSet nodes, long maxNodeId,
                double dampingFactor, int maxIterations, double tolerance) {
  if (maxNodeId > Integer.MAX_VALUE) {
    throw new UnsupportedOperationException("maxNodeId exceeds Integer.MAX_VALUE!");
  }

  this.graph = graph;
  this.nodes = nodes;
  this.maxNodeId = maxNodeId;
  this.dampingFactor = dampingFactor;
  this.nodeCount = nodes.size();
  this.maxIterations = maxIterations;
  this.tolerance = tolerance;
}
 

@Override
public DateTimeColumn unique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  DateTimeColumn column = emptyCopy(ints.size());
  column.setName(name() + " Unique values");
  column.data = LongArrayList.wrap(ints.toLongArray());
  return column;
}
 

@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 

@Override
public InstantColumn unique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  InstantColumn column = emptyCopy(ints.size());
  column.setName(name() + " Unique values");
  column.data = LongArrayList.wrap(ints.toLongArray());
  return column;
}
 

@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 

@Override
public int countUnique() {
  LongSet uniqueElements = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    uniqueElements.add(getLong(i));
  }
  return uniqueElements.size();
}
 

@Override
public LongColumn unique() {
  final LongSet values = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    values.add(getLong(i));
  }
  final LongColumn column = LongColumn.create(name() + " Unique values");
  for (long value : values) {
    column.append(value);
  }
  return column;
}
 

public static void main(String[] arg) throws IOException {
	if (arg.length == 0) {
		System.err.println("Usage: " + BuildRepetitionSet.class.getSimpleName() + " REPETITIONSET");
		System.exit(1);
	}

	final FastBufferedReader fastBufferedReader = new FastBufferedReader(new InputStreamReader(System.in, Charsets.US_ASCII));
	final MutableString s = new MutableString();
	final LongOpenHashSet repeatedSet = new LongOpenHashSet();
	final String outputFilename = arg[0];
	final ProgressLogger pl = new ProgressLogger();

	MutableString lastUrl = new MutableString();
	pl.itemsName = "lines";
	pl.start("Reading... ");
	while(fastBufferedReader.readLine(s) != null) {
		final int firstTab = s.indexOf('\t');
		final int secondTab = s.indexOf('\t', firstTab + 1);
		MutableString url = s.substring(secondTab + 1);
		if (url.equals(lastUrl)) {
			final int storeIndex = Integer.parseInt(new String(s.array(), 0, firstTab));
			final long storePosition = Long.parseLong(new String(s.array(), firstTab + 1, secondTab - firstTab - 1));
			repeatedSet.add((long)storeIndex << 48 | storePosition);
			System.out.print(storeIndex);
			System.out.print('\t');
			System.out.print(storePosition);
			System.out.print('\t');
			System.out.println(url);
		}

		lastUrl = url;
		pl.lightUpdate();
	}

	pl.done();

	fastBufferedReader.close();
	BinIO.storeObject(repeatedSet, outputFilename);
}
 

@Override
public void evaluate(Transaction transaction, LongArrayList recommendations,
		LongOpenHashSet userTransactions) {
	//if there is no ground truth, there is nothing to evaluate
	if (userTransactions == null || userTransactions.isEmpty()) {
		return;
	}
	
	// if the algorithm does not return any recommendations, count it as 0
	if (recommendations.isEmpty()) {
		results.add(0);
		return;
	}

	// calculate the MRR
	// if the algorithm retrieves less than k recommendations, we calculate
	// the real k value for this case
	int realK = Math.min(k, recommendations.size());
	// iterate over relevant items and calculate recall rank
	for (Long itemID : userTransactions) {
		for (int i = 0; i < realK; i++) {
			if (itemID == recommendations.getLong(i)) {
				results.add(1d/(i+1));
				return;
			}
		}
	}
	//nothing found -> count as zero
	results.add(0);
}
 

@Override
public void evaluate(Transaction transaction, LongArrayList recommendations,
		LongOpenHashSet userTransactions) {
	//just count either k, or the actual number of recommendation 
	//in case the algorithm did not provide k recommendations
	results.add(Math.min(recommendations.size(), k));
}