it.unimi.dsi.fastutil.longs.LongSet Java Examples

The following examples show how to use it.unimi.dsi.fastutil.longs.LongSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TweetAuthorFilterTest.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
@Test
public void testBlacklistWhitelistNoOverlap() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();
  whitelistAuthors.add(1L);
  whitelistAuthors.add(2L);

  LongSet blacklistAuthors = new LongArraySet();
  blacklistAuthors.add(3L);
  blacklistAuthors.add(4L);

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // Should only return whitelisted 10 and 20. 30 and 40 are filtered by blacklist
  assertEquals(false, authorFilter.filterResult(10, socialProofs));
  assertEquals(false, authorFilter.filterResult(20, socialProofs));
  assertEquals(true, authorFilter.filterResult(30, socialProofs));
  assertEquals(true, authorFilter.filterResult(40, socialProofs));
}
 
Example #2
Source File: KnowledgeBase.java    From fasten with Apache License 2.0 6 votes vote down vote up
/**
 * The set of all node signatures that are reachable from the signature <code>startSig</code>.
 *
 * @param start the starting node.
 * @return the set of all node signatures for which there is a directed path from
 *         <code>startSig</code> to that node.
 */
public synchronized LongSet reaches(final long startSig) {
	final LongOpenHashSet result = new LongOpenHashSet();
	// Visit queue
	final LongArrayFIFOQueue queue = new LongArrayFIFOQueue();
	queue.enqueue(startSig);
	result.add(startSig);

	while (!queue.isEmpty()) {
		final long nodeSig = queue.dequeueLong();
		for (final long s : successors(nodeSig)) if (!result.contains(s)) {
			queue.enqueue(s);
			result.add(s);
		}
	}

	return result;
}
 
Example #3
Source File: ArrayUnionFunction.java    From presto with Apache License 2.0 6 votes vote down vote up
private static void appendBigintArray(Block array, AtomicBoolean containsNull, LongSet set, BlockBuilder blockBuilder)
{
    for (int i = 0; i < array.getPositionCount(); i++) {
        if (array.isNull(i)) {
            if (!containsNull.get()) {
                containsNull.set(true);
                blockBuilder.appendNull();
            }
            continue;
        }
        long value = BIGINT.getLong(array, i);
        if (set.add(value)) {
            BIGINT.writeLong(blockBuilder, value);
        }
    }
}
 
Example #4
Source File: TopSecondDegreeByCountRequestForUser.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
/**
 * @param queryNode                 is the query node for running TopSecondDegreeByCountForUser
 * @param leftSeedNodesWithWeight   is the set of seed nodes and their weights to use for calculation
 * @param toBeFiltered              is the list of users to be excluded from recommendations
 * @param maxNumResults             is the maximum number of recommendations returned in the response
 * @param maxNumSocialProofs        is the maximum number of social proofs per recommendation
 * @param maxSocialProofTypeSize    is the number of social proof types in the graph
 * @param minUserPerSocialProof     for each social proof, require a minimum number of users to be valid
 * @param socialProofTypes          is the list of valid social proofs, (i.e, Follow, Mention, Mediatag)
 * @param maxRightNodeAgeInMillis   is the max right node age in millisecond, such as tweet age
 * @param maxEdgeAgeInMillis        is the max edge age in millisecond such as reply edge age
 * @param resultFilterChain         is the chain of filters to be applied
 */
public TopSecondDegreeByCountRequestForUser(
  long queryNode,
  Long2DoubleMap leftSeedNodesWithWeight,
  LongSet toBeFiltered,
  int maxNumResults,
  int maxNumSocialProofs,
  int maxSocialProofTypeSize,
  Map<Byte, Integer> minUserPerSocialProof,
  byte[] socialProofTypes,
  long maxRightNodeAgeInMillis,
  long maxEdgeAgeInMillis,
  ResultFilterChain resultFilterChain) {
  super(queryNode, leftSeedNodesWithWeight, toBeFiltered, maxSocialProofTypeSize,
      socialProofTypes, maxRightNodeAgeInMillis, maxEdgeAgeInMillis, resultFilterChain);
  this.maxNumResults = maxNumResults;
  this.maxNumSocialProofs = maxNumSocialProofs;
  this.minUserPerSocialProof = minUserPerSocialProof;
}
 
Example #5
Source File: AgreeSetGenerator.java    From metanome-algorithms with Apache License 2.0 6 votes vote down vote up
private boolean isSubset(IntList actuelList, Map<Long, LongSet> index) {

        boolean first = true;
        LongSet positions = new LongArraySet();
        for (long e : actuelList) {
            if (!index.containsKey(Long.valueOf(e))) {
                return false;
            }
            if (first) {
                positions.addAll(index.get(Long.valueOf(e)));
                first = false;
            } else {

                this.intersect(positions, index.get(Long.valueOf(e)));
                // FIXME: Throws UnsupportedOperationExeption within fastUtil
                // positions.retainAll(index.get(e));
            }
            if (positions.size() == 0) {
                return false;
            }
        }
        return true;
    }
 
Example #6
Source File: TransactionManager.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private boolean doTruncateInvalidTxBefore(long time) throws InvalidTruncateTimeException {
  LOG.info("Removing tx ids before {} from invalid list", time);
  long truncateWp = time * TxConstants.MAX_TX_PER_MS;
  // Check if there any in-progress transactions started earlier than truncate time
  if (inProgress.lowerKey(truncateWp) != null) {
    throw new InvalidTruncateTimeException("Transactions started earlier than " + time + " are in-progress");
  }
  
  // Find all invalid transactions earlier than truncateWp
  LongSet toTruncate = new LongArraySet();
  LongIterator it = invalidTxList.toRawList().iterator();
  while (it.hasNext()) {
    long wp = it.nextLong();
    if (wp < truncateWp) {
      toTruncate.add(wp);
    }
  }
  LOG.info("Removing tx ids {} from invalid list", toTruncate);
  return invalidTxList.removeAll(toTruncate);
}
 
Example #7
Source File: TopSecondDegreeByCountRequest.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
/**
 * @param queryNode                 is the query node for running TopSecondDegreeByCount
 * @param leftSeedNodesWithWeight   is the set of seed nodes and their weights to use for
 *                                  TopSecondDegreeByCount
 * @param toBeFiltered              is the set of RHS nodes to be filtered from the output
 * @param maxSocialProofTypeSize    is the number of social proof types in the graph
 * @param socialProofTypes          Social proof types, masked into a byte array
 * @param maxRightNodeAgeInMillis   Max right node age in millisecond, such as tweet age
 * @param maxEdgeAgeInMillis        Max edge age in millisecond such as reply edge age
 * @param resultFilterChain         Filter chain to be applied after recommendation computation
 */
public TopSecondDegreeByCountRequest(
  long queryNode,
  Long2DoubleMap leftSeedNodesWithWeight,
  LongSet toBeFiltered,
  int maxSocialProofTypeSize,
  byte[] socialProofTypes,
  long maxRightNodeAgeInMillis,
  long maxEdgeAgeInMillis,
  ResultFilterChain resultFilterChain) {
  super(queryNode, toBeFiltered, socialProofTypes);
  this.leftSeedNodesWithWeight = leftSeedNodesWithWeight;
  this.maxSocialProofTypeSize = maxSocialProofTypeSize;
  this.maxRightNodeAgeInMillis = maxRightNodeAgeInMillis;
  this.maxEdgeAgeInMillis = maxEdgeAgeInMillis;
  this.resultFilterChain = resultFilterChain;
}
 
Example #8
Source File: TopSecondDegreeByCountRequestForMoment.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
/**
 * @param queryNode                 is the query node for running TopSecondDegreeByCountRequestForMoment
 * @param leftSeedNodesWithWeight   is the set of seed nodes and their weights to use for calculation
 * @param toBeFiltered              is the set of RHS nodes to be filtered from the output
 * @param maxNumResults             is the maximum number of recommendations returned in the response
 * @param maxNumSocialProofs        is the maximum number of social proofs per recommendation
 * @param maxSocialProofTypeSize    is the number of social proof types in the graph
 * @param minUserPerSocialProof     for each social proof, require a minimum number of users to be valid
 * @param socialProofTypes          is the list of valid social proofs, (i.e. Create, Like etc)
 * @param maxRightNodeAgeInMillis   Max right node age in millisecond, such as moment age
 * @param maxEdgeAgeInMillis        Max edge age in millisecond such as like edge age
 * @param resultFilterChain         is the chain of filters to be applied
 */
public TopSecondDegreeByCountRequestForMoment(
  long queryNode,
  Long2DoubleMap leftSeedNodesWithWeight,
  LongSet toBeFiltered,
  int maxNumResults,
  int maxNumSocialProofs,
  int maxSocialProofTypeSize,
  Map<Byte, Integer> minUserPerSocialProof,
  byte[] socialProofTypes,
  long maxRightNodeAgeInMillis,
  long maxEdgeAgeInMillis,
  ResultFilterChain resultFilterChain) {
  super(queryNode, leftSeedNodesWithWeight, toBeFiltered, maxSocialProofTypeSize,
      socialProofTypes, maxRightNodeAgeInMillis, maxEdgeAgeInMillis, resultFilterChain);
  this.maxNumResults = maxNumResults;
  this.maxNumSocialProofs = maxNumSocialProofs;
  this.minUserPerSocialProof = minUserPerSocialProof;
}
 
Example #9
Source File: TweetAuthorFilterTest.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
@Test
public void testBlacklistWhitelistFullOverlap() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();
  whitelistAuthors.add(1L);
  whitelistAuthors.add(2L);
  whitelistAuthors.add(3L);

  LongSet blacklistAuthors = new LongArraySet();
  blacklistAuthors.add(1L);
  blacklistAuthors.add(2L);
  blacklistAuthors.add(3L);

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // No tweet should be returned. 10, 20, 30 filtered by blacklist, 40 filtered by whitelist
  assertEquals(true, authorFilter.filterResult(10, socialProofs));
  assertEquals(true, authorFilter.filterResult(20, socialProofs));
  assertEquals(true, authorFilter.filterResult(30, socialProofs));
  assertEquals(true, authorFilter.filterResult(40, socialProofs));
}
 
Example #10
Source File: SocialProofGenerator.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
/**
 * Given a nodeInfo containing the social proofs and weight information regarding a rightNode,
 * convert and store these data in a SocialProofResult object, to comply with the class interface.
 *
 * @param nodeInfo Contains all the social proofs on a particular right node, along with the
 * accumulated node weight
 */
private SocialProofResult makeSocialProofResult(NodeInfo nodeInfo) {
  Byte2ObjectArrayMap<LongSet> socialProofsMap = new Byte2ObjectArrayMap<>();

  for (int socialProofType = 0; socialProofType < NUM_OF_SOCIAL_PROOF_TYPES; socialProofType++) {
    SmallArrayBasedLongToDoubleMap socialProofsByType = nodeInfo.getSocialProofs()[socialProofType];
    if (socialProofsByType == null || socialProofsByType.size() == 0) {
      continue;
    }
    LongSet rightNodeIds = new LongArraySet(
      Arrays.copyOfRange(socialProofsByType.keys(), 0, socialProofsByType.size()));
    socialProofsMap.put((byte)socialProofType, rightNodeIds);
  }

  return new SocialProofResult(
    nodeInfo.getNodeId(),
    socialProofsMap,
    nodeInfo.getWeight(),
    recommendationType
  );
}
 
Example #11
Source File: SocialProofGenerator.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
@Override
public SocialProofResponse computeRecommendations(SocialProofRequest request, Random rand) {
  reset();

  Long2DoubleMap leftSeedNodesWithWeight = request.getLeftSeedNodesWithWeight();
  LongSet rightNodeIds = request.getRightNodeIds();

  if (shouldRemoveUnfavoritedEdges(request)) {
    collectRightNodeInfo(
      leftSeedNodesWithWeight, rightNodeIds, appendUnfavoriteType(request.getSocialProofTypes()));
    return removeUnfavoritesAndGenerateRecommendationsFromNodeInfo();
  } else {
    collectRightNodeInfo(leftSeedNodesWithWeight, rightNodeIds, request.getSocialProofTypes());
    return generateRecommendationFromNodeInfo();
  }
}
 
Example #12
Source File: TweetAuthorFilter.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
public TweetAuthorFilter(
    LeftIndexedMultiSegmentBipartiteGraph leftIndexedBipartiteGraph,
    LongSet whitelistTweetAuthors,
    LongSet blacklistTweetAuthors,
    StatsReceiver statsReceiver) {
  super(statsReceiver);
  this.isIgnoreWhitelist = whitelistTweetAuthors.isEmpty();
  if (this.isIgnoreWhitelist) {
    this.whitelistedTweets = new LongOpenHashSet();
    this.blacklistedTweets = getTweetsByAuthors(leftIndexedBipartiteGraph, blacklistTweetAuthors);
  } else {
    // Performance hack. Remove blacklisted authors from the whitelist, and only check whitelist
    LongSet dedupedWhitelistAuthors = dedupWhitelistAuthors(whitelistTweetAuthors, blacklistTweetAuthors);
    this.whitelistedTweets = getTweetsByAuthors(leftIndexedBipartiteGraph, dedupedWhitelistAuthors);
    this.blacklistedTweets = new LongOpenHashSet();
  }
}
 
Example #13
Source File: TweetAuthorFilter.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
/**
 * Return the list of tweets authored by the input list of users
 */
private LongSet getTweetsByAuthors(
    LeftIndexedMultiSegmentBipartiteGraph leftIndexedBipartiteGraph,
    LongSet tweetAuthors) {
  LongSet authoredTweets = new LongOpenHashSet();
  for (long authorId: tweetAuthors) {
    EdgeIterator edgeIterator = leftIndexedBipartiteGraph.getLeftNodeEdges(authorId);
    if (edgeIterator == null) {
      continue;
    }

    // Sequentially iterating through the latest MAX_EDGES_PER_NODE edges per node
    int numEdgesPerNode = 0;
    while (edgeIterator.hasNext() && numEdgesPerNode++ < RecommendationRequest.MAX_EDGES_PER_NODE) {
      long rightNode = edgeIterator.nextLong();
      byte edgeType = edgeIterator.currentEdgeType();
      if (edgeType == RecommendationRequest.AUTHOR_SOCIAL_PROOF_TYPE) {
        authoredTweets.add(rightNode);
      }
    }
  }
  return authoredTweets;
}
 
Example #14
Source File: TweetAuthorFilterTest.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
@Test
public void testBlacklistWhitelistPartialOverlap() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();
  whitelistAuthors.add(1L);
  whitelistAuthors.add(2L);
  whitelistAuthors.add(3L);

  LongSet blacklistAuthors = new LongArraySet();
  blacklistAuthors.add(2L);
  blacklistAuthors.add(3L);
  blacklistAuthors.add(4L);

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // Only return 10, since it is whitelisted and not blacklisted
  assertEquals(false, authorFilter.filterResult(10, socialProofs));
  assertEquals(true, authorFilter.filterResult(20, socialProofs));
  assertEquals(true, authorFilter.filterResult(30, socialProofs));
  assertEquals(true, authorFilter.filterResult(40, socialProofs));
}
 
Example #15
Source File: TweetAuthorFilterTest.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyWhitelist() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();

  LongSet blacklistAuthors = new LongArraySet();

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // None of the tweets should be filtered.
  assertEquals(false, authorFilter.filterResult(10, socialProofs));
  assertEquals(false, authorFilter.filterResult(20, socialProofs));
  assertEquals(false, authorFilter.filterResult(30, socialProofs));
  assertEquals(false, authorFilter.filterResult(40, socialProofs));
}
 
Example #16
Source File: TweetAuthorFilterTest.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
@Test
public void testWhitelistOnlyTwoAuthors() {
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph();

  SmallArrayBasedLongToDoubleMap[] socialProofs = {};

  LongSet whitelistAuthors = new LongArraySet();
  whitelistAuthors.add(1L);
  whitelistAuthors.add(2L);

  LongSet blacklistAuthors = new LongArraySet();

  TweetAuthorFilter authorFilter = new TweetAuthorFilter(
      mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver());

  // 10L and 20L are authored by user 1 and 2. Do not filter them
  assertEquals(false, authorFilter.filterResult(10, socialProofs));
  assertEquals(false, authorFilter.filterResult(20, socialProofs));
  // 30L and 40L are authored by user 3 and 4. Filter them
  assertEquals(true, authorFilter.filterResult(30, socialProofs));
  assertEquals(true, authorFilter.filterResult(40, socialProofs));
}
 
Example #17
Source File: TweetSocialProofTest.java    From GraphJet with Apache License 2.0 6 votes vote down vote up
@Test
public void testTweetSocialProofWithInvalidUnfavorites() {
  // Test cases where unfavorite social proof is the only social proof type. Nothing will return
  NodeMetadataLeftIndexedMultiSegmentBipartiteGraph graph =
    BipartiteGraphTestHelper.buildTestNodeMetadataLeftIndexedMultiSegmentBipartiteGraphWithUnfavorite();

  Long2DoubleMap seedsMap = new Long2DoubleArrayMap(
    new long[] {user1, user2, user3, user4, user5, user6, user7,
      user8, user9, user10, user11, user12, user13, user14},
    new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1});
  LongSet tweets = new LongArraySet(
    new long[] {tweet1, tweet2, tweet3, tweet4, tweet5, tweet6, tweet7,
      tweet8, tweet9, tweet10, tweet11, tweet12, tweet13});

  byte[] validSocialProofTypes = new byte[] {UNFAVORITE_SOCIAL_PROOF_TYPE};

  SocialProofRequest socialProofRequest = new SocialProofRequest(
    tweets, seedsMap, validSocialProofTypes);
  HashMap<Long, SocialProofResult> results = new HashMap<>();
  new TweetSocialProofGenerator(graph)
    .computeRecommendations(socialProofRequest, new Random(0))
    .getRankedRecommendations().forEach( recInfo ->
    results.put(((SocialProofResult)recInfo).getNode(), (SocialProofResult)recInfo));

  assertEquals(0, results.size());
}
 
Example #18
Source File: LeftRegularBipartiteGraphSegmentTest.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
/**
 * Build a random left-regular bipartite graph of given left and right sizes.
 *
 * @param leftSize   is the left hand size of the bipartite graph
 * @param rightSize  is the right hand size of the bipartite graph
 * @param leftDegree is the degree of the left hand side
 * @param random     is the random number generator to use for constructing the graph
 * @return a random bipartite graph
 */
public static LeftRegularBipartiteGraphSegment buildRandomLeftRegularBipartiteGraph(
    int leftSize, int rightSize, int leftDegree, Random random) {
  LeftRegularBipartiteGraphSegment leftRegularBipartiteGraphSegment =
      new LeftRegularBipartiteGraphSegment(
          leftSize / 2,
          leftDegree,
          rightSize / 2,
          leftSize / 2,
          2.0,
          Integer.MAX_VALUE,
          new IdentityEdgeTypeMask(),
          new NullStatsReceiver());
  LongSet addedIds = new LongOpenHashSet(leftDegree);
  for (int i = 0; i < leftSize; i++) {
    addedIds.clear();
    for (int j = 0; j < leftDegree; j++) {
      long idToAdd;
      do {
        idToAdd = random.nextInt(rightSize);
      } while (addedIds.contains(idToAdd));
      addedIds.add(idToAdd);
      leftRegularBipartiteGraphSegment.addEdge(i, idToAdd, (byte) 0);
    }
  }

  return leftRegularBipartiteGraphSegment;
}
 
Example #19
Source File: TweetSocialProofTest.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
@Test
public void testTweetSocialProofsWithInvalidType() {
  // Test cases where the requested social proof types yield no results
  LeftIndexedPowerLawMultiSegmentBipartiteGraph bipartiteGraph =
    BipartiteGraphTestHelper.buildSmallTestLeftIndexedPowerLawMultiSegmentBipartiteGraphWithEdgeTypes();

  Long2DoubleMap seedsMap = new Long2DoubleArrayMap(
    new long[] {user1, user2}, new double[] {1.0, 0.5});
  LongSet tweets = new LongArraySet(new long[] {tweet2, tweet3, tweet4, tweet5, tweet6, tweet7});

  // In the graph there are no valid social proofs corresponding to these types
  byte[] validSocialProofTypes = new byte[] {
    AUTHOR_SOCIAL_PROOF_TYPE,
    IS_MENTIONED_SOCIAL_PROOF_TYPE
  };

  SocialProofRequest socialProofRequest = new SocialProofRequest(
    tweets,
    seedsMap,
    validSocialProofTypes
  );
  HashMap<Long, SocialProofResult> results = new HashMap<>();

  new TweetSocialProofGenerator(bipartiteGraph)
    .computeRecommendations(socialProofRequest, new Random(0))
    .getRankedRecommendations().forEach( recInfo ->
    results.put(((SocialProofResult)recInfo).getNode(), (SocialProofResult)recInfo));

  assertTrue(results.isEmpty());
}
 
Example #20
Source File: SocialProofResult.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
public SocialProofResult(
  Long node,
  Byte2ObjectMap<LongSet> socialProof,
  double weight,
  RecommendationType recommendationType
) {
  this.node = node;
  this.socialProof = socialProof;
  this.weight = weight;
  this.recommendationType = recommendationType;
}
 
Example #21
Source File: SocialProofRequest.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
/**
 * Create a social proof request.
 *
 * @param rightNodeIds        is the set of right nodes to query for social proof.
 * @param weightedSeedNodes   is the set of left nodes to be used as social proofs.
 * @param socialProofTypes    is the social proof types to return.
 */
public SocialProofRequest(
  LongSet rightNodeIds,
  Long2DoubleMap weightedSeedNodes,
  byte[] socialProofTypes
) {
  super(0, EMPTY_SET, socialProofTypes);
  this.leftSeedNodesWithWeight = weightedSeedNodes;
  this.rightNodeIds = rightNodeIds;
}
 
Example #22
Source File: LongColumn.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
@Override
public LongColumn unique() {
  final LongSet values = new LongOpenHashSet();
  for (int i = 0; i < size(); i++) {
    values.add(getLong(i));
  }
  final LongColumn column = LongColumn.create(name() + " Unique values");
  for (long value : values) {
    column.append(value);
  }
  return column;
}
 
Example #23
Source File: InstantColumn.java    From tablesaw with Apache License 2.0 5 votes vote down vote up
@Override
public int countUnique() {
  LongSet ints = new LongOpenHashSet(data.size());
  for (long i : data) {
    ints.add(i);
  }
  return ints.size();
}
 
Example #24
Source File: NodeMetadataSocialProofResult.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
/**
 * Calculate the total number of interactions for the current nodeMetadataId (right node's metadata)
 * given the set of users (left nodes).
 *
 * @return the number of unique edgeType/user/tweet interactions.
 * For example (0 (byte), 12 (long), 99 (long)) would be a single unique interaction.
 */
public int getSocialProofSize() {
  int socialProofSize = 0;
  for (Long2ObjectMap<LongSet> userToTweetsMap: socialProof.values()) {
    for (LongSet connectingTweets: userToTweetsMap.values()) {
      socialProofSize += connectingTweets.size();
    }
  }
  return socialProofSize;
}
 
Example #25
Source File: NodeMetadataSocialProofResult.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
public NodeMetadataSocialProofResult(
  int nodeMetadataId,
  Byte2ObjectMap<Long2ObjectMap<LongSet>> socialProof,
  double weight,
  RecommendationType recommendationType
) {
  this.nodeMetadataId = nodeMetadataId;
  this.socialProof = socialProof;
  this.weight = weight;
  this.recommendationType = recommendationType;
}
 
Example #26
Source File: ArrayDistinctFunction.java    From presto with Apache License 2.0 5 votes vote down vote up
@SqlType("array(bigint)")
public Block bigintDistinct(@SqlType("array(bigint)") Block array)
{
    if (array.getPositionCount() == 0) {
        return array;
    }

    boolean containsNull = false;
    LongSet set = new LongOpenHashSet(array.getPositionCount());
    int distinctCount = 0;

    if (pageBuilder.isFull()) {
        pageBuilder.reset();
    }

    BlockBuilder distinctElementBlockBuilder = pageBuilder.getBlockBuilder(0);
    for (int i = 0; i < array.getPositionCount(); i++) {
        if (array.isNull(i)) {
            if (!containsNull) {
                containsNull = true;
                distinctElementBlockBuilder.appendNull();
                distinctCount++;
            }
            continue;
        }
        long value = BIGINT.getLong(array, i);
        if (!set.contains(value)) {
            set.add(value);
            distinctCount++;
            BIGINT.appendTo(array, i, distinctElementBlockBuilder);
        }
    }

    pageBuilder.declarePositions(distinctCount);

    return distinctElementBlockBuilder.getRegion(distinctElementBlockBuilder.getPositionCount() - distinctCount, distinctCount);
}
 
Example #27
Source File: TopSecondDegreeByCountTweetRecsGenerator.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
private static boolean isLessThanMinUserSocialProofSizeCombined(
  SmallArrayBasedLongToDoubleMap[] socialProofs,
  int minUserSocialProofSize,
  Set<byte[]> socialProofTypeUnions) {
  if (socialProofTypeUnions.isEmpty() ||
    // check if the size of any social proof union is greater than minUserSocialProofSize before dedupping
    isSocialProofUnionSizeLessThanMin(socialProofs, minUserSocialProofSize, socialProofTypeUnions)) {
    return true;
  }

  LongSet uniqueNodes = new LongOpenHashSet(minUserSocialProofSize);

  for (byte[] socialProofTypeUnion: socialProofTypeUnions) {
    // Clear removes all elements, but does not change the size of the set.
    // Thus, we only use one LongOpenHashSet with at most a size of 2*minUserSocialProofSize
    uniqueNodes.clear();
    for (byte socialProofType: socialProofTypeUnion) {
      if (socialProofs[socialProofType] != null) {
        for (int i = 0; i < socialProofs[socialProofType].size(); i++) {
          uniqueNodes.add(socialProofs[socialProofType].keys()[i]);
          if (uniqueNodes.size() >= minUserSocialProofSize) {
            return false;
          }
        }
      }
    }
  }
  return true;
}
 
Example #28
Source File: ValueInTransformFunction.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
private static long[] filterLongs(LongSet longSet, long[] source) {
  LongList longList = new LongArrayList();
  for (long value : source) {
    if (longSet.contains(value)) {
      longList.add(value);
    }
  }
  if (longList.size() == source.length) {
    return source;
  } else {
    return longList.toLongArray();
  }
}
 
Example #29
Source File: TopSecondDegreeByCountTweetRecsGenerator.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
private static boolean isLessThanMinUserSocialProofSizeCombined(
  SmallArrayBasedLongToDoubleMap[] socialProofs,
  int minUserSocialProofSize,
  Set<byte[]> socialProofTypeUnions) {
  if (socialProofTypeUnions.isEmpty() ||
    // check if the size of any social proof union is greater than minUserSocialProofSize before dedupping
    isSocialProofUnionSizeLessThanMin(socialProofs, minUserSocialProofSize, socialProofTypeUnions)) {
    return true;
  }

  LongSet uniqueNodes = new LongOpenHashSet(minUserSocialProofSize);

  for (byte[] socialProofTypeUnion: socialProofTypeUnions) {
    // Clear removes all elements, but does not change the size of the set.
    // Thus, we only use one LongOpenHashSet with at most a size of 2*minUserSocialProofSize
    uniqueNodes.clear();
    for (byte socialProofType: socialProofTypeUnion) {
      if (socialProofs[socialProofType] != null) {
        for (int i = 0; i < socialProofs[socialProofType].size(); i++) {
          uniqueNodes.add(socialProofs[socialProofType].keys()[i]);
          if (uniqueNodes.size() >= minUserSocialProofSize) {
            return false;
          }
        }
      }
    }
  }
  return true;
}
 
Example #30
Source File: SalsaRequest.java    From GraphJet with Apache License 2.0 5 votes vote down vote up
/**
 * The constructor should only be called via {@link SalsaRequestBuilder}.
 * @param queryNode                 is the query node for running SALSA
 * @param leftSeedNodesWithWeight   is the set of seed nodes to use for SALSA, with weights being
 *                                  the proportion of random walks to start here. We do NOT assume
 *                                  that the queryNode is added to this.
 * @param toBeFiltered              is the set of RHS nodes to be filtered from the output
 * @param numRandomWalks            is the total number of random walks to run
 * @param maxRandomWalkLength       is the maximum length of a random walk
 * @param resetProbability          is the probability of reset in SALSA. Note that reset is only
 *                                  done on backward iterations.
 * @param maxNumResults             is the maximum number of results that SALSA will return
 * @param maxSocialProofSize        is the maximum size of social proof per type to return. Set
 *                                  this to 0 to return no social proof
 * @param maxSocialProofTypeSize    is the maximum size of social proof types in the graph.
 * @param socialProofTypes          is the social proof types to return
 * @param queryNodeWeightFraction   is the relative proportion of random walks to start at the
 *                                  queryNode in the first iteration. This parameter is only used
 * @param removeCustomizedBitsNodes removes tweets with metadata information embedded into top
 *                                  four bits
 * @param resultFilterChain         is the chain of filters to be applied
 */
protected SalsaRequest(
    long queryNode,
    Long2DoubleMap leftSeedNodesWithWeight,
    LongSet toBeFiltered,
    int numRandomWalks,
    int maxRandomWalkLength,
    double resetProbability,
    int maxNumResults,
    int maxSocialProofSize,
    int maxSocialProofTypeSize,
    byte[] socialProofTypes,
    double queryNodeWeightFraction,
    boolean removeCustomizedBitsNodes,
    ResultFilterChain resultFilterChain) {
  super(queryNode, toBeFiltered, socialProofTypes);
  this.leftSeedNodesWithWeight = leftSeedNodesWithWeight;
  this.numRandomWalks = numRandomWalks;
  this.maxRandomWalkLength = maxRandomWalkLength;
  this.resetProbability = resetProbability;
  this.maxNumResults = maxNumResults;
  this.maxSocialProofSize = maxSocialProofSize;
  this.maxSocialProofTypeSize = maxSocialProofTypeSize;
  this.queryNodeWeightFraction = queryNodeWeightFraction;
  this.removeCustomizedBitsNodes = removeCustomizedBitsNodes;
  this.resultFilterChain = resultFilterChain;
}