it.unimi.dsi.fastutil.longs.LongSet Java Examples
The following examples show how to use
it.unimi.dsi.fastutil.longs.LongSet.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TweetAuthorFilterTest.java From GraphJet with Apache License 2.0 | 6 votes |
@Test public void testBlacklistWhitelistNoOverlap() { NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph(); SmallArrayBasedLongToDoubleMap[] socialProofs = {}; LongSet whitelistAuthors = new LongArraySet(); whitelistAuthors.add(1L); whitelistAuthors.add(2L); LongSet blacklistAuthors = new LongArraySet(); blacklistAuthors.add(3L); blacklistAuthors.add(4L); TweetAuthorFilter authorFilter = new TweetAuthorFilter( mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver()); // Should only return whitelisted 10 and 20. 30 and 40 are filtered by blacklist assertEquals(false, authorFilter.filterResult(10, socialProofs)); assertEquals(false, authorFilter.filterResult(20, socialProofs)); assertEquals(true, authorFilter.filterResult(30, socialProofs)); assertEquals(true, authorFilter.filterResult(40, socialProofs)); }
Example #2
Source File: KnowledgeBase.java From fasten with Apache License 2.0 | 6 votes |
/** * The set of all node signatures that are reachable from the signature <code>startSig</code>. * * @param start the starting node. * @return the set of all node signatures for which there is a directed path from * <code>startSig</code> to that node. */ public synchronized LongSet reaches(final long startSig) { final LongOpenHashSet result = new LongOpenHashSet(); // Visit queue final LongArrayFIFOQueue queue = new LongArrayFIFOQueue(); queue.enqueue(startSig); result.add(startSig); while (!queue.isEmpty()) { final long nodeSig = queue.dequeueLong(); for (final long s : successors(nodeSig)) if (!result.contains(s)) { queue.enqueue(s); result.add(s); } } return result; }
Example #3
Source File: ArrayUnionFunction.java From presto with Apache License 2.0 | 6 votes |
private static void appendBigintArray(Block array, AtomicBoolean containsNull, LongSet set, BlockBuilder blockBuilder) { for (int i = 0; i < array.getPositionCount(); i++) { if (array.isNull(i)) { if (!containsNull.get()) { containsNull.set(true); blockBuilder.appendNull(); } continue; } long value = BIGINT.getLong(array, i); if (set.add(value)) { BIGINT.writeLong(blockBuilder, value); } } }
Example #4
Source File: TopSecondDegreeByCountRequestForUser.java From GraphJet with Apache License 2.0 | 6 votes |
/** * @param queryNode is the query node for running TopSecondDegreeByCountForUser * @param leftSeedNodesWithWeight is the set of seed nodes and their weights to use for calculation * @param toBeFiltered is the list of users to be excluded from recommendations * @param maxNumResults is the maximum number of recommendations returned in the response * @param maxNumSocialProofs is the maximum number of social proofs per recommendation * @param maxSocialProofTypeSize is the number of social proof types in the graph * @param minUserPerSocialProof for each social proof, require a minimum number of users to be valid * @param socialProofTypes is the list of valid social proofs, (i.e, Follow, Mention, Mediatag) * @param maxRightNodeAgeInMillis is the max right node age in millisecond, such as tweet age * @param maxEdgeAgeInMillis is the max edge age in millisecond such as reply edge age * @param resultFilterChain is the chain of filters to be applied */ public TopSecondDegreeByCountRequestForUser( long queryNode, Long2DoubleMap leftSeedNodesWithWeight, LongSet toBeFiltered, int maxNumResults, int maxNumSocialProofs, int maxSocialProofTypeSize, Map<Byte, Integer> minUserPerSocialProof, byte[] socialProofTypes, long maxRightNodeAgeInMillis, long maxEdgeAgeInMillis, ResultFilterChain resultFilterChain) { super(queryNode, leftSeedNodesWithWeight, toBeFiltered, maxSocialProofTypeSize, socialProofTypes, maxRightNodeAgeInMillis, maxEdgeAgeInMillis, resultFilterChain); this.maxNumResults = maxNumResults; this.maxNumSocialProofs = maxNumSocialProofs; this.minUserPerSocialProof = minUserPerSocialProof; }
Example #5
Source File: AgreeSetGenerator.java From metanome-algorithms with Apache License 2.0 | 6 votes |
private boolean isSubset(IntList actuelList, Map<Long, LongSet> index) { boolean first = true; LongSet positions = new LongArraySet(); for (long e : actuelList) { if (!index.containsKey(Long.valueOf(e))) { return false; } if (first) { positions.addAll(index.get(Long.valueOf(e))); first = false; } else { this.intersect(positions, index.get(Long.valueOf(e))); // FIXME: Throws UnsupportedOperationExeption within fastUtil // positions.retainAll(index.get(e)); } if (positions.size() == 0) { return false; } } return true; }
Example #6
Source File: TransactionManager.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private boolean doTruncateInvalidTxBefore(long time) throws InvalidTruncateTimeException { LOG.info("Removing tx ids before {} from invalid list", time); long truncateWp = time * TxConstants.MAX_TX_PER_MS; // Check if there any in-progress transactions started earlier than truncate time if (inProgress.lowerKey(truncateWp) != null) { throw new InvalidTruncateTimeException("Transactions started earlier than " + time + " are in-progress"); } // Find all invalid transactions earlier than truncateWp LongSet toTruncate = new LongArraySet(); LongIterator it = invalidTxList.toRawList().iterator(); while (it.hasNext()) { long wp = it.nextLong(); if (wp < truncateWp) { toTruncate.add(wp); } } LOG.info("Removing tx ids {} from invalid list", toTruncate); return invalidTxList.removeAll(toTruncate); }
Example #7
Source File: TopSecondDegreeByCountRequest.java From GraphJet with Apache License 2.0 | 6 votes |
/** * @param queryNode is the query node for running TopSecondDegreeByCount * @param leftSeedNodesWithWeight is the set of seed nodes and their weights to use for * TopSecondDegreeByCount * @param toBeFiltered is the set of RHS nodes to be filtered from the output * @param maxSocialProofTypeSize is the number of social proof types in the graph * @param socialProofTypes Social proof types, masked into a byte array * @param maxRightNodeAgeInMillis Max right node age in millisecond, such as tweet age * @param maxEdgeAgeInMillis Max edge age in millisecond such as reply edge age * @param resultFilterChain Filter chain to be applied after recommendation computation */ public TopSecondDegreeByCountRequest( long queryNode, Long2DoubleMap leftSeedNodesWithWeight, LongSet toBeFiltered, int maxSocialProofTypeSize, byte[] socialProofTypes, long maxRightNodeAgeInMillis, long maxEdgeAgeInMillis, ResultFilterChain resultFilterChain) { super(queryNode, toBeFiltered, socialProofTypes); this.leftSeedNodesWithWeight = leftSeedNodesWithWeight; this.maxSocialProofTypeSize = maxSocialProofTypeSize; this.maxRightNodeAgeInMillis = maxRightNodeAgeInMillis; this.maxEdgeAgeInMillis = maxEdgeAgeInMillis; this.resultFilterChain = resultFilterChain; }
Example #8
Source File: TopSecondDegreeByCountRequestForMoment.java From GraphJet with Apache License 2.0 | 6 votes |
/** * @param queryNode is the query node for running TopSecondDegreeByCountRequestForMoment * @param leftSeedNodesWithWeight is the set of seed nodes and their weights to use for calculation * @param toBeFiltered is the set of RHS nodes to be filtered from the output * @param maxNumResults is the maximum number of recommendations returned in the response * @param maxNumSocialProofs is the maximum number of social proofs per recommendation * @param maxSocialProofTypeSize is the number of social proof types in the graph * @param minUserPerSocialProof for each social proof, require a minimum number of users to be valid * @param socialProofTypes is the list of valid social proofs, (i.e. Create, Like etc) * @param maxRightNodeAgeInMillis Max right node age in millisecond, such as moment age * @param maxEdgeAgeInMillis Max edge age in millisecond such as like edge age * @param resultFilterChain is the chain of filters to be applied */ public TopSecondDegreeByCountRequestForMoment( long queryNode, Long2DoubleMap leftSeedNodesWithWeight, LongSet toBeFiltered, int maxNumResults, int maxNumSocialProofs, int maxSocialProofTypeSize, Map<Byte, Integer> minUserPerSocialProof, byte[] socialProofTypes, long maxRightNodeAgeInMillis, long maxEdgeAgeInMillis, ResultFilterChain resultFilterChain) { super(queryNode, leftSeedNodesWithWeight, toBeFiltered, maxSocialProofTypeSize, socialProofTypes, maxRightNodeAgeInMillis, maxEdgeAgeInMillis, resultFilterChain); this.maxNumResults = maxNumResults; this.maxNumSocialProofs = maxNumSocialProofs; this.minUserPerSocialProof = minUserPerSocialProof; }
Example #9
Source File: TweetAuthorFilterTest.java From GraphJet with Apache License 2.0 | 6 votes |
@Test public void testBlacklistWhitelistFullOverlap() { NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph(); SmallArrayBasedLongToDoubleMap[] socialProofs = {}; LongSet whitelistAuthors = new LongArraySet(); whitelistAuthors.add(1L); whitelistAuthors.add(2L); whitelistAuthors.add(3L); LongSet blacklistAuthors = new LongArraySet(); blacklistAuthors.add(1L); blacklistAuthors.add(2L); blacklistAuthors.add(3L); TweetAuthorFilter authorFilter = new TweetAuthorFilter( mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver()); // No tweet should be returned. 10, 20, 30 filtered by blacklist, 40 filtered by whitelist assertEquals(true, authorFilter.filterResult(10, socialProofs)); assertEquals(true, authorFilter.filterResult(20, socialProofs)); assertEquals(true, authorFilter.filterResult(30, socialProofs)); assertEquals(true, authorFilter.filterResult(40, socialProofs)); }
Example #10
Source File: SocialProofGenerator.java From GraphJet with Apache License 2.0 | 6 votes |
/** * Given a nodeInfo containing the social proofs and weight information regarding a rightNode, * convert and store these data in a SocialProofResult object, to comply with the class interface. * * @param nodeInfo Contains all the social proofs on a particular right node, along with the * accumulated node weight */ private SocialProofResult makeSocialProofResult(NodeInfo nodeInfo) { Byte2ObjectArrayMap<LongSet> socialProofsMap = new Byte2ObjectArrayMap<>(); for (int socialProofType = 0; socialProofType < NUM_OF_SOCIAL_PROOF_TYPES; socialProofType++) { SmallArrayBasedLongToDoubleMap socialProofsByType = nodeInfo.getSocialProofs()[socialProofType]; if (socialProofsByType == null || socialProofsByType.size() == 0) { continue; } LongSet rightNodeIds = new LongArraySet( Arrays.copyOfRange(socialProofsByType.keys(), 0, socialProofsByType.size())); socialProofsMap.put((byte)socialProofType, rightNodeIds); } return new SocialProofResult( nodeInfo.getNodeId(), socialProofsMap, nodeInfo.getWeight(), recommendationType ); }
Example #11
Source File: SocialProofGenerator.java From GraphJet with Apache License 2.0 | 6 votes |
@Override public SocialProofResponse computeRecommendations(SocialProofRequest request, Random rand) { reset(); Long2DoubleMap leftSeedNodesWithWeight = request.getLeftSeedNodesWithWeight(); LongSet rightNodeIds = request.getRightNodeIds(); if (shouldRemoveUnfavoritedEdges(request)) { collectRightNodeInfo( leftSeedNodesWithWeight, rightNodeIds, appendUnfavoriteType(request.getSocialProofTypes())); return removeUnfavoritesAndGenerateRecommendationsFromNodeInfo(); } else { collectRightNodeInfo(leftSeedNodesWithWeight, rightNodeIds, request.getSocialProofTypes()); return generateRecommendationFromNodeInfo(); } }
Example #12
Source File: TweetAuthorFilter.java From GraphJet with Apache License 2.0 | 6 votes |
public TweetAuthorFilter( LeftIndexedMultiSegmentBipartiteGraph leftIndexedBipartiteGraph, LongSet whitelistTweetAuthors, LongSet blacklistTweetAuthors, StatsReceiver statsReceiver) { super(statsReceiver); this.isIgnoreWhitelist = whitelistTweetAuthors.isEmpty(); if (this.isIgnoreWhitelist) { this.whitelistedTweets = new LongOpenHashSet(); this.blacklistedTweets = getTweetsByAuthors(leftIndexedBipartiteGraph, blacklistTweetAuthors); } else { // Performance hack. Remove blacklisted authors from the whitelist, and only check whitelist LongSet dedupedWhitelistAuthors = dedupWhitelistAuthors(whitelistTweetAuthors, blacklistTweetAuthors); this.whitelistedTweets = getTweetsByAuthors(leftIndexedBipartiteGraph, dedupedWhitelistAuthors); this.blacklistedTweets = new LongOpenHashSet(); } }
Example #13
Source File: TweetAuthorFilter.java From GraphJet with Apache License 2.0 | 6 votes |
/** * Return the list of tweets authored by the input list of users */ private LongSet getTweetsByAuthors( LeftIndexedMultiSegmentBipartiteGraph leftIndexedBipartiteGraph, LongSet tweetAuthors) { LongSet authoredTweets = new LongOpenHashSet(); for (long authorId: tweetAuthors) { EdgeIterator edgeIterator = leftIndexedBipartiteGraph.getLeftNodeEdges(authorId); if (edgeIterator == null) { continue; } // Sequentially iterating through the latest MAX_EDGES_PER_NODE edges per node int numEdgesPerNode = 0; while (edgeIterator.hasNext() && numEdgesPerNode++ < RecommendationRequest.MAX_EDGES_PER_NODE) { long rightNode = edgeIterator.nextLong(); byte edgeType = edgeIterator.currentEdgeType(); if (edgeType == RecommendationRequest.AUTHOR_SOCIAL_PROOF_TYPE) { authoredTweets.add(rightNode); } } } return authoredTweets; }
Example #14
Source File: TweetAuthorFilterTest.java From GraphJet with Apache License 2.0 | 6 votes |
@Test public void testBlacklistWhitelistPartialOverlap() { NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph(); SmallArrayBasedLongToDoubleMap[] socialProofs = {}; LongSet whitelistAuthors = new LongArraySet(); whitelistAuthors.add(1L); whitelistAuthors.add(2L); whitelistAuthors.add(3L); LongSet blacklistAuthors = new LongArraySet(); blacklistAuthors.add(2L); blacklistAuthors.add(3L); blacklistAuthors.add(4L); TweetAuthorFilter authorFilter = new TweetAuthorFilter( mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver()); // Only return 10, since it is whitelisted and not blacklisted assertEquals(false, authorFilter.filterResult(10, socialProofs)); assertEquals(true, authorFilter.filterResult(20, socialProofs)); assertEquals(true, authorFilter.filterResult(30, socialProofs)); assertEquals(true, authorFilter.filterResult(40, socialProofs)); }
Example #15
Source File: TweetAuthorFilterTest.java From GraphJet with Apache License 2.0 | 6 votes |
@Test public void testEmptyWhitelist() { NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph(); SmallArrayBasedLongToDoubleMap[] socialProofs = {}; LongSet whitelistAuthors = new LongArraySet(); LongSet blacklistAuthors = new LongArraySet(); TweetAuthorFilter authorFilter = new TweetAuthorFilter( mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver()); // None of the tweets should be filtered. assertEquals(false, authorFilter.filterResult(10, socialProofs)); assertEquals(false, authorFilter.filterResult(20, socialProofs)); assertEquals(false, authorFilter.filterResult(30, socialProofs)); assertEquals(false, authorFilter.filterResult(40, socialProofs)); }
Example #16
Source File: TweetAuthorFilterTest.java From GraphJet with Apache License 2.0 | 6 votes |
@Test public void testWhitelistOnlyTwoAuthors() { NodeMetadataLeftIndexedMultiSegmentBipartiteGraph mockGraph = getMockGraph(); SmallArrayBasedLongToDoubleMap[] socialProofs = {}; LongSet whitelistAuthors = new LongArraySet(); whitelistAuthors.add(1L); whitelistAuthors.add(2L); LongSet blacklistAuthors = new LongArraySet(); TweetAuthorFilter authorFilter = new TweetAuthorFilter( mockGraph, whitelistAuthors, blacklistAuthors, new NullStatsReceiver()); // 10L and 20L are authored by user 1 and 2. Do not filter them assertEquals(false, authorFilter.filterResult(10, socialProofs)); assertEquals(false, authorFilter.filterResult(20, socialProofs)); // 30L and 40L are authored by user 3 and 4. Filter them assertEquals(true, authorFilter.filterResult(30, socialProofs)); assertEquals(true, authorFilter.filterResult(40, socialProofs)); }
Example #17
Source File: TweetSocialProofTest.java From GraphJet with Apache License 2.0 | 6 votes |
@Test public void testTweetSocialProofWithInvalidUnfavorites() { // Test cases where unfavorite social proof is the only social proof type. Nothing will return NodeMetadataLeftIndexedMultiSegmentBipartiteGraph graph = BipartiteGraphTestHelper.buildTestNodeMetadataLeftIndexedMultiSegmentBipartiteGraphWithUnfavorite(); Long2DoubleMap seedsMap = new Long2DoubleArrayMap( new long[] {user1, user2, user3, user4, user5, user6, user7, user8, user9, user10, user11, user12, user13, user14}, new double[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); LongSet tweets = new LongArraySet( new long[] {tweet1, tweet2, tweet3, tweet4, tweet5, tweet6, tweet7, tweet8, tweet9, tweet10, tweet11, tweet12, tweet13}); byte[] validSocialProofTypes = new byte[] {UNFAVORITE_SOCIAL_PROOF_TYPE}; SocialProofRequest socialProofRequest = new SocialProofRequest( tweets, seedsMap, validSocialProofTypes); HashMap<Long, SocialProofResult> results = new HashMap<>(); new TweetSocialProofGenerator(graph) .computeRecommendations(socialProofRequest, new Random(0)) .getRankedRecommendations().forEach( recInfo -> results.put(((SocialProofResult)recInfo).getNode(), (SocialProofResult)recInfo)); assertEquals(0, results.size()); }
Example #18
Source File: LeftRegularBipartiteGraphSegmentTest.java From GraphJet with Apache License 2.0 | 5 votes |
/** * Build a random left-regular bipartite graph of given left and right sizes. * * @param leftSize is the left hand size of the bipartite graph * @param rightSize is the right hand size of the bipartite graph * @param leftDegree is the degree of the left hand side * @param random is the random number generator to use for constructing the graph * @return a random bipartite graph */ public static LeftRegularBipartiteGraphSegment buildRandomLeftRegularBipartiteGraph( int leftSize, int rightSize, int leftDegree, Random random) { LeftRegularBipartiteGraphSegment leftRegularBipartiteGraphSegment = new LeftRegularBipartiteGraphSegment( leftSize / 2, leftDegree, rightSize / 2, leftSize / 2, 2.0, Integer.MAX_VALUE, new IdentityEdgeTypeMask(), new NullStatsReceiver()); LongSet addedIds = new LongOpenHashSet(leftDegree); for (int i = 0; i < leftSize; i++) { addedIds.clear(); for (int j = 0; j < leftDegree; j++) { long idToAdd; do { idToAdd = random.nextInt(rightSize); } while (addedIds.contains(idToAdd)); addedIds.add(idToAdd); leftRegularBipartiteGraphSegment.addEdge(i, idToAdd, (byte) 0); } } return leftRegularBipartiteGraphSegment; }
Example #19
Source File: TweetSocialProofTest.java From GraphJet with Apache License 2.0 | 5 votes |
@Test public void testTweetSocialProofsWithInvalidType() { // Test cases where the requested social proof types yield no results LeftIndexedPowerLawMultiSegmentBipartiteGraph bipartiteGraph = BipartiteGraphTestHelper.buildSmallTestLeftIndexedPowerLawMultiSegmentBipartiteGraphWithEdgeTypes(); Long2DoubleMap seedsMap = new Long2DoubleArrayMap( new long[] {user1, user2}, new double[] {1.0, 0.5}); LongSet tweets = new LongArraySet(new long[] {tweet2, tweet3, tweet4, tweet5, tweet6, tweet7}); // In the graph there are no valid social proofs corresponding to these types byte[] validSocialProofTypes = new byte[] { AUTHOR_SOCIAL_PROOF_TYPE, IS_MENTIONED_SOCIAL_PROOF_TYPE }; SocialProofRequest socialProofRequest = new SocialProofRequest( tweets, seedsMap, validSocialProofTypes ); HashMap<Long, SocialProofResult> results = new HashMap<>(); new TweetSocialProofGenerator(bipartiteGraph) .computeRecommendations(socialProofRequest, new Random(0)) .getRankedRecommendations().forEach( recInfo -> results.put(((SocialProofResult)recInfo).getNode(), (SocialProofResult)recInfo)); assertTrue(results.isEmpty()); }
Example #20
Source File: SocialProofResult.java From GraphJet with Apache License 2.0 | 5 votes |
public SocialProofResult( Long node, Byte2ObjectMap<LongSet> socialProof, double weight, RecommendationType recommendationType ) { this.node = node; this.socialProof = socialProof; this.weight = weight; this.recommendationType = recommendationType; }
Example #21
Source File: SocialProofRequest.java From GraphJet with Apache License 2.0 | 5 votes |
/** * Create a social proof request. * * @param rightNodeIds is the set of right nodes to query for social proof. * @param weightedSeedNodes is the set of left nodes to be used as social proofs. * @param socialProofTypes is the social proof types to return. */ public SocialProofRequest( LongSet rightNodeIds, Long2DoubleMap weightedSeedNodes, byte[] socialProofTypes ) { super(0, EMPTY_SET, socialProofTypes); this.leftSeedNodesWithWeight = weightedSeedNodes; this.rightNodeIds = rightNodeIds; }
Example #22
Source File: LongColumn.java From tablesaw with Apache License 2.0 | 5 votes |
@Override public LongColumn unique() { final LongSet values = new LongOpenHashSet(); for (int i = 0; i < size(); i++) { values.add(getLong(i)); } final LongColumn column = LongColumn.create(name() + " Unique values"); for (long value : values) { column.append(value); } return column; }
Example #23
Source File: InstantColumn.java From tablesaw with Apache License 2.0 | 5 votes |
@Override public int countUnique() { LongSet ints = new LongOpenHashSet(data.size()); for (long i : data) { ints.add(i); } return ints.size(); }
Example #24
Source File: NodeMetadataSocialProofResult.java From GraphJet with Apache License 2.0 | 5 votes |
/** * Calculate the total number of interactions for the current nodeMetadataId (right node's metadata) * given the set of users (left nodes). * * @return the number of unique edgeType/user/tweet interactions. * For example (0 (byte), 12 (long), 99 (long)) would be a single unique interaction. */ public int getSocialProofSize() { int socialProofSize = 0; for (Long2ObjectMap<LongSet> userToTweetsMap: socialProof.values()) { for (LongSet connectingTweets: userToTweetsMap.values()) { socialProofSize += connectingTweets.size(); } } return socialProofSize; }
Example #25
Source File: NodeMetadataSocialProofResult.java From GraphJet with Apache License 2.0 | 5 votes |
public NodeMetadataSocialProofResult( int nodeMetadataId, Byte2ObjectMap<Long2ObjectMap<LongSet>> socialProof, double weight, RecommendationType recommendationType ) { this.nodeMetadataId = nodeMetadataId; this.socialProof = socialProof; this.weight = weight; this.recommendationType = recommendationType; }
Example #26
Source File: ArrayDistinctFunction.java From presto with Apache License 2.0 | 5 votes |
@SqlType("array(bigint)") public Block bigintDistinct(@SqlType("array(bigint)") Block array) { if (array.getPositionCount() == 0) { return array; } boolean containsNull = false; LongSet set = new LongOpenHashSet(array.getPositionCount()); int distinctCount = 0; if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder distinctElementBlockBuilder = pageBuilder.getBlockBuilder(0); for (int i = 0; i < array.getPositionCount(); i++) { if (array.isNull(i)) { if (!containsNull) { containsNull = true; distinctElementBlockBuilder.appendNull(); distinctCount++; } continue; } long value = BIGINT.getLong(array, i); if (!set.contains(value)) { set.add(value); distinctCount++; BIGINT.appendTo(array, i, distinctElementBlockBuilder); } } pageBuilder.declarePositions(distinctCount); return distinctElementBlockBuilder.getRegion(distinctElementBlockBuilder.getPositionCount() - distinctCount, distinctCount); }
Example #27
Source File: TopSecondDegreeByCountTweetRecsGenerator.java From GraphJet with Apache License 2.0 | 5 votes |
private static boolean isLessThanMinUserSocialProofSizeCombined( SmallArrayBasedLongToDoubleMap[] socialProofs, int minUserSocialProofSize, Set<byte[]> socialProofTypeUnions) { if (socialProofTypeUnions.isEmpty() || // check if the size of any social proof union is greater than minUserSocialProofSize before dedupping isSocialProofUnionSizeLessThanMin(socialProofs, minUserSocialProofSize, socialProofTypeUnions)) { return true; } LongSet uniqueNodes = new LongOpenHashSet(minUserSocialProofSize); for (byte[] socialProofTypeUnion: socialProofTypeUnions) { // Clear removes all elements, but does not change the size of the set. // Thus, we only use one LongOpenHashSet with at most a size of 2*minUserSocialProofSize uniqueNodes.clear(); for (byte socialProofType: socialProofTypeUnion) { if (socialProofs[socialProofType] != null) { for (int i = 0; i < socialProofs[socialProofType].size(); i++) { uniqueNodes.add(socialProofs[socialProofType].keys()[i]); if (uniqueNodes.size() >= minUserSocialProofSize) { return false; } } } } } return true; }
Example #28
Source File: ValueInTransformFunction.java From incubator-pinot with Apache License 2.0 | 5 votes |
private static long[] filterLongs(LongSet longSet, long[] source) { LongList longList = new LongArrayList(); for (long value : source) { if (longSet.contains(value)) { longList.add(value); } } if (longList.size() == source.length) { return source; } else { return longList.toLongArray(); } }
Example #29
Source File: TopSecondDegreeByCountTweetRecsGenerator.java From GraphJet with Apache License 2.0 | 5 votes |
private static boolean isLessThanMinUserSocialProofSizeCombined( SmallArrayBasedLongToDoubleMap[] socialProofs, int minUserSocialProofSize, Set<byte[]> socialProofTypeUnions) { if (socialProofTypeUnions.isEmpty() || // check if the size of any social proof union is greater than minUserSocialProofSize before dedupping isSocialProofUnionSizeLessThanMin(socialProofs, minUserSocialProofSize, socialProofTypeUnions)) { return true; } LongSet uniqueNodes = new LongOpenHashSet(minUserSocialProofSize); for (byte[] socialProofTypeUnion: socialProofTypeUnions) { // Clear removes all elements, but does not change the size of the set. // Thus, we only use one LongOpenHashSet with at most a size of 2*minUserSocialProofSize uniqueNodes.clear(); for (byte socialProofType: socialProofTypeUnion) { if (socialProofs[socialProofType] != null) { for (int i = 0; i < socialProofs[socialProofType].size(); i++) { uniqueNodes.add(socialProofs[socialProofType].keys()[i]); if (uniqueNodes.size() >= minUserSocialProofSize) { return false; } } } } } return true; }
Example #30
Source File: SalsaRequest.java From GraphJet with Apache License 2.0 | 5 votes |
/** * The constructor should only be called via {@link SalsaRequestBuilder}. * @param queryNode is the query node for running SALSA * @param leftSeedNodesWithWeight is the set of seed nodes to use for SALSA, with weights being * the proportion of random walks to start here. We do NOT assume * that the queryNode is added to this. * @param toBeFiltered is the set of RHS nodes to be filtered from the output * @param numRandomWalks is the total number of random walks to run * @param maxRandomWalkLength is the maximum length of a random walk * @param resetProbability is the probability of reset in SALSA. Note that reset is only * done on backward iterations. * @param maxNumResults is the maximum number of results that SALSA will return * @param maxSocialProofSize is the maximum size of social proof per type to return. Set * this to 0 to return no social proof * @param maxSocialProofTypeSize is the maximum size of social proof types in the graph. * @param socialProofTypes is the social proof types to return * @param queryNodeWeightFraction is the relative proportion of random walks to start at the * queryNode in the first iteration. This parameter is only used * @param removeCustomizedBitsNodes removes tweets with metadata information embedded into top * four bits * @param resultFilterChain is the chain of filters to be applied */ protected SalsaRequest( long queryNode, Long2DoubleMap leftSeedNodesWithWeight, LongSet toBeFiltered, int numRandomWalks, int maxRandomWalkLength, double resetProbability, int maxNumResults, int maxSocialProofSize, int maxSocialProofTypeSize, byte[] socialProofTypes, double queryNodeWeightFraction, boolean removeCustomizedBitsNodes, ResultFilterChain resultFilterChain) { super(queryNode, toBeFiltered, socialProofTypes); this.leftSeedNodesWithWeight = leftSeedNodesWithWeight; this.numRandomWalks = numRandomWalks; this.maxRandomWalkLength = maxRandomWalkLength; this.resetProbability = resetProbability; this.maxNumResults = maxNumResults; this.maxSocialProofSize = maxSocialProofSize; this.maxSocialProofTypeSize = maxSocialProofTypeSize; this.queryNodeWeightFraction = queryNodeWeightFraction; this.removeCustomizedBitsNodes = removeCustomizedBitsNodes; this.resultFilterChain = resultFilterChain; }