com.carrotsearch.hppc.IntSet Java Examples
The following examples show how to use
com.carrotsearch.hppc.IntSet.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RoutingBuilderTest.java From crate with Apache License 2.0 | 6 votes |
@Test public void testBuildReaderAllocations() { RoutingBuilder routingBuilder = new RoutingBuilder(clusterService.state(), routingProvider); routingBuilder.allocateRouting(tableInfo, WhereClause.MATCH_ALL, RoutingProvider.ShardSelection.ANY, null); ReaderAllocations readerAllocations = routingBuilder.buildReaderAllocations(); assertThat(readerAllocations.indices().size(), is(1)); assertThat(readerAllocations.indices().get(0), is(relationName.indexNameOrAlias())); assertThat(readerAllocations.nodeReaders().size(), is(2)); IntSet n1 = readerAllocations.nodeReaders().get("n1"); assertThat(n1.size(), is(2)); assertThat(n1.contains(0), is(true)); assertThat(n1.contains(2), is(true)); IntSet n2 = readerAllocations.nodeReaders().get("n2"); assertThat(n2.size(), is(2)); assertThat(n2.contains(1), is(true)); assertThat(n2.contains(3), is(true)); assertThat(readerAllocations.bases().get(relationName.indexNameOrAlias()), is(0)); ReaderAllocations readerAllocations2 = routingBuilder.buildReaderAllocations(); assertThat(readerAllocations, CoreMatchers.not(is(readerAllocations2))); }
Example #2
Source File: FetchMapper.java From crate with Apache License 2.0 | 6 votes |
@Override public CompletableFuture<? extends Iterator<Row>> apply(ReaderBuckets readerBuckets, boolean isLastCall) { List<CompletableFuture<IntObjectMap<? extends Bucket>>> futures = new ArrayList<>(); Iterator<Map.Entry<String, IntSet>> it = readerIdsByNode.entrySet().iterator(); while (it.hasNext()) { Map.Entry<String, IntSet> entry = it.next(); IntObjectHashMap<IntContainer> toFetch = readerBuckets.generateToFetch(entry.getValue()); if (toFetch.isEmpty() && !isLastCall) { continue; } final String nodeId = entry.getKey(); try { futures.add(fetchOperation.fetch(nodeId, toFetch, isLastCall)); } catch (Throwable t) { futures.add(CompletableFuture.failedFuture(t)); } if (isLastCall) { it.remove(); } } return CompletableFutures.allAsList(futures).thenApply(readerBuckets::getOutputRows); }
Example #3
Source File: FetchProjection.java From crate with Apache License 2.0 | 6 votes |
@SuppressWarnings({"rawtypes"}) public Map<String, ? extends IntObjectMap<Streamer[]>> generateStreamersGroupedByReaderAndNode() { HashMap<String, IntObjectHashMap<Streamer[]>> streamersByReaderByNode = new HashMap<>(); for (Map.Entry<String, IntSet> entry : nodeReaders.entrySet()) { IntObjectHashMap<Streamer[]> streamersByReaderId = new IntObjectHashMap<>(); String nodeId = entry.getKey(); streamersByReaderByNode.put(nodeId, streamersByReaderId); for (IntCursor readerIdCursor : entry.getValue()) { int readerId = readerIdCursor.value; String index = readerIndices.floorEntry(readerId).getValue(); RelationName relationName = indicesToIdents.get(index); FetchSource fetchSource = fetchSources.get(relationName); if (fetchSource == null) { continue; } streamersByReaderId.put(readerIdCursor.value, Symbols.streamerArray(fetchSource.references())); } } return streamersByReaderByNode; }
Example #4
Source File: FetchProjection.java From crate with Apache License 2.0 | 6 votes |
public FetchProjection(int fetchPhaseId, int suppliedFetchSize, Map<RelationName, FetchSource> fetchSources, List<Symbol> outputSymbols, Map<String, IntSet> nodeReaders, TreeMap<Integer, String> readerIndices, Map<String, RelationName> indicesToIdents) { assert outputSymbols.stream().noneMatch(s -> SymbolVisitors.any(x -> x instanceof ScopedSymbol || x instanceof SelectSymbol, s)) : "Cannot operate on Field or SelectSymbol symbols: " + outputSymbols; this.fetchPhaseId = fetchPhaseId; this.fetchSources = fetchSources; this.outputSymbols = outputSymbols; this.nodeReaders = nodeReaders; this.readerIndices = readerIndices; this.indicesToIdents = indicesToIdents; this.fetchSize = boundedFetchSize(suppliedFetchSize, MAX_FETCH_SIZE); }
Example #5
Source File: IndexRoutingTable.java From crate with Apache License 2.0 | 6 votes |
/** * Initializes an index, to be restored from snapshot */ private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) { assert indexMetaData.getIndex().equals(index); if (!shards.isEmpty()) { throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) { ShardId shardId = new ShardId(index, shardNumber); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { boolean primary = i == 0; if (asNew && ignoreShards.contains(shardNumber)) { // This shards wasn't completely snapshotted - restore it as new shard indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo)); } else { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo)); } } shards.put(shardNumber, indexShardRoutingBuilder.build()); } return this; }
Example #6
Source File: IndexRoutingTable.java From Elasticsearch with Apache License 2.0 | 6 votes |
/** * Initializes an index, to be restored from snapshot */ private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) { if (!shards.isEmpty()) { throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) { IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId)); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { if (asNew && ignoreShards.contains(shardId)) { // This shards wasn't completely snapshotted - restore it as new shard indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo)); } else { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo)); } } shards.put(shardId, indexShardRoutingBuilder.build()); } return this; }
Example #7
Source File: TitanPartitionGraphTest.java From titan1withtp3.1 with Apache License 2.0 | 6 votes |
@Test public void testKeybasedGraphPartitioning() { Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), false, option(VertexIDAssigner.PLACEMENT_STRATEGY), PropertyPlacementStrategy.class.getName(), option(PropertyPlacementStrategy.PARTITION_KEY), "clusterId"}; clopen(options); int[] groupDegrees = {5,5,5,5,5,5,5,5}; int numVertices = setupGroupClusters(groupDegrees,CommitMode.PER_VERTEX); IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids for (int i=0;i<groupDegrees.length;i++) { TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i)); int partitionId = -1; for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) { if (partitionId<0) partitionId = getPartitionID(v); assertEquals(partitionId,getPartitionID(v)); partitionIds.add(partitionId); } } assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail }
Example #8
Source File: FetchProjection.java From Elasticsearch with Apache License 2.0 | 5 votes |
public FetchProjection(int collectPhaseId, Map<TableIdent, FetchSource> fetchSources, List<Symbol> outputSymbols, Map<String, IntSet> nodeReaders, TreeMap<Integer, String> readerIndices, Map<String, TableIdent> indicesToIdents) { this.collectPhaseId = collectPhaseId; this.fetchSources = fetchSources; this.outputSymbols = outputSymbols; this.nodeReaders = nodeReaders; this.readerIndices = readerIndices; this.indicesToIdents = indicesToIdents; }
Example #9
Source File: ReaderBuckets.java From crate with Apache License 2.0 | 5 votes |
public IntObjectHashMap<IntContainer> generateToFetch(IntSet readerIds) { IntObjectHashMap<IntContainer> toFetch = new IntObjectHashMap<>(readerIds.size()); for (IntCursor readerIdCursor : readerIds) { ReaderBucket readerBucket = readerBuckets.get(readerIdCursor.value); if (readerBucket != null && readerBucket.docs.size() > 0) { toFetch.put(readerIdCursor.value, readerBucket.docs.keys()); } } return toFetch; }
Example #10
Source File: IndexRoutingTable.java From crate with Apache License 2.0 | 5 votes |
/** * Initializes a new empty index, to be restored from a snapshot */ public Builder initializeAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) { final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED, "restore_source[" + recoverySource.snapshot().getRepository() + "/" + recoverySource.snapshot().getSnapshotId().getName() + "]"); return initializeAsRestore(indexMetaData, recoverySource, ignoreShards, true, unassignedInfo); }
Example #11
Source File: TitanPartitionGraphTest.java From titan1withtp3.1 with Apache License 2.0 | 5 votes |
private void testPartitionSpread(boolean flush, boolean batchCommit) { Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), flush}; clopen(options); int[] groupDegrees = {10,15,10,17,10,4,7,20,11}; int numVertices = setupGroupClusters(groupDegrees,batchCommit?CommitMode.BATCH:CommitMode.PER_VERTEX); IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids for (int i=0;i<groupDegrees.length;i++) { TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i)); assertCount(groupDegrees[i],g.edges(Direction.OUT,"contain")); assertCount(groupDegrees[i],g.edges(Direction.IN,"member")); assertCount(groupDegrees[i],g.query().direction(Direction.OUT).edges()); assertCount(groupDegrees[i],g.query().direction(Direction.IN).edges()); assertCount(groupDegrees[i]*2,g.query().edges()); for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) { int pid = getPartitionID(v); partitionIds.add(pid); assertEquals(g, getOnlyElement(v.query().direction(Direction.OUT).labels("member").vertices())); VertexList vlist = v.query().direction(Direction.IN).labels("contain").vertexIds(); assertEquals(1,vlist.size()); assertEquals(pid,idManager.getPartitionId(vlist.getID(0))); assertEquals(g,vlist.get(0)); } } if (flush || !batchCommit) { //In these cases we would expect significant spread across partitions assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail } else { assertEquals(1,partitionIds.size()); //No spread in this case } }
Example #12
Source File: FetchProjector.java From Elasticsearch with Apache License 2.0 | 5 votes |
public FetchProjector(TransportFetchNodeAction transportFetchNodeAction, ThreadPool threadPool, Functions functions, UUID jobId, int collectPhaseId, Map<TableIdent, FetchSource> fetchSources, List<Symbol> outputSymbols, Map<String, IntSet> nodeReaders, TreeMap<Integer, String> readerIndices, Map<String, TableIdent> indicesToIdents) { this.transportFetchNodeAction = transportFetchNodeAction; this.threadPool = threadPool; this.jobId = jobId; this.collectPhaseId = collectPhaseId; this.fetchSources = fetchSources; this.nodeReaders = nodeReaders; this.readerIndices = readerIndices; this.indicesToIdents = indicesToIdents; FetchRowInputSymbolVisitor rowInputSymbolVisitor = new FetchRowInputSymbolVisitor(functions); this.collectRowContext = new FetchRowInputSymbolVisitor.Context(fetchSources); List<Input<?>> inputs = new ArrayList<>(outputSymbols.size()); for (Symbol symbol : outputSymbols) { inputs.add(rowInputSymbolVisitor.process(symbol, collectRowContext)); } outputRow = new InputRow(inputs); }
Example #13
Source File: RoutingTable.java From crate with Apache License 2.0 | 4 votes |
public Builder addAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) .initializeAsNewRestore(indexMetaData, recoverySource, ignoreShards); add(indexRoutingBuilder); return this; }
Example #14
Source File: IndexRoutingTable.java From Elasticsearch with Apache License 2.0 | 4 votes |
/** * Initializes a new empty index, to be restored from a snapshot */ public Builder initializeAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) { return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED, "restore_source[" + restoreSource.snapshotId().getRepository() + "/" + restoreSource.snapshotId().getSnapshot() + "]")); }
Example #15
Source File: RoutingTable.java From Elasticsearch with Apache License 2.0 | 4 votes |
public Builder addAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) .initializeAsNewRestore(indexMetaData, restoreSource, ignoreShards); add(indexRoutingBuilder); return this; }
Example #16
Source File: ReaderAllocations.java From crate with Apache License 2.0 | 4 votes |
public Map<String, IntSet> nodeReaders() { return nodeReaders; }
Example #17
Source File: AbstractInsertAnalyzedStatement.java From Elasticsearch with Apache License 2.0 | 4 votes |
public IntSet primaryKeyColumnIndices() { return primaryKeyColumnIndices; }
Example #18
Source File: FetchProjection.java From crate with Apache License 2.0 | 4 votes |
public Map<String, IntSet> nodeReaders() { return nodeReaders; }
Example #19
Source File: AbstractInsertAnalyzedStatement.java From Elasticsearch with Apache License 2.0 | 4 votes |
public IntSet partitionedByIndices() { return partitionedByColumnsIndices; }
Example #20
Source File: Planner.java From Elasticsearch with Apache License 2.0 | 4 votes |
public Map<String, IntSet> nodeReaders() { return nodeReaders; }
Example #21
Source File: FetchMapper.java From crate with Apache License 2.0 | 4 votes |
public FetchMapper(FetchOperation fetchOperation, Map<String, IntSet> readerIdsByNode) { this.fetchOperation = fetchOperation; this.readerIdsByNode = readerIdsByNode; }
Example #22
Source File: FetchProjection.java From Elasticsearch with Apache License 2.0 | 4 votes |
public Map<String, IntSet> nodeReaders() { return nodeReaders; }