com.carrotsearch.hppc.IntSet Java Examples

The following examples show how to use com.carrotsearch.hppc.IntSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RoutingBuilderTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testBuildReaderAllocations() {
    RoutingBuilder routingBuilder = new RoutingBuilder(clusterService.state(), routingProvider);
    routingBuilder.allocateRouting(tableInfo, WhereClause.MATCH_ALL, RoutingProvider.ShardSelection.ANY, null);

    ReaderAllocations readerAllocations = routingBuilder.buildReaderAllocations();

    assertThat(readerAllocations.indices().size(), is(1));
    assertThat(readerAllocations.indices().get(0), is(relationName.indexNameOrAlias()));
    assertThat(readerAllocations.nodeReaders().size(), is(2));

    IntSet n1 = readerAllocations.nodeReaders().get("n1");
    assertThat(n1.size(), is(2));
    assertThat(n1.contains(0), is(true));
    assertThat(n1.contains(2), is(true));

    IntSet n2 = readerAllocations.nodeReaders().get("n2");
    assertThat(n2.size(), is(2));
    assertThat(n2.contains(1), is(true));
    assertThat(n2.contains(3), is(true));

    assertThat(readerAllocations.bases().get(relationName.indexNameOrAlias()), is(0));

    ReaderAllocations readerAllocations2 = routingBuilder.buildReaderAllocations();
    assertThat(readerAllocations, CoreMatchers.not(is(readerAllocations2)));
}
 
Example #2
Source File: FetchMapper.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<? extends Iterator<Row>> apply(ReaderBuckets readerBuckets, boolean isLastCall) {
    List<CompletableFuture<IntObjectMap<? extends Bucket>>> futures = new ArrayList<>();
    Iterator<Map.Entry<String, IntSet>> it = readerIdsByNode.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<String, IntSet> entry = it.next();
        IntObjectHashMap<IntContainer> toFetch = readerBuckets.generateToFetch(entry.getValue());
        if (toFetch.isEmpty() && !isLastCall) {
            continue;
        }
        final String nodeId = entry.getKey();
        try {
            futures.add(fetchOperation.fetch(nodeId, toFetch, isLastCall));
        } catch (Throwable t) {
            futures.add(CompletableFuture.failedFuture(t));
        }
        if (isLastCall) {
            it.remove();
        }
    }
    return CompletableFutures.allAsList(futures).thenApply(readerBuckets::getOutputRows);
}
 
Example #3
Source File: FetchProjection.java    From crate with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"rawtypes"})
public Map<String, ? extends IntObjectMap<Streamer[]>> generateStreamersGroupedByReaderAndNode() {
    HashMap<String, IntObjectHashMap<Streamer[]>> streamersByReaderByNode = new HashMap<>();
    for (Map.Entry<String, IntSet> entry : nodeReaders.entrySet()) {
        IntObjectHashMap<Streamer[]> streamersByReaderId = new IntObjectHashMap<>();
        String nodeId = entry.getKey();
        streamersByReaderByNode.put(nodeId, streamersByReaderId);
        for (IntCursor readerIdCursor : entry.getValue()) {
            int readerId = readerIdCursor.value;
            String index = readerIndices.floorEntry(readerId).getValue();
            RelationName relationName = indicesToIdents.get(index);
            FetchSource fetchSource = fetchSources.get(relationName);
            if (fetchSource == null) {
                continue;
            }
            streamersByReaderId.put(readerIdCursor.value, Symbols.streamerArray(fetchSource.references()));
        }
    }
    return streamersByReaderByNode;
}
 
Example #4
Source File: FetchProjection.java    From crate with Apache License 2.0 6 votes vote down vote up
public FetchProjection(int fetchPhaseId,
                       int suppliedFetchSize,
                       Map<RelationName, FetchSource> fetchSources,
                       List<Symbol> outputSymbols,
                       Map<String, IntSet> nodeReaders,
                       TreeMap<Integer, String> readerIndices,
                       Map<String, RelationName> indicesToIdents) {
    assert outputSymbols.stream().noneMatch(s ->
        SymbolVisitors.any(x -> x instanceof ScopedSymbol || x instanceof SelectSymbol, s))
        : "Cannot operate on Field or SelectSymbol symbols: " + outputSymbols;
    this.fetchPhaseId = fetchPhaseId;
    this.fetchSources = fetchSources;
    this.outputSymbols = outputSymbols;
    this.nodeReaders = nodeReaders;
    this.readerIndices = readerIndices;
    this.indicesToIdents = indicesToIdents;
    this.fetchSize = boundedFetchSize(suppliedFetchSize, MAX_FETCH_SIZE);
}
 
Example #5
Source File: IndexRoutingTable.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Initializes an index, to be restored from snapshot
 */
private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
    assert indexMetaData.getIndex().equals(index);
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) {
        ShardId shardId = new ShardId(index, shardNumber);
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            boolean primary = i == 0;
            if (asNew && ignoreShards.contains(shardNumber)) {
                // This shards wasn't completely snapshotted - restore it as new shard
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
                    primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo));
            } else {
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary,
                    primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo));
            }
        }
        shards.put(shardNumber, indexShardRoutingBuilder.build());
    }
    return this;
}
 
Example #6
Source File: IndexRoutingTable.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Initializes an index, to be restored from snapshot
 */
private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId));
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            if (asNew && ignoreShards.contains(shardId)) {
                // This shards wasn't completely snapshotted - restore it as new shard
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
            } else {
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo));
            }
        }
        shards.put(shardId, indexShardRoutingBuilder.build());
    }
    return this;
}
 
Example #7
Source File: TitanPartitionGraphTest.java    From titan1withtp3.1 with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeybasedGraphPartitioning() {
    Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), false,
                        option(VertexIDAssigner.PLACEMENT_STRATEGY), PropertyPlacementStrategy.class.getName(),
            option(PropertyPlacementStrategy.PARTITION_KEY), "clusterId"};
    clopen(options);

    int[] groupDegrees = {5,5,5,5,5,5,5,5};
    int numVertices = setupGroupClusters(groupDegrees,CommitMode.PER_VERTEX);

    IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids
    for (int i=0;i<groupDegrees.length;i++) {
        TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i));
        int partitionId = -1;
        for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) {
            if (partitionId<0) partitionId = getPartitionID(v);
            assertEquals(partitionId,getPartitionID(v));
            partitionIds.add(partitionId);
        }
    }
    assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail
}
 
Example #8
Source File: FetchProjection.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public FetchProjection(int collectPhaseId,
                       Map<TableIdent, FetchSource> fetchSources,
                       List<Symbol> outputSymbols,
                       Map<String, IntSet> nodeReaders,
                       TreeMap<Integer, String> readerIndices,
                       Map<String, TableIdent> indicesToIdents) {
    this.collectPhaseId = collectPhaseId;
    this.fetchSources = fetchSources;
    this.outputSymbols = outputSymbols;
    this.nodeReaders = nodeReaders;
    this.readerIndices = readerIndices;
    this.indicesToIdents = indicesToIdents;
}
 
Example #9
Source File: ReaderBuckets.java    From crate with Apache License 2.0 5 votes vote down vote up
public IntObjectHashMap<IntContainer> generateToFetch(IntSet readerIds) {
    IntObjectHashMap<IntContainer> toFetch = new IntObjectHashMap<>(readerIds.size());
    for (IntCursor readerIdCursor : readerIds) {
        ReaderBucket readerBucket = readerBuckets.get(readerIdCursor.value);
        if (readerBucket != null && readerBucket.docs.size() > 0) {
            toFetch.put(readerIdCursor.value, readerBucket.docs.keys());
        }
    }
    return toFetch;
}
 
Example #10
Source File: IndexRoutingTable.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Initializes a new empty index, to be restored from a snapshot
 */
public Builder initializeAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) {
    final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED,
                                                             "restore_source[" + recoverySource.snapshot().getRepository() + "/" +
                                                                 recoverySource.snapshot().getSnapshotId().getName() + "]");
    return initializeAsRestore(indexMetaData, recoverySource, ignoreShards, true, unassignedInfo);
}
 
Example #11
Source File: TitanPartitionGraphTest.java    From titan1withtp3.1 with Apache License 2.0 5 votes vote down vote up
private void testPartitionSpread(boolean flush, boolean batchCommit) {
    Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), flush};
    clopen(options);

    int[] groupDegrees = {10,15,10,17,10,4,7,20,11};
    int numVertices = setupGroupClusters(groupDegrees,batchCommit?CommitMode.BATCH:CommitMode.PER_VERTEX);

    IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids
    for (int i=0;i<groupDegrees.length;i++) {
        TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i));
        assertCount(groupDegrees[i],g.edges(Direction.OUT,"contain"));
        assertCount(groupDegrees[i],g.edges(Direction.IN,"member"));
        assertCount(groupDegrees[i],g.query().direction(Direction.OUT).edges());
        assertCount(groupDegrees[i],g.query().direction(Direction.IN).edges());
        assertCount(groupDegrees[i]*2,g.query().edges());
        for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) {
            int pid = getPartitionID(v);
            partitionIds.add(pid);
            assertEquals(g, getOnlyElement(v.query().direction(Direction.OUT).labels("member").vertices()));
            VertexList vlist = v.query().direction(Direction.IN).labels("contain").vertexIds();
            assertEquals(1,vlist.size());
            assertEquals(pid,idManager.getPartitionId(vlist.getID(0)));
            assertEquals(g,vlist.get(0));
        }
    }
    if (flush || !batchCommit) { //In these cases we would expect significant spread across partitions
        assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail
    } else {
        assertEquals(1,partitionIds.size()); //No spread in this case
    }
}
 
Example #12
Source File: FetchProjector.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public FetchProjector(TransportFetchNodeAction transportFetchNodeAction,
                      ThreadPool threadPool,
                      Functions functions,
                      UUID jobId,
                      int collectPhaseId,
                      Map<TableIdent, FetchSource> fetchSources,
                      List<Symbol> outputSymbols,
                      Map<String, IntSet> nodeReaders,
                      TreeMap<Integer, String> readerIndices,
                      Map<String, TableIdent> indicesToIdents) {
    this.transportFetchNodeAction = transportFetchNodeAction;
    this.threadPool = threadPool;
    this.jobId = jobId;
    this.collectPhaseId = collectPhaseId;
    this.fetchSources = fetchSources;
    this.nodeReaders = nodeReaders;
    this.readerIndices = readerIndices;
    this.indicesToIdents = indicesToIdents;

    FetchRowInputSymbolVisitor rowInputSymbolVisitor = new FetchRowInputSymbolVisitor(functions);

    this.collectRowContext = new FetchRowInputSymbolVisitor.Context(fetchSources);

    List<Input<?>> inputs = new ArrayList<>(outputSymbols.size());
    for (Symbol symbol : outputSymbols) {
        inputs.add(rowInputSymbolVisitor.process(symbol, collectRowContext));
    }

    outputRow = new InputRow(inputs);
}
 
Example #13
Source File: RoutingTable.java    From crate with Apache License 2.0 4 votes vote down vote up
public Builder addAsNewRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards) {
    IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
            .initializeAsNewRestore(indexMetaData, recoverySource, ignoreShards);
    add(indexRoutingBuilder);
    return this;
}
 
Example #14
Source File: IndexRoutingTable.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
/**
 * Initializes a new empty index, to be restored from a snapshot
 */
public Builder initializeAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) {
    return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED, "restore_source[" + restoreSource.snapshotId().getRepository() + "/" + restoreSource.snapshotId().getSnapshot() + "]"));
}
 
Example #15
Source File: RoutingTable.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public Builder addAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) {
    IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
            .initializeAsNewRestore(indexMetaData, restoreSource, ignoreShards);
    add(indexRoutingBuilder);
    return this;
}
 
Example #16
Source File: ReaderAllocations.java    From crate with Apache License 2.0 4 votes vote down vote up
public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}
 
Example #17
Source File: AbstractInsertAnalyzedStatement.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public IntSet primaryKeyColumnIndices() {
    return primaryKeyColumnIndices;
}
 
Example #18
Source File: FetchProjection.java    From crate with Apache License 2.0 4 votes vote down vote up
public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}
 
Example #19
Source File: AbstractInsertAnalyzedStatement.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public IntSet partitionedByIndices() {
    return partitionedByColumnsIndices;
}
 
Example #20
Source File: Planner.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}
 
Example #21
Source File: FetchMapper.java    From crate with Apache License 2.0 4 votes vote down vote up
public FetchMapper(FetchOperation fetchOperation, Map<String, IntSet> readerIdsByNode) {
    this.fetchOperation = fetchOperation;
    this.readerIdsByNode = readerIdsByNode;
}
 
Example #22
Source File: FetchProjection.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public Map<String, IntSet> nodeReaders() {
    return nodeReaders;
}