org.elasticsearch.common.util.set.Sets Java Examples

The following examples show how to use org.elasticsearch.common.util.set.Sets. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BalancedShardsAllocator.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public ShardAllocationDecision decideShardAllocation(final ShardRouting shard, final RoutingAllocation allocation) {
    Balancer balancer = new Balancer(LOGGER, allocation, weightFunction, threshold);
    AllocateUnassignedDecision allocateUnassignedDecision = AllocateUnassignedDecision.NOT_TAKEN;
    MoveDecision moveDecision = MoveDecision.NOT_TAKEN;
    if (shard.unassigned()) {
        allocateUnassignedDecision = balancer.decideAllocateUnassigned(shard, Sets.newHashSet());
    } else {
        moveDecision = balancer.decideMove(shard);
        if (moveDecision.isDecisionTaken() && moveDecision.canRemain()) {
            MoveDecision rebalanceDecision = balancer.decideRebalance(shard);
            moveDecision = rebalanceDecision.withRemainDecision(moveDecision.getCanRemainDecision());
        }
    }
    return new ShardAllocationDecision(allocateUnassignedDecision, moveDecision);
}
 
Example #2
Source File: MetaDataDeleteIndexService.java    From crate with Apache License 2.0 6 votes vote down vote up
public void deleteIndices(final DeleteIndexClusterStateUpdateRequest request,
        final ActionListener<ClusterStateUpdateResponse> listener) {
    if (request.indices() == null || request.indices().length == 0) {
        throw new IllegalArgumentException("Index name is required");
    }

    clusterService.submitStateUpdateTask(
        "delete-index " + Arrays.toString(request.indices()),
        new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {

            @Override
            protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
                return new ClusterStateUpdateResponse(acknowledged);
            }

            @Override
            public ClusterState execute(final ClusterState currentState) {
                return deleteIndices(currentState, Sets.newHashSet(request.indices()));
            }
        }
    );
}
 
Example #3
Source File: AliasMetaData.java    From crate with Apache License 2.0 6 votes vote down vote up
public AliasMetaData(StreamInput in) throws IOException {
    alias = in.readString();
    if (in.readBoolean()) {
        filter = CompressedXContent.readCompressedString(in);
    } else {
        filter = null;
    }
    if (in.readBoolean()) {
        indexRouting = in.readString();
    } else {
        indexRouting = null;
    }
    if (in.readBoolean()) {
        searchRouting = in.readString();
        searchRoutingValues = Collections.unmodifiableSet(Sets.newHashSet(Strings.splitStringByCommaToArray(searchRouting)));
    } else {
        searchRouting = null;
        searchRoutingValues = emptySet();
    }
    writeIndex = in.readOptionalBoolean();
}
 
Example #4
Source File: MockPageCacheRecycler.java    From crate with Apache License 2.0 6 votes vote down vote up
public static void ensureAllPagesAreReleased() throws Exception {
    final Map<Object, Throwable> masterCopy = new HashMap<>(ACQUIRED_PAGES);
    if (!masterCopy.isEmpty()) {
        // not empty, we might be executing on a shared cluster that keeps on obtaining
        // and releasing pages, lets make sure that after a reasonable timeout, all master
        // copy (snapshot) have been released
        boolean success =
                ESTestCase.awaitBusy(() -> Sets.haveEmptyIntersection(masterCopy.keySet(), ACQUIRED_PAGES.keySet()));
        if (!success) {
            masterCopy.keySet().retainAll(ACQUIRED_PAGES.keySet());
            ACQUIRED_PAGES.keySet().removeAll(masterCopy.keySet()); // remove all existing master copy we will report on
            if (!masterCopy.isEmpty()) {
                Iterator<Throwable> causes = masterCopy.values().iterator();
                Throwable firstCause = causes.next();
                RuntimeException exception = new RuntimeException(masterCopy.size() + " pages have not been released", firstCause);
                while (causes.hasNext()) {
                    exception.addSuppressed(causes.next());
                }
                throw exception;
            }
        }
    }
}
 
Example #5
Source File: MockBigArrays.java    From crate with Apache License 2.0 6 votes vote down vote up
public static void ensureAllArraysAreReleased() throws Exception {
    final Map<Object, Object> masterCopy = new HashMap<>(ACQUIRED_ARRAYS);
    if (!masterCopy.isEmpty()) {
        // not empty, we might be executing on a shared cluster that keeps on obtaining
        // and releasing arrays, lets make sure that after a reasonable timeout, all master
        // copy (snapshot) have been released
        boolean success = ESTestCase.awaitBusy(() -> Sets.haveEmptyIntersection(masterCopy.keySet(), ACQUIRED_ARRAYS.keySet()));
        if (!success) {
            masterCopy.keySet().retainAll(ACQUIRED_ARRAYS.keySet());
            ACQUIRED_ARRAYS.keySet().removeAll(masterCopy.keySet()); // remove all existing master copy we will report on
            if (!masterCopy.isEmpty()) {
                Iterator<Object> causes = masterCopy.values().iterator();
                Object firstCause = causes.next();
                RuntimeException exception = new RuntimeException(masterCopy.size() + " arrays have not been released",
                        firstCause instanceof Throwable ? (Throwable) firstCause : null);
                while (causes.hasNext()) {
                    Object cause = causes.next();
                    if (cause instanceof Throwable) {
                        exception.addSuppressed((Throwable) cause);
                    }
                }
                throw exception;
            }
        }
    }
}
 
Example #6
Source File: CoordinatorTests.java    From crate with Apache License 2.0 6 votes vote down vote up
public void testCannotSetInitialConfigurationWithoutQuorum() {
    final Cluster cluster = new Cluster(randomIntBetween(1, 5));
    final Coordinator coordinator = cluster.getAnyNode().coordinator;
    final VotingConfiguration unknownNodeConfiguration = new VotingConfiguration(
        Sets.newHashSet(coordinator.getLocalNode().getId(), "unknown-node"));
    final String exceptionMessage = expectThrows(CoordinationStateRejectedException.class,
        () -> coordinator.setInitialConfiguration(unknownNodeConfiguration)).getMessage();
    assertThat(exceptionMessage,
        startsWith("not enough nodes discovered to form a quorum in the initial configuration [knownNodes=["));
    assertThat(exceptionMessage, containsString("unknown-node"));
    assertThat(exceptionMessage, containsString(coordinator.getLocalNode().toString()));

    // This is VERY BAD: setting a _different_ initial configuration. Yet it works if the first attempt will never be a quorum.
    assertTrue(coordinator.setInitialConfiguration(new VotingConfiguration(Collections.singleton(coordinator.getLocalNode().getId()))));
    cluster.stabilise();
}
 
Example #7
Source File: PublicationTests.java    From crate with Apache License 2.0 5 votes vote down vote up
public void testClusterStatePublishingFailsOrTimesOutBeforeCommit() throws InterruptedException {
    VotingConfiguration config = new VotingConfiguration(Sets.newHashSet(n1.getId(), n2.getId()));
    initializeCluster(config);

    AssertingAckListener ackListener = new AssertingAckListener(nodes.size());
    DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(n1).add(n2).add(n3).localNodeId(n1.getId()).build();
    MockPublication publication = node1.publish(CoordinationStateTests.clusterState(1L, 2L,
        discoveryNodes, config, config, 42L), ackListener, Collections.emptySet());

    boolean timeOut = randomBoolean();
    publication.pendingPublications.entrySet().stream().collect(shuffle()).forEach(e -> {
        if (e.getKey().equals(n2)) {
            if (timeOut) {
                publication.cancel("timed out");
            } else {
                e.getValue().onFailure(new TransportException(new Exception("dummy failure")));
            }
            assertTrue(publication.completed);
            assertFalse(publication.committed);
        } else if (randomBoolean()) {
            PublishResponse publishResponse = nodeResolver.apply(e.getKey()).coordinationState.handlePublishRequest(
                publication.publishRequest);
            e.getValue().onResponse(new PublishWithJoinResponse(publishResponse, Optional.empty()));
        }
    });

    assertThat(publication.pendingCommits.keySet(), equalTo(Collections.emptySet()));
    assertNull(publication.applyCommit);
    assertTrue(publication.completed);
    assertFalse(publication.committed);

    List<Tuple<DiscoveryNode, Throwable>> errors = ackListener.awaitErrors(0L, TimeUnit.SECONDS);
    assertThat(errors.size(), equalTo(3));
    assertThat(errors.stream().map(Tuple::v1).collect(Collectors.toList()), containsInAnyOrder(n1, n2, n3));
    errors.stream().forEach(tuple ->
        assertThat(tuple.v2().getMessage(), containsString(timeOut ? "timed out" :
            tuple.v1().equals(n2) ? "dummy failure" : "non-failed nodes do not form a quorum")));
}
 
Example #8
Source File: PrivilegesMetaDataUpgraderTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testExistingUserWithPrivilegesDoesntGetMore() throws Exception {
    Map<String, MetaData.Custom> customMap = new HashMap<>(1);
    customMap.put(UsersMetaData.TYPE, new UsersMetaData(UserDefinitions.SINGLE_USER_ONLY));
    customMap.put(UsersPrivilegesMetaData.TYPE, new UsersPrivilegesMetaData(
        MapBuilder.<String, Set<Privilege>>newMapBuilder()
            .put("Arthur", Sets.newHashSet(
                new Privilege(Privilege.State.GRANT, Privilege.Type.DQL, Privilege.Clazz.CLUSTER, null, "crate")))
            .map()));
    Map<String, MetaData.Custom> oldCustomMap = new HashMap<>(customMap);

    Map<String, MetaData.Custom> newCustomMap = UPGRADER.apply(Settings.EMPTY, customMap);
    assertThat(newCustomMap, is(oldCustomMap));
}
 
Example #9
Source File: CrateDummyClusterServiceUnitTest.java    From crate with Apache License 2.0 5 votes vote down vote up
protected ClusterService createClusterService(Collection<Setting<?>> additionalClusterSettings, Version version) {
    Set<Setting<?>> clusterSettingsSet = Sets.newHashSet(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    clusterSettingsSet.addAll(additionalClusterSettings);
    ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, clusterSettingsSet);
    ClusterService clusterService = new ClusterService(
        Settings.builder()
            .put("cluster.name", "ClusterServiceTests")
            .put(Node.NODE_NAME_SETTING.getKey(), NODE_NAME)
            .build(),
        clusterSettings,
        THREAD_POOL
    );
    clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService());
    DiscoveryNode discoveryNode = new DiscoveryNode(
        NODE_NAME,
        NODE_ID,
        buildNewFakeTransportAddress(),
        Collections.emptyMap(),
        new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())),
        version
    );
    DiscoveryNodes nodes = DiscoveryNodes.builder()
        .add(discoveryNode)
        .localNodeId(NODE_ID)
        .masterNodeId(NODE_ID)
        .build();
    ClusterState clusterState = ClusterState.builder(new ClusterName(this.getClass().getSimpleName()))
        .nodes(nodes).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build();

    ClusterApplierService clusterApplierService = clusterService.getClusterApplierService();
    clusterApplierService.setInitialState(clusterState);

    MasterService masterService = clusterService.getMasterService();
    masterService.setClusterStatePublisher(createClusterStatePublisher(clusterApplierService));
    masterService.setClusterStateSupplier(clusterApplierService::state);

    clusterService.start();
    return clusterService;
}
 
Example #10
Source File: UserPrivilegesTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testMatchPrivilegeComplexSetIncludingDeny() throws Exception {
    Collection<Privilege> privileges = Sets.newHashSet(
        new Privilege(Privilege.State.GRANT, Privilege.Type.DQL, Privilege.Clazz.CLUSTER, null, "crate"),
        new Privilege(Privilege.State.DENY, Privilege.Type.DQL, Privilege.Clazz.SCHEMA, "doc", "crate"),
        new Privilege(Privilege.State.GRANT, Privilege.Type.DQL, Privilege.Clazz.TABLE, "doc.t1", "crate")
    );
    UserPrivileges userPrivileges = new UserPrivileges(privileges);
    assertThat(userPrivileges.matchPrivilege(Privilege.Type.DQL, Privilege.Clazz.TABLE, "doc.t1", "doc"), is(true));
    assertThat(userPrivileges.matchPrivilege(Privilege.Type.DQL, Privilege.Clazz.TABLE, "doc.t2", "doc"), is(false));
    assertThat(userPrivileges.matchPrivilege(Privilege.Type.DQL, Privilege.Clazz.SCHEMA, "my_schema", "doc"), is(true));
}
 
Example #11
Source File: UserPrivilegesTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testMatchPrivilegeDenyResultsInNoMatch() throws Exception {
    Collection<Privilege> privileges = Sets.newHashSet(
        new Privilege(Privilege.State.DENY, Privilege.Type.DQL, Privilege.Clazz.CLUSTER, null, "crate")
    );
    UserPrivileges userPrivileges = new UserPrivileges(privileges);
    assertThat(userPrivileges.matchPrivilege(Privilege.Type.DQL, Privilege.Clazz.CLUSTER, null, "doc"), is(false));
    assertThat(userPrivileges.matchPrivilege(Privilege.Type.DQL, Privilege.Clazz.SCHEMA, "doc", "doc"), is(false));
    assertThat(userPrivileges.matchPrivilege(Privilege.Type.DQL, Privilege.Clazz.TABLE, "doc.t1", "doc"), is(false));
    assertThat(userPrivileges.matchPrivilegeOfAnyType(Privilege.Clazz.CLUSTER, null), is(false));
    assertThat(userPrivileges.matchPrivilegeOfAnyType(Privilege.Clazz.SCHEMA, "doc"), is(false));
    assertThat(userPrivileges.matchPrivilegeOfAnyType(Privilege.Clazz.TABLE, "doc.t1"), is(false));
}
 
Example #12
Source File: CoordinationStateTests.java    From crate with Apache License 2.0 5 votes vote down vote up
public void testVoteCollection() {
    final CoordinationState.VoteCollection voteCollection = new CoordinationState.VoteCollection();
    assertTrue(voteCollection.isEmpty());
    voteCollection.addVote(node1);
    assertFalse(voteCollection.isEmpty());
    assertTrue(voteCollection.containsVoteFor(node1));
    assertFalse(voteCollection.containsVoteFor(node2));
    assertFalse(voteCollection.containsVoteFor(node3));
    voteCollection.addVote(node2);
    assertTrue(voteCollection.containsVoteFor(node1));
    assertTrue(voteCollection.containsVoteFor(node2));
    assertFalse(voteCollection.containsVoteFor(node3));
    assertTrue(voteCollection.isQuorum(new VotingConfiguration(Sets.newHashSet(node1.getId(), node2.getId()))));
    assertTrue(voteCollection.isQuorum(new VotingConfiguration(Sets.newHashSet(node1.getId()))));
    assertFalse(voteCollection.isQuorum(new VotingConfiguration(Sets.newHashSet(node3.getId()))));

    EqualsHashCodeTestUtils.CopyFunction<CoordinationState.VoteCollection> copyFunction =
        vc -> {
            CoordinationState.VoteCollection voteCollection1 = new CoordinationState.VoteCollection();
            for (DiscoveryNode node : vc.nodes()) {
                voteCollection1.addVote(node);
            }
            return voteCollection1;
        };
    EqualsHashCodeTestUtils.checkEqualsAndHashCode(voteCollection, copyFunction,
        vc -> {
            CoordinationState.VoteCollection copy = copyFunction.copy(vc);
            copy.addVote(createNode(randomAlphaOfLength(10)));
            return copy;
        });
}
 
Example #13
Source File: PublicationTests.java    From crate with Apache License 2.0 5 votes vote down vote up
public void testPublishingToMastersFirst() {
    VotingConfiguration singleNodeConfig = new VotingConfiguration(Sets.newHashSet(n1.getId()));
    initializeCluster(singleNodeConfig);

    DiscoveryNodes.Builder discoNodesBuilder = DiscoveryNodes.builder();
    randomNodes(10).forEach(dn -> discoNodesBuilder.add(dn));
    DiscoveryNodes discoveryNodes = discoNodesBuilder.add(n1).localNodeId(n1.getId()).build();
    MockPublication publication = node1.publish(CoordinationStateTests.clusterState(1L, 2L,
        discoveryNodes, singleNodeConfig, singleNodeConfig, 42L), null, Collections.emptySet());

    List<DiscoveryNode> publicationTargets = new ArrayList<>(publication.pendingPublications.keySet());
    List<DiscoveryNode> sortedPublicationTargets = new ArrayList<>(publicationTargets);
    Collections.sort(sortedPublicationTargets, Comparator.comparing(n -> n.isMasterNode() == false));
    assertEquals(sortedPublicationTargets, publicationTargets);
}
 
Example #14
Source File: ElasticSearchSqlServiceImpl.java    From elasticsearch-sql with Apache License 2.0 5 votes vote down vote up
@Override
public SearchResultDTO search(String sql) {
    SearchResultDTO resultDTO = new SearchResultDTO();
    try {
        long before = System.currentTimeMillis();
        SearchDao searchDao = new SearchDao(transportClient);
        QueryAction queryAction = searchDao.explain(sql);
        Object execution = QueryActionElasticExecutor.executeAnyAction(searchDao.getClient(), queryAction);
        ObjectResult result = getObjectResult(execution, true, false, false, true);
        resultDTO.setResultColumns(Sets.newHashSet(result.getHeaders()));
        List<IndexRowData> indexRowDatas = new ArrayList<>();
        for (List<Object> line : result.getLines()) {
            IndexRowData indexRowData = new IndexRowData();
            for (int i = 0; i < result.getHeaders().size(); i++) {
                indexRowData.build(result.getHeaders().get(i), line.get(i));
            }
            indexRowDatas.add(indexRowData);
        }
        resultDTO.setResultSize(indexRowDatas.size());
        if (execution instanceof SearchHits) {
            resultDTO.setTotal(((SearchHits) execution).getTotalHits());
        } else {
            resultDTO.setTotal(indexRowDatas.size());
        }
        resultDTO.setResult(indexRowDatas);
        resultDTO.setTime((System.currentTimeMillis() - before) / 1000);
        logger.info("查询数据结果集: {}", JSONObject.toJSONString(resultDTO));
    } catch (Exception e) {
        logger.error("根据ES-SQL查询数据异常: {}", e.getMessage());
        throw new ElasticsearchException(e.getMessage());
    }
    return resultDTO;
}
 
Example #15
Source File: CoordinatorTests.java    From crate with Apache License 2.0 5 votes vote down vote up
public void testCannotSetInitialConfigurationWithoutLocalNode() {
    final Cluster cluster = new Cluster(randomIntBetween(1, 5));
    final Coordinator coordinator = cluster.getAnyNode().coordinator;
    final VotingConfiguration unknownNodeConfiguration = new VotingConfiguration(Sets.newHashSet("unknown-node"));
    final String exceptionMessage = expectThrows(CoordinationStateRejectedException.class,
        () -> coordinator.setInitialConfiguration(unknownNodeConfiguration)).getMessage();
    assertThat(exceptionMessage,
        equalTo("local node is not part of initial configuration"));
}
 
Example #16
Source File: AliasMetaData.java    From crate with Apache License 2.0 5 votes vote down vote up
private AliasMetaData(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex) {
    this.alias = alias;
    this.filter = filter;
    this.indexRouting = indexRouting;
    this.searchRouting = searchRouting;
    if (searchRouting != null) {
        searchRoutingValues = Collections.unmodifiableSet(Sets.newHashSet(Strings.splitStringByCommaToArray(searchRouting)));
    } else {
        searchRoutingValues = emptySet();
    }
    this.writeIndex = writeIndex;
}
 
Example #17
Source File: PolygonBuilder.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Validates only 1 vertex is tangential (shared) between the interior and exterior of a polygon
 */
protected void validateHole(LineStringBuilder shell, LineStringBuilder hole) {
    HashSet<Coordinate> exterior = Sets.newHashSet(shell.coordinates);
    HashSet<Coordinate> interior = Sets.newHashSet(hole.coordinates);
    exterior.retainAll(interior);
    if (exterior.size() >= 2) {
        throw new InvalidShapeException("Invalid polygon, interior cannot share more than one point with the exterior");
    }
}
 
Example #18
Source File: JoinPlanBuilder.java    From crate with Apache License 2.0 5 votes vote down vote up
@Nullable
private static <V> V removeMatch(Map<Set<RelationName>, V> valuesByNames, Set<RelationName> names, RelationName nextName) {
    for (RelationName name : names) {
        V v = valuesByNames.remove(Sets.newHashSet(name, nextName));
        if (v != null) {
            return v;
        }
    }
    return null;
}
 
Example #19
Source File: JoinPlanBuilder.java    From crate with Apache License 2.0 5 votes vote down vote up
private static Symbol removeParts(Map<Set<RelationName>, Symbol> queryParts, RelationName lhsName, RelationName rhsName) {
    // query parts can affect a single relation without being pushed down in the outer-join case
    Symbol left = queryParts.remove(Collections.singleton(lhsName));
    Symbol right = queryParts.remove(Collections.singleton(rhsName));
    Symbol both = queryParts.remove(Sets.newHashSet(lhsName, rhsName));
    return AndOperator.join(
        Stream.of(left, right, both).filter(Objects::nonNull).iterator()
    );
}
 
Example #20
Source File: ExtractedEntities.java    From elasticsearch-ingest-opennlp with Apache License 2.0 4 votes vote down vote up
ExtractedEntities(String[] tokens, Span[] spans) {
    this.tokens = tokens;
    this.spans = spans;
    this.entities = Sets.newHashSet(Span.spansToStrings(spans, tokens));
}
 
Example #21
Source File: CoordinationMetaData.java    From crate with Apache License 2.0 4 votes vote down vote up
public VotingConfiguration(StreamInput in) throws IOException {
    nodeIds = Collections.unmodifiableSet(Sets.newHashSet(in.readStringArray()));
}
 
Example #22
Source File: DecommissioningServiceTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
protected Set<Setting<?>> additionalClusterSettings() {
    SQLPlugin sqlPlugin = new SQLPlugin(Settings.EMPTY);
    return Sets.newHashSet(sqlPlugin.getSettings());
}
 
Example #23
Source File: DecommissionAllocationDeciderTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
protected Set<Setting<?>> additionalClusterSettings() {
    SQLPlugin sqlPlugin = new SQLPlugin(Settings.EMPTY);
    return Sets.newHashSet(sqlPlugin.getSettings());
}
 
Example #24
Source File: PrimaryAllocationIT.java    From crate with Apache License 2.0 4 votes vote down vote up
private Settings createStaleReplicaScenario(String master, String schema, String indexName) throws Exception {
    execute("insert into t values ('value1')");
    refresh();

    ClusterState state = client().admin().cluster().prepareState().all().get().getState();
    List<ShardRouting> shards = state.routingTable().allShards(indexName);
    assertThat(shards.size(), equalTo(2));

    final String primaryNode;
    final String replicaNode;
    if (shards.get(0).primary()) {
        primaryNode = state.getRoutingNodes().node(shards.get(0).currentNodeId()).node().getName();
        replicaNode = state.getRoutingNodes().node(shards.get(1).currentNodeId()).node().getName();
    } else {
        primaryNode = state.getRoutingNodes().node(shards.get(1).currentNodeId()).node().getName();
        replicaNode = state.getRoutingNodes().node(shards.get(0).currentNodeId()).node().getName();
    }

    NetworkDisruption partition = new NetworkDisruption(
        new TwoPartitions(Sets.newHashSet(master, replicaNode), Collections.singleton(primaryNode)),
        new NetworkDisconnect());
    internalCluster().setDisruptionScheme(partition);
    logger.info("--> partitioning node with primary shard from rest of cluster");
    partition.startDisrupting();

    ensureStableCluster(2, master);

    logger.info("--> index a document into previous replica shard (that is now primary)");
    systemExecute("insert into t values ('value2')", schema, replicaNode);

    logger.info("--> shut down node that has new acknowledged document");
    final Settings inSyncDataPathSettings = internalCluster().dataPathSettings(replicaNode);
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));

    ensureStableCluster(1, master);

    partition.stopDisrupting();

    logger.info("--> waiting for node with old primary shard to rejoin the cluster");
    ensureStableCluster(2, master);

    logger.info("--> check that old primary shard does not get promoted to primary again");
    // kick reroute and wait for all shard states to be fetched
    client(master).admin().cluster().prepareReroute().get();
    assertBusy(() -> assertThat(internalCluster().getInstance(GatewayAllocator.class, master).getNumberOfInFlightFetch(),
        equalTo(0)));
    // kick reroute a second time and check that all shards are unassigned
    assertThat(client(master).admin().cluster().prepareReroute().get().getState().getRoutingNodes().unassigned().size(),
        equalTo(2));
    return inSyncDataPathSettings;
}
 
Example #25
Source File: JobsLogsTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
protected Set<Setting<?>> additionalClusterSettings() {
    SQLPlugin sqlPlugin = new SQLPlugin(Settings.EMPTY);
    return Sets.newHashSet(sqlPlugin.getSettings());
}
 
Example #26
Source File: InternalTestCluster.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Returns a predicate that only accepts settings of nodes with one of the given names.
 */
public static Predicate<Settings> nameFilter(String... nodeNames) {
    final Set<String> nodes = Sets.newHashSet(nodeNames);
    return settings -> nodes.contains(settings.get("node.name"));
}
 
Example #27
Source File: CoordinationMetaDataTests.java    From crate with Apache License 2.0 4 votes vote down vote up
private static VotingConfiguration randomVotingConfig() {
    return new VotingConfiguration(Sets.newHashSet(generateRandomStringArray(randomInt(10), 20, false)));
}
 
Example #28
Source File: CoordinationMetaDataTests.java    From crate with Apache License 2.0 4 votes vote down vote up
public void testVotingConfiguration() {
    VotingConfiguration config0 = new VotingConfiguration(Sets.newHashSet());
    assertThat(config0, equalTo(VotingConfiguration.EMPTY_CONFIG));
    assertThat(config0.getNodeIds(), equalTo(Sets.newHashSet()));
    assertThat(config0.isEmpty(), equalTo(true));
    assertThat(config0.hasQuorum(Sets.newHashSet()), equalTo(false));
    assertThat(config0.hasQuorum(Sets.newHashSet("id1")), equalTo(false));

    VotingConfiguration config1 = new VotingConfiguration(Sets.newHashSet("id1"));
    assertThat(config1.getNodeIds(), equalTo(Sets.newHashSet("id1")));
    assertThat(config1.isEmpty(), equalTo(false));
    assertThat(config1.hasQuorum(Sets.newHashSet("id1")), equalTo(true));
    assertThat(config1.hasQuorum(Sets.newHashSet("id1", "id2")), equalTo(true));
    assertThat(config1.hasQuorum(Sets.newHashSet("id2")), equalTo(false));
    assertThat(config1.hasQuorum(Sets.newHashSet()), equalTo(false));

    VotingConfiguration config2 = new VotingConfiguration(Sets.newHashSet("id1", "id2"));
    assertThat(config2.getNodeIds(), equalTo(Sets.newHashSet("id1", "id2")));
    assertThat(config2.isEmpty(), equalTo(false));
    assertThat(config2.hasQuorum(Sets.newHashSet("id1", "id2")), equalTo(true));
    assertThat(config2.hasQuorum(Sets.newHashSet("id1", "id2", "id3")), equalTo(true));
    assertThat(config2.hasQuorum(Sets.newHashSet("id1")), equalTo(false));
    assertThat(config2.hasQuorum(Sets.newHashSet("id2")), equalTo(false));
    assertThat(config2.hasQuorum(Sets.newHashSet("id3")), equalTo(false));
    assertThat(config2.hasQuorum(Sets.newHashSet("id1", "id3")), equalTo(false));
    assertThat(config2.hasQuorum(Sets.newHashSet()), equalTo(false));

    VotingConfiguration config3 = new VotingConfiguration(Sets.newHashSet("id1", "id2", "id3"));
    assertThat(config3.getNodeIds(), equalTo(Sets.newHashSet("id1", "id2", "id3")));
    assertThat(config3.isEmpty(), equalTo(false));
    assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id2")), equalTo(true));
    assertThat(config3.hasQuorum(Sets.newHashSet("id2", "id3")), equalTo(true));
    assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id3")), equalTo(true));
    assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id2", "id3")), equalTo(true));
    assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id2", "id4")), equalTo(true));
    assertThat(config3.hasQuorum(Sets.newHashSet("id1")), equalTo(false));
    assertThat(config3.hasQuorum(Sets.newHashSet("id2")), equalTo(false));
    assertThat(config3.hasQuorum(Sets.newHashSet("id3")), equalTo(false));
    assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id4")), equalTo(false));
    assertThat(config3.hasQuorum(Sets.newHashSet("id1", "id4", "id5")), equalTo(false));
    assertThat(config3.hasQuorum(Sets.newHashSet()), equalTo(false));
}
 
Example #29
Source File: Validators.java    From crate with Apache License 2.0 4 votes vote down vote up
public static Setting.Validator<String> stringValidator(String key, String... allowedValues) {
    if (allowedValues.length == 0) {
        return new StrictStringValidator(key);
    }
    return new StringValidatorAllowedValuesOnly(key, Sets.newHashSet(allowedValues));
}
 
Example #30
Source File: PublicationTests.java    From crate with Apache License 2.0 4 votes vote down vote up
public void testClusterStatePublishingTimesOutAfterCommit() throws InterruptedException {
    VotingConfiguration config = new VotingConfiguration(randomBoolean() ?
        Sets.newHashSet(n1.getId(), n2.getId()) : Sets.newHashSet(n1.getId(), n2.getId(), n3.getId()));
    initializeCluster(config);

    AssertingAckListener ackListener = new AssertingAckListener(nodes.size());
    DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(n1).add(n2).add(n3).localNodeId(n1.getId()).build();
    MockPublication publication = node1.publish(CoordinationStateTests.clusterState(1L, 2L,
        discoveryNodes, config, config, 42L), ackListener, Collections.emptySet());

    boolean publishedToN3 = randomBoolean();
    publication.pendingPublications.entrySet().stream().collect(shuffle()).forEach(e -> {
        if (e.getKey().equals(n3) == false || publishedToN3) {
            PublishResponse publishResponse = nodeResolver.apply(e.getKey()).coordinationState.handlePublishRequest(
                publication.publishRequest);
            e.getValue().onResponse(new PublishWithJoinResponse(publishResponse, Optional.empty()));
        }
    });

    assertNotNull(publication.applyCommit);

    Set<DiscoveryNode> committingNodes = new HashSet<>(randomSubsetOf(discoNodes));
    if (publishedToN3 == false) {
        committingNodes.remove(n3);
    }

    logger.info("Committing nodes: {}", committingNodes);

    publication.pendingCommits.entrySet().stream().collect(shuffle()).forEach(e -> {
        if (committingNodes.contains(e.getKey())) {
            nodeResolver.apply(e.getKey()).coordinationState.handleCommit(publication.applyCommit);
            e.getValue().onResponse(TransportResponse.Empty.INSTANCE);
        }
    });

    publication.cancel("timed out");
    assertTrue(publication.completed);
    assertTrue(publication.committed);
    assertEquals(committingNodes, ackListener.await(0L, TimeUnit.SECONDS));

    // check that acking still works after publication completed
    if (publishedToN3 == false) {
        publication.pendingPublications.get(n3).onResponse(
            new PublishWithJoinResponse(node3.coordinationState.handlePublishRequest(publication.publishRequest), Optional.empty()));
    }

    assertEquals(discoNodes, publication.pendingCommits.keySet());

    Set<DiscoveryNode> nonCommittedNodes = Sets.difference(discoNodes, committingNodes);
    logger.info("Non-committed nodes: {}", nonCommittedNodes);
    nonCommittedNodes.stream().collect(shuffle()).forEach(n ->
        publication.pendingCommits.get(n).onResponse(TransportResponse.Empty.INSTANCE));

    assertEquals(discoNodes, ackListener.await(0L, TimeUnit.SECONDS));
}