org.apache.cassandra.gms.Gossiper Java Examples

@Test
public void testRac() throws IOException, ConfigurationException
{
    az = "us-central1-a";
    GoogleCloudSnitch snitch = new TestGoogleCloudSnitch();
    InetAddress local = InetAddress.getByName("127.0.0.1");
    InetAddress nonlocal = InetAddress.getByName("127.0.0.7");

    Gossiper.instance.addSavedEndpoint(nonlocal);
    Map<ApplicationState,VersionedValue> stateMap = Gossiper.instance.getEndpointStateForEndpoint(nonlocal).getApplicationStateMap();
    stateMap.put(ApplicationState.DC, StorageService.instance.valueFactory.datacenter("europe-west1"));
    stateMap.put(ApplicationState.RACK, StorageService.instance.valueFactory.datacenter("a"));

    assertEquals("europe-west1", snitch.getDatacenter(nonlocal));
    assertEquals("a", snitch.getRack(nonlocal));

    assertEquals("us-central1", snitch.getDatacenter(local));
    assertEquals("a", snitch.getRack(local));
}
 

public void run()
{
    double report = -1;
    try
    {
        report = getIOWait();
    }
    catch (IOException e)
    {
        // ignore;
        if (FBUtilities.hasProcFS())
            logger.warn("Couldn't read /proc/stats");
    }
    if (report == -1d)
        report = compaction_severity.get();

    if (!Gossiper.instance.isEnabled())
        return;
    report += manual_severity.get(); // add manual severity setting.
    VersionedValue updated = StorageService.instance.valueFactory.severity(report);
    Gossiper.instance.addLocalApplicationState(ApplicationState.SEVERITY, updated);
}
 

@Test
public void testCommitFailurePolicy_stop() throws ConfigurationException
{
    // Need storage service active so stop policy can shutdown gossip
    StorageService.instance.initServer();
    Assert.assertTrue(Gossiper.instance.isEnabled());

    Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
    try
    {
        DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.stop);
        CommitLog.handleCommitError("Test stop error", new Throwable());
        Assert.assertFalse(Gossiper.instance.isEnabled());
    }
    finally
    {
        DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
    }
}
 

private void bootstrap(Collection<Token> tokens)
{
    isBootstrapMode = true;
    SystemKeyspace.updateTokens(tokens); // DON'T use setToken, that makes us part of the ring locally which is incorrect until we are done bootstrapping
    if (!DatabaseDescriptor.isReplacing())
    {
        // if not an existing token then bootstrap
        List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<Pair<ApplicationState, VersionedValue>>();
        states.add(Pair.create(ApplicationState.TOKENS, valueFactory.tokens(tokens)));
        states.add(Pair.create(ApplicationState.STATUS, valueFactory.bootstrapping(tokens)));
        Gossiper.instance.addLocalApplicationStates(states);
        setMode(Mode.JOINING, "sleeping " + RING_DELAY + " ms for pending range setup", true);
        Uninterruptibles.sleepUninterruptibly(RING_DELAY, TimeUnit.MILLISECONDS);
    }
    else
    {
        // Dont set any state for the node which is bootstrapping the existing token...
        tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddress());
        SystemKeyspace.removeEndpoint(DatabaseDescriptor.getReplaceAddress());
    }
    if (!Gossiper.instance.seenAnySeed())
        throw new IllegalStateException("Unable to contact any seeds!");
    setMode(Mode.JOINING, "Starting to bootstrap...", true);
    new BootStrapper(FBUtilities.getBroadcastAddress(), tokens, tokenMetadata).bootstrap(); // handles token update
    logger.info("Bootstrap completed! for the tokens {}", tokens);
}
 

/**
 * Force a remove operation to complete. This may be necessary if a remove operation
 * blocks forever due to node/stream failure. removeToken() must be called
 * first, this is a last resort measure.  No further attempt will be made to restore replicas.
 */
public void forceRemoveCompletion()
{
    if (!replicatingNodes.isEmpty()  || !tokenMetadata.getLeavingEndpoints().isEmpty())
    {
        logger.warn("Removal not confirmed for for {}", StringUtils.join(this.replicatingNodes, ","));
        for (InetAddress endpoint : tokenMetadata.getLeavingEndpoints())
        {
            UUID hostId = tokenMetadata.getHostId(endpoint);
            Gossiper.instance.advertiseTokenRemoved(endpoint, hostId);
            excise(tokenMetadata.getTokens(endpoint), endpoint);
        }
        replicatingNodes.clear();
        removingNode = null;
    }
    else
    {
        logger.warn("No tokens to force removal on, call 'removenode' first");
    }
}
 

/**
 * be careful about just blindly updating ApplicationState.INTERNAL_IP everytime we read the yaml file,
 * as that can cause connections to get unnecessarily reset (via IESCS.onChange()).
 */
private void maybeSetApplicationState()
{
    if (localNodeData.dcLocalAddress == null)
        return;
    final EndpointState es = Gossiper.instance.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddress());
    if (es == null)
        return;
    final VersionedValue vv = es.getApplicationState(ApplicationState.INTERNAL_IP);
    if ((vv != null && !vv.value.equals(localNodeData.dcLocalAddress.getHostAddress()))
        || vv == null)
    {
        Gossiper.instance.addLocalApplicationState(ApplicationState.INTERNAL_IP,
            StorageService.instance.valueFactory.internalIP(localNodeData.dcLocalAddress.getHostAddress()));
    }
}
 

public static boolean shouldHint(InetAddress ep)
{
    if (DatabaseDescriptor.shouldHintByDC())
    {
        final String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(ep);
        //Disable DC specific hints
        if(!DatabaseDescriptor.hintedHandoffEnabled(dc))
        {
            HintedHandOffManager.instance.metrics.incrPastWindow(ep);
            return false;
        }
    }
    else if (!DatabaseDescriptor.hintedHandoffEnabled())
    {
        HintedHandOffManager.instance.metrics.incrPastWindow(ep);
        return false;
    }

    boolean hintWindowExpired = Gossiper.instance.getEndpointDowntime(ep) > DatabaseDescriptor.getMaxHintWindow();
    if (hintWindowExpired)
    {
        HintedHandOffManager.instance.metrics.incrPastWindow(ep);
        Tracing.trace("Not hinting {} which has been down {}ms", ep, Gossiper.instance.getEndpointDowntime(ep));
    }
    return !hintWindowExpired;
}
 

public synchronized void initClient(int ringDelay) throws ConfigurationException
{
    if (initialized)
    {
        if (!isClientMode)
            throw new UnsupportedOperationException("StorageService does not support switching modes.");
        return;
    }
    initialized = true;
    isClientMode = true;
    logger.info("Starting up client gossip");
    setMode(Mode.CLIENT, false);
    Gossiper.instance.register(this);
    Gossiper.instance.start((int) (System.currentTimeMillis() / 1000)); // needed for node-ring gathering.
    Gossiper.instance.addLocalApplicationState(ApplicationState.NET_VERSION, valueFactory.networkVersion());

    if (!MessagingService.instance().isListening())
        MessagingService.instance().listen(FBUtilities.getLocalAddress());
    Uninterruptibles.sleepUninterruptibly(ringDelay, TimeUnit.MILLISECONDS);
}
 

/**
 * Return the rack for which an endpoint resides in
 *
 * @param endpoint the endpoint to process
 * @return string of rack
 */
public String getRack(InetAddress endpoint)
{
    if (endpoint.equals(FBUtilities.getBroadcastAddress()))
        return myRack;

    EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
    if (epState == null || epState.getApplicationState(ApplicationState.RACK) == null)
    {
        if (psnitch == null)
        {
            if (savedEndpoints == null)
                savedEndpoints = SystemKeyspace.loadDcRackInfo();
            if (savedEndpoints.containsKey(endpoint))
                return savedEndpoints.get(endpoint).get("rack");
            return DEFAULT_RACK;
        }
        else
            return psnitch.getRack(endpoint);
    }
    return epState.getApplicationState(ApplicationState.RACK).value;
}
 

/**
 * Return the data center for which an endpoint resides in
 *
 * @param endpoint the endpoint to process
 * @return string of data center
 */
public String getDatacenter(InetAddress endpoint)
{
    if (endpoint.equals(FBUtilities.getBroadcastAddress()))
        return myDC;

    EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
    if (epState == null || epState.getApplicationState(ApplicationState.DC) == null)
    {
        if (psnitch == null)
        {
            if (savedEndpoints == null)
                savedEndpoints = SystemKeyspace.loadDcRackInfo();
            if (savedEndpoints.containsKey(endpoint))
                return savedEndpoints.get(endpoint).get("data_center");
            return DEFAULT_DC;
        }
        else
            return psnitch.getDatacenter(endpoint);
    }
    return epState.getApplicationState(ApplicationState.DC).value;
}
 

@Test
public void testRac() throws IOException, ConfigurationException
{
    az = "us-east-1d";
    Ec2Snitch snitch = new TestEC2Snitch();
    InetAddress local = InetAddress.getByName("127.0.0.1");
    InetAddress nonlocal = InetAddress.getByName("127.0.0.7");

    Gossiper.instance.addSavedEndpoint(nonlocal);
    Map<ApplicationState,VersionedValue> stateMap = Gossiper.instance.getEndpointStateForEndpoint(nonlocal).getApplicationStateMap();
    stateMap.put(ApplicationState.DC, StorageService.instance.valueFactory.datacenter("us-west"));
    stateMap.put(ApplicationState.RACK, StorageService.instance.valueFactory.datacenter("1a"));

    assertEquals("us-west", snitch.getDatacenter(nonlocal));
    assertEquals("1a", snitch.getRack(nonlocal));

    assertEquals("us-east", snitch.getDatacenter(local));
    assertEquals("1d", snitch.getRack(local));
}
 

public synchronized void initClient() throws ConfigurationException
{
    // We don't wait, because we're going to actually try to work on
    initClient(0);

    // sleep a while to allow gossip to warm up (the other nodes need to know about this one before they can reply).
    outer:
    while (true)
    {
        Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
        for (InetAddress address : Gossiper.instance.getLiveMembers())
        {
            if (!Gossiper.instance.isFatClient(address))
                break outer;
        }
    }

    // sleep until any schema migrations have finished
    while (!MigrationManager.isReadyForBootstrap())
    {
        Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
    }
}
 

@Test
public void testRacks() throws IOException, ConfigurationException
{
    az = "ch-gva-1";
    CloudstackSnitch snitch = new TestCloudstackSnitch();
    InetAddress local = InetAddress.getByName("127.0.0.1");
    InetAddress nonlocal = InetAddress.getByName("127.0.0.7");

    Gossiper.instance.addSavedEndpoint(nonlocal);
    Map<ApplicationState,VersionedValue> stateMap = Gossiper.instance.getEndpointStateForEndpoint(nonlocal).getApplicationStateMap();
    stateMap.put(ApplicationState.DC, StorageService.instance.valueFactory.datacenter("ch-zrh"));
    stateMap.put(ApplicationState.RACK, StorageService.instance.valueFactory.rack("2"));

    assertEquals("ch-zrh", snitch.getDatacenter(nonlocal));
    assertEquals("2", snitch.getRack(nonlocal));

    assertEquals("ch-gva", snitch.getDatacenter(local));
    assertEquals("1", snitch.getRack(local));

}
 

public String getRack(InetAddress endpoint)
{
    if (endpoint.equals(FBUtilities.getBroadcastAddress()))
        return csZoneRack;
    EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
    if (state == null || state.getApplicationState(ApplicationState.RACK) == null) 
    {
        if (savedEndpoints == null)
            savedEndpoints = SystemKeyspace.loadDcRackInfo();
        if (savedEndpoints.containsKey(endpoint))
            return savedEndpoints.get(endpoint).get("rack");
        return DEFAULT_RACK;
    }
    return state.getApplicationState(ApplicationState.RACK).value;
}
 

@Test
public void testStateJumpToLeft() throws UnknownHostException
{
    StorageService ss = StorageService.instance;
    TokenMetadata tmd = ss.getTokenMetadata();
    tmd.clearUnsafe();
    IPartitioner partitioner = new RandomPartitioner();
    VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);

    ArrayList<Token> endpointTokens = new ArrayList<Token>();
    ArrayList<Token> keyTokens = new ArrayList<Token>();
    List<InetAddress> hosts = new ArrayList<InetAddress>();
    List<UUID> hostIds = new ArrayList<UUID>();

    // create a ring of 6 nodes
    Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 7);

    // node hosts.get(2) goes jumps to left
    ss.onChange(hosts.get(2), ApplicationState.STATUS,
            valueFactory.left(Collections.singleton(endpointTokens.get(2)), Gossiper.computeExpireTime()));

    assertFalse(tmd.isMember(hosts.get(2)));

    // node hosts.get(4) goes to bootstrap
    Gossiper.instance.injectApplicationState(hosts.get(3), ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(1))));
    ss.onChange(hosts.get(3), ApplicationState.STATUS, valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(1))));

    assertFalse(tmd.isMember(hosts.get(3)));
    assertEquals(1, tmd.getBootstrapTokens().size());
    assertEquals(hosts.get(3), tmd.getBootstrapTokens().get(keyTokens.get(1)));

    // and then directly to 'left'
    Gossiper.instance.injectApplicationState(hosts.get(2), ApplicationState.TOKENS, valueFactory.tokens(Collections.singleton(keyTokens.get(1))));
    ss.onChange(hosts.get(2), ApplicationState.STATUS,
            valueFactory.left(Collections.singleton(keyTokens.get(1)), Gossiper.computeExpireTime()));

    assertTrue(tmd.getBootstrapTokens().size() == 0);
    assertFalse(tmd.isMember(hosts.get(2)));
    assertFalse(tmd.isLeaving(hosts.get(2)));
}
 

public void gossipSnitchInfo()
{
    IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();
    String dc = snitch.getDatacenter(FBUtilities.getBroadcastAddress());
    String rack = snitch.getRack(FBUtilities.getBroadcastAddress());
    Gossiper.instance.addLocalApplicationState(ApplicationState.DC, StorageService.instance.valueFactory.datacenter(dc));
    Gossiper.instance.addLocalApplicationState(ApplicationState.RACK, StorageService.instance.valueFactory.rack(rack));
}
 

/**
 * Return the rpc address associated with an endpoint as a string.
 * @param endpoint The endpoint to get rpc address for
 * @return the rpc address
 */
public String getRpcaddress(InetAddress endpoint)
{
    if (endpoint.equals(FBUtilities.getBroadcastAddress()))
        return DatabaseDescriptor.getBroadcastRpcAddress().getHostAddress();
    else if (Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS) == null)
        return endpoint.getHostAddress();
    else
        return Gossiper.instance.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.RPC_ADDRESS).value;
}
 

private void updatePeerInfo(InetAddress endpoint)
{
    EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
    for (Map.Entry<ApplicationState, VersionedValue> entry : epState.getApplicationStateMap().entrySet())
    {
        switch (entry.getKey())
        {
            case RELEASE_VERSION:
                SystemKeyspace.updatePeerInfo(endpoint, "release_version", entry.getValue().value);
                break;
            case DC:
                SystemKeyspace.updatePeerInfo(endpoint, "data_center", entry.getValue().value);
                break;
            case RACK:
                SystemKeyspace.updatePeerInfo(endpoint, "rack", entry.getValue().value);
                break;
            case RPC_ADDRESS:
                try
                {
                    SystemKeyspace.updatePeerInfo(endpoint, "rpc_address", InetAddress.getByName(entry.getValue().value));
                }
                catch (UnknownHostException e)
                {
                    throw new RuntimeException(e);
                }
                break;
            case SCHEMA:
                SystemKeyspace.updatePeerInfo(endpoint, "schema_version", UUID.fromString(entry.getValue().value));
                break;
            case HOST_ID:
                SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(entry.getValue().value));
                break;
        }
    }
}
 

public synchronized void checkForEndpointCollision() throws ConfigurationException
{
    logger.debug("Starting shadow gossip round to check for endpoint collision");
    if (!MessagingService.instance().isListening())
        MessagingService.instance().listen(FBUtilities.getLocalAddress());
    Gossiper.instance.doShadowRound();
    EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddress());
    if (epState != null && !Gossiper.instance.isDeadState(epState) && !Gossiper.instance.isFatClient(FBUtilities.getBroadcastAddress()))
    {
        throw new RuntimeException(String.format("A node with address %s already exists, cancelling join. " +
                                                 "Use cassandra.replace_address if you want to replace this node.",
                                                 FBUtilities.getBroadcastAddress()));
    }
    if (RangeStreamer.useStrictConsistency)
    {
        for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.getEndpointStates())
        {

            if (entry.getValue().getApplicationState(ApplicationState.STATUS) == null)
                    continue;
            String[] pieces = entry.getValue().getApplicationState(ApplicationState.STATUS).value.split(VersionedValue.DELIMITER_STR, -1);
            assert (pieces.length > 0);
            String state = pieces[0];
            if (state.equals(VersionedValue.STATUS_BOOTSTRAPPING) || state.equals(VersionedValue.STATUS_LEAVING) || state.equals(VersionedValue.STATUS_MOVING))
                throw new UnsupportedOperationException("Other bootstrapping/leaving/moving nodes detected, cannot bootstrap while cassandra.consistent.rangemovement is true");
        }
    }
    Gossiper.instance.resetEndpointStateMap();
}
 

/**
 * Handle node bootstrap
 *
 * @param endpoint bootstrapping node
 */
private void handleStateBootstrap(InetAddress endpoint)
{
    Collection<Token> tokens;
    // explicitly check for TOKENS, because a bootstrapping node might be bootstrapping in legacy mode; that is, not using vnodes and no token specified
    tokens = getTokensFor(endpoint);

    if (logger.isDebugEnabled())
        logger.debug("Node {} state bootstrapping, token {}", endpoint, tokens);

    // if this node is present in token metadata, either we have missed intermediate states
    // or the node had crashed. Print warning if needed, clear obsolete stuff and
    // continue.
    if (tokenMetadata.isMember(endpoint))
    {
        // If isLeaving is false, we have missed both LEAVING and LEFT. However, if
        // isLeaving is true, we have only missed LEFT. Waiting time between completing
        // leave operation and rebootstrapping is relatively short, so the latter is quite
        // common (not enough time for gossip to spread). Therefore we report only the
        // former in the log.
        if (!tokenMetadata.isLeaving(endpoint))
            logger.info("Node {} state jump to bootstrap", endpoint);
        tokenMetadata.removeEndpoint(endpoint);
    }

    tokenMetadata.addBootstrapTokens(tokens, endpoint);
    PendingRangeCalculatorService.instance.update();

    if (Gossiper.instance.usesHostId(endpoint))
        tokenMetadata.updateHostId(Gossiper.instance.getHostId(endpoint), endpoint);
}
 

/** unlike excise we just need this endpoint gone without going through any notifications **/
private void removeEndpoint(InetAddress endpoint)
{
    Gossiper.instance.removeEndpoint(endpoint);
    if (!isClientMode)
        SystemKeyspace.removeEndpoint(endpoint);
}
 

protected void addExpireTimeIfFound(InetAddress endpoint, long expireTime)
{
    if (expireTime != 0L)
    {
        Gossiper.instance.addExpireTimeForEndpoint(endpoint, expireTime);
    }
}
 

@Before
public void setup() throws IOException, ConfigurationException
{
    tmd.clearUnsafe();

    // create a ring of 5 nodes
    Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 6);

    MessagingService.instance().listen(FBUtilities.getBroadcastAddress());
    Gossiper.instance.start(1);
    removalhost = hosts.get(5);
    hosts.remove(removalhost);
    removalId = hostIds.get(5);
    hostIds.remove(removalId);
}
 

/**
 * Broadcast leaving status and update local tokenMetadata accordingly
 */
private void startLeaving()
{
    Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.leaving(getLocalTokens()));
    tokenMetadata.addLeavingEndpoint(FBUtilities.getBroadcastAddress());
    PendingRangeCalculatorService.instance.update();
}
 

public void decommission() throws InterruptedException
{
    if (!tokenMetadata.isMember(FBUtilities.getBroadcastAddress()))
        throw new UnsupportedOperationException("local node is not a member of the token ring yet");
    if (tokenMetadata.cloneAfterAllLeft().sortedTokens().size() < 2)
        throw new UnsupportedOperationException("no other normal nodes in the ring; decommission would be pointless");

    PendingRangeCalculatorService.instance.blockUntilFinished();
    for (String keyspaceName : Schema.instance.getNonSystemKeyspaces())
    {
        if (tokenMetadata.getPendingRanges(keyspaceName, FBUtilities.getBroadcastAddress()).size() > 0)
            throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring");
    }

    if (logger.isDebugEnabled())
        logger.debug("DECOMMISSIONING");
    startLeaving();
    long timeout = Math.max(RING_DELAY, BatchlogManager.instance.getBatchlogTimeout());
    setMode(Mode.LEAVING, "sleeping " + timeout + " ms for batch processing and pending range setup", true);
    Thread.sleep(timeout);

    Runnable finishLeaving = new Runnable()
    {
        public void run()
        {
            shutdownClientServers();
            Gossiper.instance.stop();
            MessagingService.instance().shutdown();
            StageManager.shutdownNow();
            setMode(Mode.DECOMMISSIONED, true);
            // let op be responsible for killing the process
        }
    };
    unbootstrap(finishLeaving);
}
 

private void leaveRing()
{
    SystemKeyspace.setBootstrapState(SystemKeyspace.BootstrapState.NEEDS_BOOTSTRAP);
    tokenMetadata.removeEndpoint(FBUtilities.getBroadcastAddress());
    PendingRangeCalculatorService.instance.update();

    Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.left(getLocalTokens(),Gossiper.computeExpireTime()));
    int delay = Math.max(RING_DELAY, Gossiper.intervalInMillis * 2);
    logger.info("Announcing that I have left the ring for {}ms", delay);
    Uninterruptibles.sleepUninterruptibly(delay, TimeUnit.MILLISECONDS);
}
 

/**
 * Creates initial set of nodes and tokens. Nodes are added to StorageService as 'normal'
 */
public static void createInitialRing(StorageService ss, IPartitioner partitioner, List<Token> endpointTokens,
                               List<Token> keyTokens, List<InetAddress> hosts, List<UUID> hostIds, int howMany)
    throws UnknownHostException
{
    // Expand pool of host IDs as necessary
    for (int i = hostIdPool.size(); i < howMany; i++)
        hostIdPool.add(UUID.randomUUID());

    for (int i=0; i<howMany; i++)
    {
        endpointTokens.add(new BigIntegerToken(String.valueOf(10 * i)));
        keyTokens.add(new BigIntegerToken(String.valueOf(10 * i + 5)));
        hostIds.add(hostIdPool.get(i));
    }

    for (int i=0; i<endpointTokens.size(); i++)
    {
        InetAddress ep = InetAddress.getByName("127.0.0." + String.valueOf(i + 1));
        Gossiper.instance.initializeNodeUnsafe(ep, hostIds.get(i), 1);
        Gossiper.instance.injectApplicationState(ep, ApplicationState.TOKENS, new VersionedValue.VersionedValueFactory(partitioner).tokens(Collections.singleton(endpointTokens.get(i))));
        ss.onChange(ep,
                    ApplicationState.STATUS,
                    new VersionedValue.VersionedValueFactory(partitioner).normal(Collections.singleton(endpointTokens.get(i))));
        hosts.add(ep);
    }

    // check that all nodes are in token metadata
    for (int i=0; i<endpointTokens.size(); ++i)
        assertTrue(ss.getTokenMetadata().isMember(hosts.get(i)));
}
 

/**
 * Performs the truncate operatoin, which effectively deletes all data from
 * the column family cfname
 * @param keyspace
 * @param cfname
 * @throws UnavailableException If some of the hosts in the ring are down.
 * @throws TimeoutException
 * @throws IOException
 */
public static void truncateBlocking(String keyspace, String cfname) throws UnavailableException, TimeoutException, IOException
{
    logger.debug("Starting a blocking truncate operation on keyspace {}, CF {}", keyspace, cfname);
    if (isAnyStorageHostDown())
    {
        logger.info("Cannot perform truncate, some hosts are down");
        // Since the truncate operation is so aggressive and is typically only
        // invoked by an admin, for simplicity we require that all nodes are up
        // to perform the operation.
        int liveMembers = Gossiper.instance.getLiveMembers().size();
        throw new UnavailableException(ConsistencyLevel.ALL, liveMembers + Gossiper.instance.getUnreachableMembers().size(), liveMembers);
    }

    Set<InetAddress> allEndpoints = Gossiper.instance.getLiveTokenOwners();
    
    int blockFor = allEndpoints.size();
    final TruncateResponseHandler responseHandler = new TruncateResponseHandler(blockFor);

    // Send out the truncate calls and track the responses with the callbacks.
    Tracing.trace("Enqueuing truncate messages to hosts {}", allEndpoints);
    final Truncation truncation = new Truncation(keyspace, cfname);
    MessageOut<Truncation> message = truncation.createMessage();
    for (InetAddress endpoint : allEndpoints)
        MessagingService.instance().sendRR(message, endpoint, responseHandler);

    // Wait for all
    try
    {
        responseHandler.get();
    }
    catch (TimeoutException e)
    {
        Tracing.trace("Timed out");
        throw e;
    }
}
 

private int waitForSchemaAgreement(InetAddress endpoint) throws TimeoutException
{
    Gossiper gossiper = Gossiper.instance;
    int waited = 0;
    // first, wait for schema to be gossiped.
    while (gossiper.getEndpointStateForEndpoint(endpoint) != null && gossiper.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.SCHEMA) == null)
    {
        Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
        waited += 1000;
        if (waited > 2 * StorageService.RING_DELAY)
            throw new TimeoutException("Didin't receive gossiped schema from " + endpoint + " in " + 2 * StorageService.RING_DELAY + "ms");
    }
    if (gossiper.getEndpointStateForEndpoint(endpoint) == null)
        throw new TimeoutException("Node " + endpoint + " vanished while waiting for agreement");
    waited = 0;
    // then wait for the correct schema version.
    // usually we use DD.getDefsVersion, which checks the local schema uuid as stored in the system keyspace.
    // here we check the one in gossip instead; this serves as a canary to warn us if we introduce a bug that
    // causes the two to diverge (see CASSANDRA-2946)
    while (gossiper.getEndpointStateForEndpoint(endpoint) != null && !gossiper.getEndpointStateForEndpoint(endpoint).getApplicationState(ApplicationState.SCHEMA).value.equals(
            gossiper.getEndpointStateForEndpoint(FBUtilities.getBroadcastAddress()).getApplicationState(ApplicationState.SCHEMA).value))
    {
        Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
        waited += 1000;
        if (waited > 2 * StorageService.RING_DELAY)
            throw new TimeoutException("Could not reach schema agreement with " + endpoint + " in " + 2 * StorageService.RING_DELAY + "ms");
    }
    if (gossiper.getEndpointStateForEndpoint(endpoint) == null)
        throw new TimeoutException("Node " + endpoint + " vanished while waiting for agreement");
    logger.debug("schema for {} matches local schema", endpoint);
    return waited;
}
 

public void run()
{
    MessagingService.Verb verb = message.verb;
    if (MessagingService.DROPPABLE_VERBS.contains(verb)
        && System.currentTimeMillis() > constructionTime + message.getTimeout())
    {
        MessagingService.instance().incrementDroppedMessages(verb);
        return;
    }

    IVerbHandler verbHandler = MessagingService.instance().getVerbHandler(verb);
    if (verbHandler == null)
    {
        logger.debug("Unknown verb {}", verb);
        return;
    }

    try
    {
        verbHandler.doVerb(message, id);
    }
    catch (Throwable t)
    {
        if (message.doCallbackOnFailure())
        {
            MessageOut response = new MessageOut(MessagingService.Verb.INTERNAL_RESPONSE)
                                                .withParameter(MessagingService.FAILURE_RESPONSE_PARAM, MessagingService.ONE_BYTE);
            MessagingService.instance().sendReply(response, id, message.from);
        }

        throw t;
    }
    if (GOSSIP_VERBS.contains(message.verb))
        Gossiper.instance.setLastProcessedMessageAt(constructionTime);
}