Java Code Examples for org.apache.solr.client.solrj.impl.CloudSolrClient#request()

private void testReloadAndAddSimple(String collection) throws IOException, SolrServerException {
  CloudSolrClient cloudClient = cluster.getSolrClient();

  String fieldName = "myNewField";
  addStringField(fieldName, collection, cloudClient);

  CollectionAdminRequest.Reload reloadRequest = CollectionAdminRequest.reloadCollection(collection);
  CollectionAdminResponse response = reloadRequest.process(cloudClient);
  assertEquals(0, response.getStatus());
  assertTrue(response.isSuccess());

  SolrInputDocument doc = new SolrInputDocument();
  doc.addField("id", "1");
  doc.addField(fieldName, "val");
  UpdateRequest ureq = new UpdateRequest().add(doc);
  cloudClient.request(ureq, collection);
}
 

private void assertCountsForRepFactorAndNrtReplicas(CloudSolrClient client, String collectionName) throws Exception {
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
  params.set("collection", collectionName);
  QueryRequest request = new QueryRequest(params);
  request.setPath("/admin/collections");

  NamedList<Object> rsp = client.request(request);
  @SuppressWarnings({"unchecked"})
  NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
  assertNotNull("Cluster state should not be null", cluster);
  @SuppressWarnings({"unchecked"})
  NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
  assertNotNull("Collections should not be null in cluster state", collections);
  assertEquals(1, collections.size());
  @SuppressWarnings({"unchecked"})
  Map<String, Object> collection = (Map<String, Object>) collections.get(collectionName);
  assertNotNull(collection);
  assertEquals(collection.get("replicationFactor"), collection.get("nrtReplicas"));
}
 

@Test
//2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018
public void testSelectedCollectionsByPolicy() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();
  String setSearchPolicyCommand = "{" +
          " 'set-policy': {" +
          "   'search': [" +
          "      {'replica':'<5', 'shard': '#EACH', 'node': '#ANY'}," +
          "    ]" +
          "}}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setSearchPolicyCommand);
  NamedList<Object> response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  String collectionsFilter = "{'policy': 'search'}";
  Map<String, String> createCollectionParameters = new HashMap<>();
  createCollectionParameters.put("testSelected1", "search");
  createCollectionParameters.put("testSelected2", "search");
  testCollectionsPredicate(collectionsFilter, createCollectionParameters);
}
 

/**
 * Creates the cloud collection.
 *
 * @param collectionName
 *          the collection name
 * @param numShards
 *          the num shards
 * @param replicationFactor
 *          the replication factor
 * @param confDir
 *          the conf dir
 * @throws Exception
 *           the exception
 */
private static void createCloudCollection(String collectionName,
    int numShards, int replicationFactor, Path confDir) throws Exception {
  CloudSolrClient client = cloudCluster.getSolrClient();
  String confName = collectionName + "Configuration";
  if (confDir != null) {
    SolrZkClient zkClient = client.getZkStateReader().getZkClient();
    ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
    zkConfigManager.uploadConfigDir(confDir, confName);
  }
  ModifiableSolrParams modParams = new ModifiableSolrParams();
  modParams.set(CoreAdminParams.ACTION,
      CollectionParams.CollectionAction.CREATE.name());
  modParams.set("name", collectionName);
  modParams.set("numShards", numShards);
  modParams.set("replicationFactor", replicationFactor);
  int liveNodes = client.getZkStateReader().getClusterState().getLiveNodes()
      .size();
  int maxShardsPerNode = (int) Math
      .ceil(((double) numShards * replicationFactor) / liveNodes);
  modParams.set("maxShardsPerNode", maxShardsPerNode);
  modParams.set("collection.configName", confName);
  QueryRequest request = new QueryRequest(modParams);
  request.setPath("/admin/collections");
  client.request(request);
}
 

@Test
public void testErrorHandling() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();

  String setClusterPolicyCommand = "{" +
      " 'set-cluster-policy': [" +
      "      {'cores':'<10', 'node':'#ANY'}," +
      "      {'shard': '#EACH', 'node': '#ANY'}," +
      "      {'nodeRole':'overseer', 'replica':0}" +
      "    ]" +
      "}";
  try {
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
    solrClient.request(req);
    fail("expect exception");
  } catch (BaseHttpSolrClient.RemoteExecutionException e) {
    String message = String.valueOf(getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]"));
    assertTrue(message.contains("replica is required in"));
  }

}
 

public void testCreateCollection() throws Exception {
  String prefix = "testCreateCollection";
  String xyz = prefix + "_xyz";
  String abc = prefix + "_abc";

  CloudSolrClient solrClient = cluster.getSolrClient();

  String setClusterPolicyCommand = "{" +
      " 'set-cluster-policy': [" +
      "      {'cores':'<10', 'node':'#ANY'}," +
      "    ]" +
      "}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
  solrClient.request(req);

  String chosenNode = cluster.getRandomJetty(random()).getNodeName();
  CollectionAdminRequest.createCollection(abc, 1, 1)
      .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
      .process(solrClient);
  CollectionAdminRequest.createCollection(xyz, 1, 1)
      .setWithCollection(abc)
      .process(solrClient);

  DocCollection c1 = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(xyz);
  assertNotNull(c1);
  assertEquals(abc, c1.getStr(WITH_COLLECTION));
  Replica replica = c1.getReplicas().get(0);
  String nodeName = replica.getNodeName();

  assertEquals(chosenNode, nodeName);
}
 

public static NamedList<Object> doPropertyAction(CloudSolrClient client, String... paramsIn) throws IOException, SolrServerException {
  assertTrue("paramsIn must be an even multiple of 2, it is: " + paramsIn.length, (paramsIn.length % 2) == 0);
  ModifiableSolrParams params = new ModifiableSolrParams();
  for (int idx = 0; idx < paramsIn.length; idx += 2) {
    params.set(paramsIn[idx], paramsIn[idx + 1]);
  }
  QueryRequest request = new QueryRequest(params);
  request.setPath("/admin/collections");
  return client.request(request);
}
 

@Before
public void setUp() throws Exception {
  super.setUp();

  // remove everything from autoscaling.json in ZK
  zkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, "{}".getBytes(UTF_8), true);

  if (cluster.getJettySolrRunners().size() > NODE_COUNT) {
    // stop some to get to original state
    int numJetties = cluster.getJettySolrRunners().size();
    for (int i = 0; i < numJetties - NODE_COUNT; i++) {
      JettySolrRunner randomJetty = cluster.getRandomJetty(random());
      List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
      for (int i1 = 0; i1 < jettySolrRunners.size(); i1++) {
        JettySolrRunner jettySolrRunner = jettySolrRunners.get(i1);
        if (jettySolrRunner == randomJetty) {
          JettySolrRunner j = cluster.stopJettySolrRunner(i1);
          cluster.waitForJettyToStop(j);
          break;
        }
      }
    }
  }

  cluster.deleteAllCollections();

  CloudSolrClient solrClient = cluster.getSolrClient();

  String setClusterPolicyCommand = "{" +
      " 'set-cluster-policy': [" +
      "      {'cores':'<10', 'node':'#ANY'}," +
      "      {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
      "      {'nodeRole':'overseer', 'replica':0}" +
      "    ]" +
      "}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
  NamedList<Object> response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  String setClusterPreferencesCommand = "{" +
      "'set-cluster-preferences': [" +
      "{'minimize': 'cores'}," +
      "{'maximize': 'freedisk','precision': 100}]" +
      "}";
  req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPreferencesCommand);
  response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
  deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
  deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
  deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
  deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);

  reset();
}
 

protected void checkCollectionExists(CloudSolrClient cloudSolrClient, String collection) throws Exception {
  if (!cloudSolrClient.getZkStateReader().getClusterState().hasCollection(collection)) {

    synchronized (this) {
      if (curatorClient == null) {
        curatorClient = CuratorFrameworkFactory.newClient(cloudSolrClient.getZkHost(), new ExponentialBackoffRetry(1000, 3));
        curatorClient.start();
      }
    }

    String lockPath = lockZnodePath+"/"+collection;
    InterProcessMutex lock = new InterProcessMutex(curatorClient, lockPath);
    try {
      if (!lock.acquire(60, TimeUnit.SECONDS)) {
        // couldn't acquire the lock, but let's check to see if the collection was created before failing
        cloudSolrClient.getZkStateReader().updateClusterState();
        if (!cloudSolrClient.getZkStateReader().getClusterState().hasCollection(collection)) {
          throw new IllegalStateException("Failed to acquire the create collection lock within 60 seconds! Cannot create "+collection);
        }
      }

      // we have the lock here ...
      cloudSolrClient.getZkStateReader().updateClusterState();
      if (!cloudSolrClient.getZkStateReader().getClusterState().hasCollection(collection)) {
        log.info("Acquired inter-process lock for creating " + collection);

        // ok, it doesn't exist ... go ahead and create it
        long startMs = System.currentTimeMillis();
        createCollection(cloudSolrClient, collection);
        log.info("Collection created, took "+(System.currentTimeMillis()-startMs)+" ms ... updating alias: "+alias);

        // add the new collection to the collection alias if one is registered
        if (alias != null) {
          List<String> aliasList = getAliasList(cloudSolrClient, alias);
          if (!aliasList.contains(collection)) {
            aliasList.add(collection);
            log.info("Added " + collection + " to the " + alias + " alias");

            // trim the alias down to the desired size
            int numColls = aliasList.size();
            if (maxCollectionsInAlias > 0 && numColls > maxCollectionsInAlias) {
              Collections.sort(aliasList);

              int numToRemove = numColls - maxCollectionsInAlias;
              aliasList = aliasList.subList(numToRemove, numColls);
              log.info("Removed "+numToRemove+" collections from alias: "+aliasList);
            }

            CollectionAdminRequest.CreateAlias createAliasCmd = new CollectionAdminRequest.CreateAlias();
            createAliasCmd.setAliasName(alias);
            createAliasCmd.setAliasedCollections(StrUtils.join(aliasList, ','));
            cloudSolrClient.request(createAliasCmd);
          }
        }
      } else {
        log.info("Collection "+collection+" was created by another process while we were waiting to acquire the lock ...");
      }
    } finally {
      lock.release();
    }
  }
}
 

@Test
public void testCooldown() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();
  String setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'node_added_cooldown_trigger'," +
      "'event' : 'nodeAdded'," +
      "'waitFor' : '" + waitForSeconds + "s'," +
      "'enabled' : true," +
      "'actions' : [" +
      "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}" +
      "]" +
      "}}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
  NamedList<Object> response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  String setListenerCommand1 = "{" +
      "'set-listener' : " +
      "{" +
      "'name' : 'bar'," +
      "'trigger' : 'node_added_cooldown_trigger'," +
      "'stage' : ['FAILED','SUCCEEDED', 'IGNORED']," +
      "'class' : '" + TestTriggerListener.class.getName() + "'" +
      "}" +
      "}";
  req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand1);
  response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  JettySolrRunner newNode = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
  assertTrue("The trigger did not fire at all", await);
  assertTrue(triggerFired.get());
  // wait for listener to capture the SUCCEEDED stage
  Thread.sleep(1000);

  List<CapturedEvent> capturedEvents = listenerEvents.get("bar");
  // we may get a few IGNORED events if other tests caused events within cooldown period
  assertTrue(capturedEvents.toString(), capturedEvents.size() > 0);
  long prevTimestamp = capturedEvents.get(capturedEvents.size() - 1).timestamp;

  resetTriggerAndListenerState();

  JettySolrRunner newNode2 = cluster.startJettySolrRunner();
  await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
  assertTrue("The trigger did not fire at all", await);
  // wait for listener to capture the SUCCEEDED stage
  Thread.sleep(2000);

  // there must be at least one IGNORED event due to cooldown, and one SUCCEEDED event
  capturedEvents = listenerEvents.get("bar");
  assertEquals(capturedEvents.toString(), 1, capturedEvents.size());
  CapturedEvent ev = capturedEvents.get(0);
  assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
  // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED
  // must be larger than cooldown period
  assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_COOLDOWN_PERIOD_SECONDS));
  prevTimestamp = ev.timestamp;

  // this also resets the cooldown period
  long modifiedCooldownPeriodSeconds = 7;
  String setPropertiesCommand = "{\n" +
      "\t\"set-properties\" : {\n" +
      "\t\t\"" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "\" : " + modifiedCooldownPeriodSeconds + "\n" +
      "\t}\n" +
      "}";
  solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
  req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
  response = solrClient.request(req);

  resetTriggerAndListenerState();

  JettySolrRunner newNode3 = cluster.startJettySolrRunner();
  await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
  assertTrue("The trigger did not fire at all", await);
  triggerFiredLatch = new CountDownLatch(1);
  triggerFired.compareAndSet(true, false);
  // add another node
  JettySolrRunner newNode4 = cluster.startJettySolrRunner();
  await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
  assertTrue("The trigger did not fire at all", await);
  // wait for listener to capture the SUCCEEDED stage
  Thread.sleep(2000);

  // there must be two SUCCEEDED (due to newNode3 and newNode4) and maybe some ignored events
  capturedEvents = listenerEvents.get("bar");
  assertTrue(capturedEvents.toString(), capturedEvents.size() >= 2);
  // first event should be SUCCEEDED
  ev = capturedEvents.get(0);
  assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);

  ev = capturedEvents.get(capturedEvents.size() - 1);
  assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage);
  // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED
  // must be larger than the modified cooldown period
  assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(modifiedCooldownPeriodSeconds));
}
 

@Test
// commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
// commented out on: 17-Feb-2019   @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
public void testScheduledTrigger() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();

  // this collection will place 2 cores on 1st node and 1 core on 2nd node
  String collectionName = "testScheduledTrigger";
  CollectionAdminRequest.createCollection(collectionName, 1, 3)
      .setMaxShardsPerNode(5).process(solrClient);
  
  cluster.waitForActiveCollection(collectionName, 1, 3);

  // create a policy which allows only 1 core per node thereby creating a violation for the above collection
  String setClusterPolicy = "{\n" +
      "  \"set-cluster-policy\" : [\n" +
      "    {\"cores\" : \"<2\", \"node\" : \"#EACH\"}\n" +
      "  ]\n" +
      "}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicy);
  NamedList<Object> response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  // start a new node which can be used to balance the cluster as per policy
  JettySolrRunner newNode = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);

  String setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'sched_trigger_integration1'," +
      "'event' : 'scheduled'," +
      "'startTime' : '" + new Date().toInstant().toString() + "'" +
      "'every' : '+3SECONDS'" +
      "'actions' : [" +
      "{'name' : 'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
      "{'name' : 'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
      "{'name' : 'recorder', 'class': '" + ContextPropertiesRecorderAction.class.getName() + "'}" +
      "]}}";
  req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
  response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  assertTrue("ScheduledTrigger did not fire in time", triggerFiredLatch.await(45, TimeUnit.SECONDS));
  assertEquals(1, events.size());
  Map<String, Object> actionContextProps = actionContextPropertiesRef.get();
  assertNotNull(actionContextProps);
  TriggerEvent event = events.iterator().next();
  @SuppressWarnings({"unchecked", "rawtypes"})
  List<SolrRequest> operations = (List<SolrRequest>) actionContextProps.get("operations");
  assertNotNull(operations);
  assertEquals(1, operations.size());
  for (@SuppressWarnings({"rawtypes"})SolrRequest operation : operations) {
    SolrParams params = operation.getParams();
    assertEquals(newNode.getNodeName(), params.get("targetNode"));
  }
}
 

@Test
public void testContinueTriggersOnOverseerRestart() throws Exception  {
  CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus();
  CloudSolrClient solrClient = cluster.getSolrClient();
  CollectionAdminResponse adminResponse = status.process(solrClient);
  NamedList<Object> response = adminResponse.getResponse();
  String leader = (String) response.get("leader");
  JettySolrRunner overseerNode = null;
  int index = -1;
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (int i = 0; i < jettySolrRunners.size(); i++) {
    JettySolrRunner runner = jettySolrRunners.get(i);
    if (runner.getNodeName().equals(leader)) {
      overseerNode = runner;
      index = i;
      break;
    }
  }
  assertNotNull(overseerNode);

  String setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'node_added_triggerCTOOR'," +
      "'event' : 'nodeAdded'," +
      "'waitFor' : '" + waitForSeconds + "s'," +
      "'enabled' : true," +
      "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
      "}}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
  response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");
  assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time",
             actionInitCalled.await(60, TimeUnit.SECONDS));

  // stop the overseer, somebody else will take over as the overseer
  JettySolrRunner j = cluster.stopJettySolrRunner(index);
  cluster.waitForJettyToStop(j);
  Thread.sleep(10000);
  JettySolrRunner newNode = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  assertTrue("trigger did not fire even after await()ing an excessive amount of time",
             triggerFiredLatch.await(60, TimeUnit.SECONDS));
  assertTrue(triggerFired.get());
  NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
  assertNotNull(nodeAddedEvent);
  @SuppressWarnings({"unchecked"})
  List<String> nodeNames = (List<String>)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
  assertTrue(nodeNames.contains(newNode.getNodeName()));
}
 

@Test
// commented out on: 17-Feb-2019   @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018
public void testTriggerThrottling() throws Exception  {
  // for this test we want to create two triggers so we must assert that the actions were created twice
  actionInitCalled = new CountDownLatch(2);
  // similarly we want both triggers to fire
  triggerFiredLatch = new CountDownLatch(2);

  CloudSolrClient solrClient = cluster.getSolrClient();

  // first trigger
  String setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'node_added_trigger1'," +
      "'event' : 'nodeAdded'," +
      "'waitFor' : '0s'," +
      "'enabled' : true," +
      "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" +
      "}}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
  NamedList<Object> response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  // second trigger
  setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'node_added_trigger2'," +
      "'event' : 'nodeAdded'," +
      "'waitFor' : '0s'," +
      "'enabled' : true," +
      "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" +
      "}}";
  req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
  response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  // wait until the two instances of action are created
  assertTrue("Two TriggerAction instances were not created "+
             "even after await()ing an excessive amount of time",
             actionInitCalled.await(60, TimeUnit.SECONDS));

  JettySolrRunner newNode = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  assertTrue("Both triggers did not fire event after await()ing an excessive amount of time",
             triggerFiredLatch.await(60, TimeUnit.SECONDS));

  // reset shared state
  lastActionExecutedAt.set(0);
  TriggerIntegrationTest.actionInitCalled = new CountDownLatch(2);
  triggerFiredLatch = new CountDownLatch(2);

  setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'node_lost_trigger1'," +
      "'event' : 'nodeLost'," +
      "'waitFor' : '0s'," +
      "'enabled' : true," +
      "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" +
      "}}";
  req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
  response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'node_lost_trigger2'," +
      "'event' : 'nodeLost'," +
      "'waitFor' : '0s'," +
      "'enabled' : true," +
      "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" +
      "}}";
  req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
  response = solrClient.request(req);
  assertEquals(response.get("result").toString(), "success");

  // wait until the two instances of action are created
  assertTrue("Two TriggerAction instances were not created "+
             "even after await()ing an excessive amount of time",
             actionInitCalled.await(60, TimeUnit.SECONDS));

  // stop the node we had started earlier
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (int i = 0; i < jettySolrRunners.size(); i++) {
    JettySolrRunner jettySolrRunner = jettySolrRunners.get(i);
    if (jettySolrRunner == newNode) {
      JettySolrRunner j = cluster.stopJettySolrRunner(i);
      cluster.waitForJettyToStop(j);
      break;
    }
  }

  assertTrue("Both triggers did not fire event after await()ing an excessive amount of time",
             triggerFiredLatch.await(60, TimeUnit.SECONDS));
}
 

public void testMetricsTag() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();
  String setClusterPolicyCommand = "{" +
      " 'set-cluster-policy': [" +
      "      {'cores':'<10', 'node':'#ANY'}," +
      "      {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
      "      {'metrics:abc':'overseer', 'replica':0}" +
      "    ]" +
      "}";
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
  try {
    solrClient.request(req);
    fail("expected exception");
  } catch (BaseHttpSolrClient.RemoteExecutionException e) {
    // expected
    assertTrue(String.valueOf(getObjectByPath(e.getMetaData(),
        false, "error/details[0]/errorMessages[0]")).contains("Invalid metrics: param in"));
  }
  setClusterPolicyCommand = "{" +
      " 'set-cluster-policy': [" +
      "      {'cores':'<10', 'node':'#ANY'}," +
      "      {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
      "      {'metrics:solr.node:ADMIN./admin/authorization.clientErrors:count':'>58768765', 'replica':0}" +
      "    ]" +
      "}";
  req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
  solrClient.request(req);

  final String collectionName = "metrics_tags";
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1)
      .process(cluster.getSolrClient());
  cluster.waitForActiveCollection(collectionName, 1, 1);
  DocCollection collection = getCollectionState(collectionName);
  DistributedQueueFactory queueFactory = new ZkDistributedQueueFactory(cluster.getZkClient());
  try (SolrCloudManager provider = new SolrClientCloudManager(queueFactory, solrClient)) {
    List<String> tags = Arrays.asList("metrics:solr.node:ADMIN./admin/authorization.clientErrors:count",
        "metrics:solr.jvm:buffers.direct.Count");
    Map<String, Object> val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), tags);
    for (String tag : tags) {
      assertNotNull("missing : " + tag, val.get(tag));
    }
    val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), Collections.singleton("diskType"));

    Set<String> diskTypes = ImmutableSet.of("rotational", "ssd");
    assertTrue(diskTypes.contains(val.get("diskType")));
  }
}
 

@Test
public void test() throws Exception {
  CloudSolrClient cloudClient = cluster.getSolrClient();
  String coll = "deletenodetest_coll";
  ClusterState state = cloudClient.getZkStateReader().getClusterState();
  Set<String> liveNodes = state.getLiveNodes();
  ArrayList<String> l = new ArrayList<>(liveNodes);
  Collections.shuffle(l, random());
  CollectionAdminRequest.Create create = pickRandom(
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1),
      // check RF=1
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 0),
      CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 0)
      );
  create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3);
  cloudClient.request(create);
  state = cloudClient.getZkStateReader().getClusterState();
  String node2bdecommissioned = l.get(0);
  // check what replicas are on the node, and whether the call should fail
  boolean shouldFail = false;
  DocCollection docColl = state.getCollection(coll);
  log.info("#### DocCollection: {}", docColl);
  List<Replica> replicas = docColl.getReplicas(node2bdecommissioned);
  if (replicas != null) {
    for (Replica replica : replicas) {
      String shard = docColl.getShardId(node2bdecommissioned, replica.getStr(ZkStateReader.CORE_NAME_PROP));
      Slice slice = docColl.getSlice(shard);
      boolean hasOtherNonPullReplicas = false;
      for (Replica r: slice.getReplicas()) {
        if (!r.getName().equals(replica.getName()) &&
            !r.getNodeName().equals(node2bdecommissioned) &&
            r.getType() != Replica.Type.PULL) {
          hasOtherNonPullReplicas = true;
          break;
        }
      }
      if (!hasOtherNonPullReplicas) {
        shouldFail = true;
        break;
      }
    }
  }
  new CollectionAdminRequest.DeleteNode(node2bdecommissioned).processAsync("003", cloudClient);
  CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("003");
  CollectionAdminRequest.RequestStatusResponse rsp = null;
  for (int i = 0; i < 200; i++) {
    rsp = requestStatus.process(cloudClient);
    if (rsp.getRequestStatus() == RequestStatusState.FAILED || rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
      break;
    }
    Thread.sleep(50);
  }
  if (log.isInfoEnabled()) {
    log.info("####### DocCollection after: {}", cloudClient.getZkStateReader().getClusterState().getCollection(coll));
  }
  if (shouldFail) {
    assertTrue(String.valueOf(rsp), rsp.getRequestStatus() == RequestStatusState.FAILED);
  } else {
    assertFalse(String.valueOf(rsp), rsp.getRequestStatus() == RequestStatusState.FAILED);
  }
}
 

@Test
public void test() throws Exception {
  
  String collection = "addreplicatest_coll";

  CloudSolrClient cloudClient = cluster.getSolrClient();

  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collection, "conf1", 2, 1);
  create.setMaxShardsPerNode(2);
  cloudClient.request(create);
  
  cluster.waitForActiveCollection(collection, 2, 2);

  ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
  DocCollection coll = clusterState.getCollection(collection);
  String sliceName = coll.getSlices().iterator().next().getName();
  Collection<Replica> replicas = coll.getSlice(sliceName).getReplicas();
  CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard(collection, sliceName);
  addReplica.processAsync("000", cloudClient);
  CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("000");
  CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
  assertNotSame(rsp.getRequestStatus(), COMPLETED);
  
  // wait for async request success
  boolean success = false;
  for (int i = 0; i < 200; i++) {
    rsp = requestStatus.process(cloudClient);
    if (rsp.getRequestStatus() == COMPLETED) {
      success = true;
      break;
    }
    assertNotSame(rsp.toString(), rsp.getRequestStatus(), RequestStatusState.FAILED);
    Thread.sleep(500);
  }
  assertTrue(success);
  
  Collection<Replica> replicas2 = cloudClient.getZkStateReader().getClusterState().getCollection(collection).getSlice(sliceName).getReplicas();
  replicas2.removeAll(replicas);
  assertEquals(1, replicas2.size());

  // use waitForFinalState
  addReplica.setWaitForFinalState(true);
  addReplica.processAsync("001", cloudClient);
  requestStatus = CollectionAdminRequest.requestStatus("001");
  rsp = requestStatus.process(cloudClient);
  assertNotSame(rsp.getRequestStatus(), COMPLETED);
  // wait for async request success
  success = false;
  for (int i = 0; i < 200; i++) {
    rsp = requestStatus.process(cloudClient);
    if (rsp.getRequestStatus() == COMPLETED) {
      success = true;
      break;
    }
    assertNotSame(rsp.toString(), rsp.getRequestStatus(), RequestStatusState.FAILED);
    Thread.sleep(500);
  }
  assertTrue(success);
  // let the client watch fire
  Thread.sleep(1000);
  clusterState = cloudClient.getZkStateReader().getClusterState();
  coll = clusterState.getCollection(collection);
  Collection<Replica> replicas3 = coll.getSlice(sliceName).getReplicas();
  replicas3.removeAll(replicas);
  String replica2 = replicas2.iterator().next().getName();
  assertEquals(2, replicas3.size());
  for (Replica replica : replicas3) {
    if (replica.getName().equals(replica2)) {
      continue; // may be still recovering
    }
    assertSame(coll.toString() + "\n" + replica.toString(), replica.getState(), Replica.State.ACTIVE);
  }
}
 

@Test
public void test() throws Exception {
  cluster.waitForAllNodes(5);
  String coll = "utilizenodecoll";
  CloudSolrClient cloudClient = cluster.getSolrClient();
  
  log.info("Creating Collection...");
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 2)
      .setMaxShardsPerNode(2);
  cloudClient.request(create);

  log.info("Spinning up additional jettyX...");
  JettySolrRunner jettyX = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);

  assertNoReplicas("jettyX should not yet be utilized: ", coll, jettyX);

  if (log.isInfoEnabled()) {
    log.info("Sending UTILIZE command for jettyX ({})", jettyX.getNodeName());
  }
  cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyX.getNodeName()));

  // TODO: aparently we can't assert this? ...
  //
  // assertSomeReplicas("jettyX should now be utilized: ", coll, jettyX);
  //
  // ... it appears from the docs that unless there are policy violations,
  // this can be ignored unless jettyX has less "load" then other jetty instances?
  //
  // if the above is true, that means that this test is incredibly weak...
  // unless we know jettyX has at least one replica, then all the subsequent testing of the
  // port blacklist & additional UTILIZE command for jettyY are a waste of time.
  //
  // should we skip spinning up a *new* jettyX, and instead just pick an existing jetty?

  if (log.isInfoEnabled()) {
    log.info("jettyX replicas prior to being blacklisted: {}", getReplicaList(coll, jettyX));
  }
  
  String setClusterPolicyCommand = "{" +
    " 'set-cluster-policy': [" +
    "    {'port':" + jettyX.getLocalPort() +
    "     , 'replica':0}" +
    "  ]" +
    "}";
  if (log.isInfoEnabled()) {
    log.info("Setting new policy to blacklist jettyX ({}) port={}",
        jettyX.getNodeName(), jettyX.getLocalPort());
  }
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
  NamedList<Object> response = cloudClient.request(req);
  assertEquals(req + " => " + response,
               "success", response.get("result").toString());

  log.info("Spinning up additional jettyY...");
  JettySolrRunner jettyY = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  
  assertNoReplicas("jettyY should not yet be utilized: ", coll, jettyY);
  if (log.isInfoEnabled()) {
    log.info("jettyX replicas prior to utilizing jettyY: {}", getReplicaList(coll, jettyX));
    log.info("Sending UTILIZE command for jettyY ({})", jettyY.getNodeName()); // logOk
  }
  cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyY.getNodeName()));

  assertSomeReplicas("jettyY should now be utilized: ", coll, jettyY);
}
 

@Test
public void testRepFactor1LeaderStartup() throws Exception {

  CloudSolrClient solrClient = cluster.getSolrClient();

  String collectionName = "testRepFactor1LeaderStartup";
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
      .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName());
  create.process(solrClient);

 cluster.waitForActiveCollection(collectionName, 1, 1);

  solrClient.setDefaultCollection(collectionName);

  String addListenerCommand = "{" +
      "'add-listener' : {'name':'newSearcherListener','event':'newSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "'add-listener' : {'name':'firstSearcherListener','event':'firstSearcher', 'class':'" + SleepingSolrEventListener.class.getName() + "'}" +
      "}";

  ConfigRequest request = new ConfigRequest(addListenerCommand);
  solrClient.request(request);

  solrClient.add(new SolrInputDocument("id", "1"));
  solrClient.commit();

  AtomicInteger expectedDocs = new AtomicInteger(1);
  AtomicReference<String> failingCoreNodeName = new AtomicReference<>();
  CollectionStateWatcher stateWatcher = createActiveReplicaSearcherWatcher(expectedDocs, failingCoreNodeName);

  JettySolrRunner runner = cluster.getJettySolrRunner(0);
  runner.stop();
  
  cluster.waitForJettyToStop(runner);
  // check waitForState only after we are sure the node has shutdown and have forced an update to liveNodes
  // ie: workaround SOLR-13490
  cluster.getSolrClient().getZkStateReader().updateLiveNodes();
  waitForState("jetty count:" + cluster.getJettySolrRunners().size(), collectionName, clusterShape(1, 0));
  
  // restart
  sleepTime.set(1000);
  runner.start();
  cluster.waitForAllNodes(30);
  cluster.getSolrClient().getZkStateReader().registerCollectionStateWatcher(collectionName, stateWatcher);
  cluster.waitForActiveCollection(collectionName, 1, 1);
  assertNull("No replica should have been active without registering a searcher, found: " + failingCoreNodeName.get(), failingCoreNodeName.get());
  cluster.getSolrClient().getZkStateReader().removeCollectionStateWatcher(collectionName, stateWatcher);
}
 

@Test
  public void testMoveReplicaMainCollection() throws Exception {
    String prefix = "testMoveReplicaMainCollection";
    String xyz = prefix + "_xyz";
    String abc = prefix + "_abc";

    CloudSolrClient solrClient = cluster.getSolrClient();

    String setClusterPolicyCommand = "{" +
        " 'set-cluster-policy': [" +
        "      {'cores':'<10', 'node':'#ANY'}," +
        "      {'replica':'<2', 'node':'#ANY'}," +
        "    ]" +
        "}";
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
    solrClient.request(req);

    String chosenNode = cluster.getRandomJetty(random()).getNodeName();
    log.info("Chosen node {} for collection {}", chosenNode, abc);
    CollectionAdminRequest.createCollection(abc, 1, 1)
        .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
        .process(solrClient);
    CollectionAdminRequest.createCollection(xyz, 1, 1)
        .setWithCollection(abc)
        .process(solrClient);

    String otherNode = null;
    for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
      if (!chosenNode.equals(jettySolrRunner.getNodeName())) {
        otherNode = jettySolrRunner.getNodeName();
      }
    }

    DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz);
    DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc);
    assertNull(collection.getReplicas(otherNode)); // sanity check
    assertNull(withCollection.getReplicas(otherNode)); // sanity check

    CollectionAdminRequest.MoveReplica moveReplica = new CollectionAdminRequest.MoveReplica(xyz, collection.getReplicas().iterator().next().getName(), otherNode);
    moveReplica.setWaitForFinalState(true);
    moveReplica.process(solrClient);
//    zkClient().printLayoutToStdOut();
    collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); // refresh
    DocCollection withCollectionRefreshed = solrClient.getZkStateReader().getClusterState().getCollection(abc); // refresh
    assertTrue(collection.getReplicas().stream().noneMatch(
        replica -> withCollectionRefreshed.getReplicas(replica.getNodeName()) == null
            || withCollectionRefreshed.getReplicas(replica.getNodeName()).isEmpty()));
  }
 

@Test
public void testAddMultipleReplicas() throws Exception  {

  String collection = "testAddMultipleReplicas";
  CloudSolrClient cloudClient = cluster.getSolrClient();

  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collection, "conf1", 1, 1);
  create.setMaxShardsPerNode(2);
  cloudClient.request(create);
  cluster.waitForActiveCollection(collection, 1, 1);

  CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard(collection, "shard1")
      .setNrtReplicas(1)
      .setTlogReplicas(1)
      .setPullReplicas(1);
  RequestStatusState status = addReplica.processAndWait(collection + "_xyz1", cloudClient, 120);
  assertEquals(COMPLETED, status);
  
  cluster.waitForActiveCollection(collection, 1, 4);
  
  DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollectionOrNull(collection);
  assertNotNull(docCollection);
  assertEquals(4, docCollection.getReplicas().size());
  assertEquals(2, docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).size());
  assertEquals(1, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).size());
  assertEquals(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).size());

  // try to add 5 more replicas which should fail because numNodes(4)*maxShardsPerNode(2)=8 and 4 replicas already exist
  addReplica = CollectionAdminRequest.addReplicaToShard(collection, "shard1")
      .setNrtReplicas(3)
      .setTlogReplicas(1)
      .setPullReplicas(1);
  status = addReplica.processAndWait(collection + "_xyz1", cloudClient, 120);
  assertEquals(FAILED, status);
  docCollection = cloudClient.getZkStateReader().getClusterState().getCollectionOrNull(collection);
  assertNotNull(docCollection);
  // sanity check that everything is as before
  assertEquals(4, docCollection.getReplicas().size());
  assertEquals(2, docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).size());
  assertEquals(1, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).size());
  assertEquals(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).size());

  // but adding any number of replicas is supported if an explicit create node set is specified
  // so test that as well
  LinkedHashSet<String> createNodeSet = new LinkedHashSet<>(2);
  createNodeSet.add(cluster.getRandomJetty(random()).getNodeName());
  while (true)  {
    String nodeName = cluster.getRandomJetty(random()).getNodeName();
    if (createNodeSet.add(nodeName))  break;
  }
  addReplica = CollectionAdminRequest.addReplicaToShard(collection, "shard1")
      .setNrtReplicas(3)
      .setTlogReplicas(1)
      .setPullReplicas(1)
      .setCreateNodeSet(String.join(",", createNodeSet));
  status = addReplica.processAndWait(collection + "_xyz1", cloudClient, 120);
  assertEquals(COMPLETED, status);
  waitForState("Timedout wait for collection to be created", collection, clusterShape(1, 9));
  docCollection = cloudClient.getZkStateReader().getClusterState().getCollectionOrNull(collection);
  assertNotNull(docCollection);
  // sanity check that everything is as before
  assertEquals(9, docCollection.getReplicas().size());
  assertEquals(5, docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).size());
  assertEquals(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).size());
  assertEquals(2, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).size());
}