Java Code Examples for org.apache.solr.client.solrj.embedded.JettySolrRunner#getLocalPort()

public void startJetty() throws Exception {

      Properties props = new Properties();
      props.setProperty("solrconfig", "bad_solrconfig.xml");
      props.setProperty("solr.data.dir", getDataDir());

      JettyConfig jettyConfig = JettyConfig.builder(buildJettyConfig("/solr")).setPort(port).build();

      jetty = new JettySolrRunner(getHomeDir(), props, jettyConfig);
      jetty.start();
      int newPort = jetty.getLocalPort();
      if (port != 0 && newPort != port) {
        fail("TESTING FAILURE: could not grab requested port.");
      }
      this.port = newPort;
//      System.out.println("waiting.........");
//      Thread.sleep(5000);
    }
 

public void startJetty() throws Exception {

      Properties props = new Properties();
      props.setProperty("solrconfig", "bad_solrconfig.xml");
      props.setProperty("solr.data.dir", getDataDir());

      JettyConfig jettyConfig = JettyConfig.builder(buildJettyConfig("/solr")).setPort(port).build();

      jetty = new JettySolrRunner(getHomeDir(), props, jettyConfig);
      jetty.start();
      int newPort = jetty.getLocalPort();
      if (port != 0 && newPort != port) {
        fail("TESTING FAILURE: could not grab requested port.");
      }
      this.port = newPort;
//      System.out.println("waiting.........");
//      Thread.sleep(5000);
    }
 

public static JettySolrRunner createAndStartJetty(String solrHome, Properties nodeProperties, JettyConfig jettyConfig) throws Exception {

    initCore(null, null, solrHome);

    Path coresDir = createTempDir().resolve("cores");

    Properties props = new Properties();
    props.setProperty("name", DEFAULT_TEST_CORENAME);
    props.setProperty("configSet", "collection1");
    props.setProperty("config", "${solrconfig:solrconfig.xml}");
    props.setProperty("schema", "${schema:schema.xml}");

    writeCoreProperties(coresDir.resolve("core"), props, "RestTestBase");

    Properties nodeProps = new Properties(nodeProperties);
    nodeProps.setProperty("coreRootDirectory", coresDir.toString());
    nodeProps.setProperty("configSetBaseDir", solrHome);

    jetty = new JettySolrRunner(solrHome, nodeProps, jettyConfig);
    jetty.start();
    port = jetty.getLocalPort();
    log.info("Jetty Assigned Port#{}", port);
    return jetty;
  }
 

protected JettySolrRunner getJettyOnPort(int port) {
  JettySolrRunner theJetty = null;
  for (JettySolrRunner jetty : jettys) {
    if (port == jetty.getLocalPort()) {
      theJetty = jetty;
      break;
    }
  }

  if (theJetty == null) {
    if (controlJetty.getLocalPort() == port) {
      theJetty = controlJetty;
    }
  }

  if (theJetty == null)
    fail("Not able to find JettySolrRunner for port: "+port);

  return theJetty;
}
 

public void testCreateCollectionAddReplica() throws Exception  {
  final JettySolrRunner jetty = cluster.getRandomJetty(random());
  final String jettyNodeName = jetty.getNodeName();
  final int port = jetty.getLocalPort();

  final String commands =  "{set-policy :{c1 : [{replica:0 , shard:'#EACH', port: '!" + port + "'}]}}";
  cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));

  final String collectionName = "testCreateCollectionAddReplica";
  log.info("Creating collection {}", collectionName);
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1)
      .setPolicy("c1")
      .process(cluster.getSolrClient());

  waitForState("Should have found exactly one replica, only on expected jetty: " +
               jettyNodeName + "/" + port,
               collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 1),
               120, TimeUnit.SECONDS);

  log.info("Adding replica to {}", collectionName);
  CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
    .process(cluster.getSolrClient());
  
  waitForState("Should have found exactly two replicas, only on expected jetty: " +
               jettyNodeName + "/" + port,
               collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 2),
               120, TimeUnit.SECONDS);

}
 

public void testCreateCollectionAddShardUsingPolicy() throws Exception {
  JettySolrRunner jetty = cluster.getRandomJetty(random());
  int port = jetty.getLocalPort();

  String commands =  "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" + port + "'}]}}";
  cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
  Map<String, Object> json = Utils.getJson(cluster.getZkClient(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, true);
  assertEquals("full json:"+ Utils.toJSONString(json) , "#EACH",
      Utils.getObjectByPath(json, true, "/policies/c1[0]/shard"));

  final String collectionName = "addshard_using_policy";
  CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "s1,s2", 1)
      .setPolicy("c1")
      .process(cluster.getSolrClient());

  cluster.waitForActiveCollection(collectionName, 2, 2);
  DocCollection coll = getCollectionState(collectionName);
  assertEquals("c1", coll.getPolicyName());
  assertEquals(2,coll.getReplicas().size());
  coll.forEachReplica((s, replica) -> assertEquals(jetty.getNodeName(), replica.getNodeName()));
  
  CollectionAdminRequest.createShard(collectionName, "s3").process(cluster.getSolrClient());

  cluster.waitForActiveCollection(collectionName, 3, 3);

  coll = getCollectionState(collectionName);
  assertEquals(1, coll.getSlice("s3").getReplicas().size());
  coll.getSlice("s3").forEach(replica -> assertEquals(jetty.getNodeName(), replica.getNodeName()));
}
 

@BeforeClass
public static void beforeTest() throws Exception {
  File homeDir = createTempDir().toFile();

  File collDir = new File(homeDir, "collection1");
  File dataDir = new File(collDir, "data");
  File confDir = new File(collDir, "conf");

  homeDir.mkdirs();
  collDir.mkdirs();
  dataDir.mkdirs();
  confDir.mkdirs();

  FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(homeDir, "solr.xml"));
  String src_dir = TEST_HOME() + "/collection1/conf";
  FileUtils.copyFile(new File(src_dir, "schema-tiny.xml"), 
                     new File(confDir, "schema.xml"));
  FileUtils.copyFile(new File(src_dir, "solrconfig-solcoreproperties.xml"), 
                     new File(confDir, "solrconfig.xml"));
  FileUtils.copyFile(new File(src_dir, "solrconfig.snippet.randomindexconfig.xml"), 
                     new File(confDir, "solrconfig.snippet.randomindexconfig.xml"));

  Properties p = new Properties();
  p.setProperty("foo.foo1", "f1");
  p.setProperty("foo.foo2", "f2");
  Writer fos = new OutputStreamWriter(new FileOutputStream(new File(confDir, "solrcore.properties")), StandardCharsets.UTF_8);
  p.store(fos, null);
  IOUtils.close(fos);

  Files.createFile(collDir.toPath().resolve("core.properties"));


  Properties nodeProperties = new Properties();
  // this sets the property for jetty starting SolrDispatchFilter
  if (System.getProperty("solr.data.dir") == null && System.getProperty("solr.hdfs.home") == null) {
    nodeProperties.setProperty("solr.data.dir", createTempDir().toFile().getCanonicalPath());
  }
  jetty = new JettySolrRunner(homeDir.getAbsolutePath(), nodeProperties, buildJettyConfig("/solr"));

  jetty.start();
  port = jetty.getLocalPort();

  //createJetty(homeDir.getAbsolutePath(), null, null);
}
 

public void testCreateCollectionAddShardWithReplicaTypeUsingPolicy() throws Exception {
  JettySolrRunner jetty = cluster.getJettySolrRunners().get(0);
  String nrtNodeName = jetty.getNodeName();
  int nrtPort = jetty.getLocalPort();

  jetty = cluster.getJettySolrRunners().get(1);
  String pullNodeName = jetty.getNodeName();
  int pullPort = jetty.getLocalPort();

  jetty = cluster.getJettySolrRunners().get(2);
  String tlogNodeName = jetty.getNodeName();
  int tlogPort = jetty.getLocalPort();
  log.info("NRT {} PULL {} , TLOG {} ", nrtNodeName, pullNodeName, tlogNodeName);

  String commands = "{set-cluster-policy :[" +
      "{replica:0 , shard:'#EACH', type: NRT, port: '!" + nrtPort + "'}" +
      "{replica:0 , shard:'#EACH', type: PULL, port: '!" + pullPort + "'}" +
      "{replica:0 , shard:'#EACH', type: TLOG, port: '!" + tlogPort + "'}" +
      "]}";


  cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
  Map<String, Object> json = Utils.getJson(cluster.getZkClient(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, true);
  assertEquals("full json:" + Utils.toJSONString(json), "!" + nrtPort,
      Utils.getObjectByPath(json, true, "cluster-policy[0]/port"));
  assertEquals("full json:" + Utils.toJSONString(json), "!" + pullPort,
      Utils.getObjectByPath(json, true, "cluster-policy[1]/port"));
  assertEquals("full json:" + Utils.toJSONString(json), "!" + tlogPort,
      Utils.getObjectByPath(json, true, "cluster-policy[2]/port"));

  final String collectionName = "addshard_with_reptype_using_policy";
  CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "s1", 1, 1, 1)
      .setMaxShardsPerNode(-1)
      .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 1, 3);

  DocCollection coll = getCollectionState(collectionName);


  BiConsumer<String, Replica> verifyReplicas = (s, replica) -> {
    switch (replica.getType()) {
      case NRT: {
        assertTrue("NRT replica should be in " + nrtNodeName, replica.getNodeName().equals(nrtNodeName));
        break;
      }
      case TLOG: {
        assertTrue("TLOG replica should be in " + tlogNodeName, replica.getNodeName().equals(tlogNodeName));
        break;
      }
      case PULL: {
        assertTrue("PULL replica should be in " + pullNodeName, replica.getNodeName().equals(pullNodeName));
        break;
      }
    }

  };
  coll.forEachReplica(verifyReplicas);

  CollectionAdminRequest.createShard(collectionName, "s3").
      process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 2, 6);
  
  coll = getCollectionState(collectionName);
  assertEquals(3, coll.getSlice("s3").getReplicas().size());
  coll.forEachReplica(verifyReplicas);
}
 

/**
 * Updates the mappings between the jetty's instances and the zookeeper cluster state.
 */
protected void updateMappingsFromZk(String collection) throws Exception {
  List<CloudJettyRunner> cloudJettys = new ArrayList<>();
  Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
  Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();

  CloudSolrClient cloudClient = this.createCloudClient(null);
  try {
    cloudClient.connect();
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(collection);

    for (JettySolrRunner jetty : jettys) {
      int port = jetty.getLocalPort();
      if (port == -1) {
        throw new RuntimeException("Cannot find the port for jetty");
      }

      nextJetty:
      for (Slice shard : coll.getSlices()) {
        Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
        for (Map.Entry<String, Replica> entry : entries) {
          Replica replica = entry.getValue();
          if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
            if (!shardToJetty.containsKey(shard.getName())) {
              shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
            }
            boolean isLeader = shard.getLeader() == replica;
            CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
            shardToJetty.get(shard.getName()).add(cjr);
            if (isLeader) {
              shardToLeaderJetty.put(shard.getName(), cjr);
            }
            cloudJettys.add(cjr);
            break nextJetty;
          }
        }
      }
    }

    List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
    if (oldRunners != null)  {
      // must close resources for the old entries
      for (CloudJettyRunner oldRunner : oldRunners) {
        IOUtils.closeQuietly(oldRunner.client);
      }
    }

    this.cloudJettys.put(collection, cloudJettys);
    this.shardToJetty.put(collection, shardToJetty);
    this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
  } finally {
    cloudClient.close();
  }
}
 

@Test
public void test() throws Exception {
  cluster.waitForAllNodes(5);
  String coll = "utilizenodecoll";
  CloudSolrClient cloudClient = cluster.getSolrClient();
  
  log.info("Creating Collection...");
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 2)
      .setMaxShardsPerNode(2);
  cloudClient.request(create);

  log.info("Spinning up additional jettyX...");
  JettySolrRunner jettyX = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);

  assertNoReplicas("jettyX should not yet be utilized: ", coll, jettyX);

  if (log.isInfoEnabled()) {
    log.info("Sending UTILIZE command for jettyX ({})", jettyX.getNodeName());
  }
  cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyX.getNodeName()));

  // TODO: aparently we can't assert this? ...
  //
  // assertSomeReplicas("jettyX should now be utilized: ", coll, jettyX);
  //
  // ... it appears from the docs that unless there are policy violations,
  // this can be ignored unless jettyX has less "load" then other jetty instances?
  //
  // if the above is true, that means that this test is incredibly weak...
  // unless we know jettyX has at least one replica, then all the subsequent testing of the
  // port blacklist & additional UTILIZE command for jettyY are a waste of time.
  //
  // should we skip spinning up a *new* jettyX, and instead just pick an existing jetty?

  if (log.isInfoEnabled()) {
    log.info("jettyX replicas prior to being blacklisted: {}", getReplicaList(coll, jettyX));
  }
  
  String setClusterPolicyCommand = "{" +
    " 'set-cluster-policy': [" +
    "    {'port':" + jettyX.getLocalPort() +
    "     , 'replica':0}" +
    "  ]" +
    "}";
  if (log.isInfoEnabled()) {
    log.info("Setting new policy to blacklist jettyX ({}) port={}",
        jettyX.getNodeName(), jettyX.getLocalPort());
  }
  @SuppressWarnings({"rawtypes"})
  SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
  NamedList<Object> response = cloudClient.request(req);
  assertEquals(req + " => " + response,
               "success", response.get("result").toString());

  log.info("Spinning up additional jettyY...");
  JettySolrRunner jettyY = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  
  assertNoReplicas("jettyY should not yet be utilized: ", coll, jettyY);
  if (log.isInfoEnabled()) {
    log.info("jettyX replicas prior to utilizing jettyY: {}", getReplicaList(coll, jettyX));
    log.info("Sending UTILIZE command for jettyY ({})", jettyY.getNodeName()); // logOk
  }
  cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyY.getNodeName()));

  assertSomeReplicas("jettyY should now be utilized: ", coll, jettyY);
}