Java Code Examples for org.apache.solr.common.cloud.Replica#Type

private Suggester getNodeAddedSuggester(SolrCloudManager cloudManager, Policy.Session session, TriggerEvent event) throws IOException {
  String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
  Replica.Type replicaType = (Replica.Type) event.getProperty(AutoScalingParams.REPLICA_TYPE, Replica.Type.NRT);
  CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);

  Suggester suggester = session.getSuggester(action)
      .hint(Suggester.Hint.TARGET_NODE, event.getProperty(NODE_NAMES));
  switch (action) {
    case ADDREPLICA:
      // add all collection/shard pairs and let policy engine figure out which one
      // to place on the target node
      ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
      Set<Pair<String, String>> collShards = new HashSet<>();
      clusterState.getCollectionStates().entrySet().stream()
              .filter(e -> collectionsPredicate.test(e.getKey()))
              .forEach(entry -> {
                DocCollection docCollection = entry.getValue().get();
                if (docCollection != null) {
                  docCollection.getActiveSlices().stream()
                          .map(slice -> new Pair<>(entry.getKey(), slice.getName()))
                          .forEach(collShards::add);
                }
              });
      suggester.hint(Suggester.Hint.COLL_SHARD, collShards);
      suggester.hint(Suggester.Hint.REPLICATYPE, replicaType);
      break;
    case MOVEREPLICA:
    case NONE:
      break;
    default:
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
          "Unsupported preferredOperation=" + preferredOp + " for node added event");
  }
  return suggester;
}
 

private ImmutableMap<Replica.Type, Integer> countsPerReplicaType(AssignRequest assignRequest) {
  return ImmutableMap.of(
      Replica.Type.NRT, assignRequest.numNrtReplicas,
      Replica.Type.TLOG, assignRequest.numTlogReplicas,
      Replica.Type.PULL, assignRequest.numPullReplicas
  );
}
 

private void queryReplicaType(CloudSolrClient cloudClient,
                                        Replica.Type typeToQuery,
                                        String collectionName)
    throws Exception
{
  SolrQuery qRequest = new SolrQuery("*:*");

  ModifiableSolrParams qParams = new ModifiableSolrParams();
  qParams.add(ShardParams.SHARDS_PREFERENCE, ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":" + typeToQuery.toString());
  qParams.add(ShardParams.SHARDS_INFO, "true");
  qRequest.add(qParams);

  Map<String, String> replicaTypeToReplicas = mapReplicasToReplicaType(getCollectionState(collectionName));

  QueryResponse qResponse = cloudClient.query(collectionName, qRequest);

  Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("Unable to obtain "+ShardParams.SHARDS_INFO, shardsInfo);

  // Iterate over shards-info and check what cores responded
  SimpleOrderedMap<?> shardsInfoMap = (SimpleOrderedMap<?>)shardsInfo;
  @SuppressWarnings({"unchecked"})
  Iterator<Map.Entry<String, ?>> itr = shardsInfoMap.asMap(100).entrySet().iterator();
  List<String> shardAddresses = new ArrayList<String>();
  while (itr.hasNext()) {
    Map.Entry<String, ?> e = itr.next();
    assertTrue("Did not find map-type value in "+ShardParams.SHARDS_INFO, e.getValue() instanceof Map);
    String shardAddress = (String)((Map)e.getValue()).get("shardAddress");
    if (shardAddress.endsWith("/")) {
      shardAddress = shardAddress.substring(0, shardAddress.length() - 1);
    }
    assertNotNull(ShardParams.SHARDS_INFO+" did not return 'shardAddress' parameter", shardAddress);
    shardAddresses.add(shardAddress);
  }
  assertEquals("Shard addresses must be of size 1, since there is only 1 shard in the collection", 1, shardAddresses.size());

  assertEquals("Make sure that the replica queried was the replicaType desired", typeToQuery.toString().toUpperCase(Locale.ROOT), replicaTypeToReplicas.get(shardAddresses.get(0)).toUpperCase(Locale.ROOT));
}
 

private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {
  switch (random().nextInt(3)) {
    case 0: // Add replica with SolrJ
      CollectionAdminResponse response = CollectionAdminRequest.addReplicaToShard(collectionName, shardName, type).process(cluster.getSolrClient());
      assertEquals("Unexpected response status: " + response.getStatus(), 0, response.getStatus());
      break;
    case 1: // Add replica with V1 API
      String url = String.format(Locale.ROOT, "%s/admin/collections?action=ADDREPLICA&collection=%s&shard=%s&type=%s",
          cluster.getRandomJetty(random()).getBaseUrl(),
          collectionName,
          shardName,
          type);
      HttpGet addReplicaGet = new HttpGet(url);
      HttpResponse httpResponse = cluster.getSolrClient().getHttpClient().execute(addReplicaGet);
      assertEquals(200, httpResponse.getStatusLine().getStatusCode());
      break;
    case 2:// Add replica with V2 API
      url = String.format(Locale.ROOT, "%s/____v2/c/%s/shards",
          cluster.getRandomJetty(random()).getBaseUrl(),
          collectionName);
      String requestBody = String.format(Locale.ROOT, "{add-replica:{shard:%s, type:%s}}",
          shardName,
          type);
      HttpPost addReplicaPost = new HttpPost(url);
      addReplicaPost.setHeader("Content-type", "application/json");
      addReplicaPost.setEntity(new StringEntity(requestBody));
      httpResponse = cluster.getSolrClient().getHttpClient().execute(addReplicaPost);
      assertEquals(200, httpResponse.getStatusLine().getStatusCode());
      break;
  }
}
 

public Row removeReplica(String coll, String shard, Replica.Type type, int recursionCount) {
  if (recursionCount > 3) {
    log.error("more than 3 levels of recursion ", new RuntimeException());
    return this;
  }
  List<OperationInfo> furtherOps = new LinkedList<>();
  Consumer<OperationInfo> opCollector = it -> furtherOps.add(it);
  Row row = session.copy().getNode(this.node);
  row.lazyCopyReplicas(coll, shard);
  Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.get(coll);
  if (c == null) return null;
  List<ReplicaInfo> r = c.get(shard);
  if (r == null) return null;
  int idx = -1;
  for (int i = 0; i < r.size(); i++) {
    ReplicaInfo info = r.get(i);
    if (type == null || info.getType() == type) {
      idx = i;
      break;
    }
  }
  if (idx == -1) return null;
  ReplicaInfo removed = r.remove(idx);
  for (Cell cell : row.cells) {
    cell.type.projectRemoveReplica(cell, removed, opCollector);
  }
  return row;

}
 

/**
 * Creates a JettySolrRunner with a socket proxy sitting infront of the Jetty server,
 * which gives us the ability to simulate network partitions without having to fuss
 * with IPTables.
 */
public JettySolrRunner createProxiedJetty(File solrHome, String dataDir,
                                   String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType)
    throws Exception {

  JettyConfig jettyconfig = JettyConfig.builder()
      .setContext(context)
      .stopAtShutdown(false)
      .withServlets(getExtraServlets())
      .withFilters(getExtraRequestFilters())
      .withSSLConfig(sslConfig.buildServerSSLConfig())
      .build();

  Properties props = new Properties();
  if (solrConfigOverride != null)
    props.setProperty("solrconfig", solrConfigOverride);
  if (schemaOverride != null)
    props.setProperty("schema", schemaOverride);
  if (shardList != null)
    props.setProperty("shards", shardList);
  if (dataDir != null)
    props.setProperty("solr.data.dir", getDataDir(dataDir));
  if (replicaType != null) {
    props.setProperty("replicaType", replicaType.toString());
  } else if (random().nextBoolean()) {
    props.setProperty("replicaType", Replica.Type.NRT.toString());
  }
  props.setProperty("coreRootDirectory", solrHome.toPath().resolve("cores").toString());

  JettySolrRunner jetty = new JettySolrRunner(solrHome.getPath(), props, jettyconfig, true);

  return jetty;
}
 

/**
 * this simulates adding a replica of a certain coll+shard to node. as a result of adding a replica ,
 * values of certain attributes will be modified, in this node as well as other nodes. Please note that
 * the state of the current session is kept intact while this operation is being performed
 *
 * @param coll           collection name
 * @param shard          shard name
 * @param type           replica type
 * @param recursionCount the number of times we have recursed to add more replicas
 * @param strictMode     whether suggester is operating in strict mode or not
 */
Row addReplica(String coll, String shard, Replica.Type type, int recursionCount, boolean strictMode) {
  if (recursionCount > 3) {
    log.error("more than 3 levels of recursion ", new RuntimeException());
    return this;
  }
  lazyCopyReplicas(coll, shard);
  List<OperationInfo> furtherOps = new LinkedList<>();
  Consumer<OperationInfo> opCollector = it -> furtherOps.add(it);
  Row row = null;
  row = session.copy().getNode(this.node);
  if (row == null) throw new RuntimeException("couldn't get a row");
  row.lazyCopyReplicas(coll, shard);
  Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
  List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
  String replicaname = "SYNTHETIC." + new Random().nextInt(1000) + 1000;
  ReplicaInfo ri = new ReplicaInfo(replicaname, replicaname, coll, shard, type, this.node,
      Utils.makeMap(ZkStateReader.REPLICA_TYPE, type != null ? type.toString() : Replica.Type.NRT.toString()));
  replicas.add(ri);
  for (Cell cell : row.cells) {
    cell.type.projectAddReplica(cell, ri, opCollector, strictMode);
  }
  for (OperationInfo op : furtherOps) {
    if (op.isAdd) {
      row = row.session.getNode(op.node).addReplica(op.coll, op.shard, op.type, recursionCount + 1, strictMode);
    } else {
      row.session.getNode(op.node).removeReplica(op.coll, op.shard, op.type, recursionCount + 1);
    }
  }

  return row;
}
 

/**
 * Overrides the parent implementation to install a SocketProxy in-front of the Jetty server.
 */
@Override
public JettySolrRunner createJetty(File solrHome, String dataDir,
                                   String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType)
    throws Exception {
  return createProxiedJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, replicaType);
}
 

/**
 * Overrides the parent implementation to install a SocketProxy in-front of the Jetty server.
 */
@Override
public JettySolrRunner createJetty(File solrHome, String dataDir,
                                   String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType)
    throws Exception
{
  return createProxiedJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, replicaType);
}
 

public void increment(Replica.Type type) {
  switch (type) {
    case NRT:
      nrt++;
      break;
    case PULL:
      pull++;
      break;
    case TLOG:
      tlog++;
      break;
    default:
      nrt++;
  }
}
 

/**
 * Overrides the parent implementation to install a SocketProxy in-front of the Jetty server.
 */
@Override
public JettySolrRunner createJetty(File solrHome, String dataDir,
    String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType)
    throws Exception
{
  return createProxiedJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, replicaType);
}
 

public static CreateReplica assignReplicaDetails(SolrCloudManager cloudManager, ClusterState clusterState,
                                               ZkNodeProps message, ReplicaPosition replicaPosition) {
  boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);

  String collection = message.getStr(COLLECTION_PROP);
  String node = replicaPosition.node;
  String shard = message.getStr(SHARD_ID_PROP);
  String coreName = message.getStr(CoreAdminParams.NAME);
  String coreNodeName = message.getStr(CoreAdminParams.CORE_NODE_NAME);
  Replica.Type replicaType = replicaPosition.type;

  if (StringUtils.isBlank(coreName)) {
    coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
  }

  log.info("Node Identified {} for creating new replica of shard {} for collection {}", node, shard, collection);
  if (!clusterState.liveNodesContain(node)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
  }
  DocCollection coll = clusterState.getCollection(collection);
  if (coreName == null) {
    coreName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), coll, shard, replicaType);
  } else if (!skipCreateReplicaInClusterState) {
    //Validate that the core name is unique in that collection
    for (Slice slice : coll.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        String replicaCoreName = replica.getStr(CORE_NAME_PROP);
        if (coreName.equals(replicaCoreName)) {
          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" +
              " for this collection");
        }
      }
    }
  }
  log.info("Returning CreateReplica command.");
  return new CreateReplica(collection, shard, node, replicaType, coreName, coreNodeName);
}
 

public Row addReplica(String coll, String shard, Replica.Type type) {
  return addReplica(coll, shard, type, 0, true);
}
 

public int delta(int expectedReplicaCount, Replica.Type type) {
  if (type == Replica.Type.NRT) return (int) (nrt - expectedReplicaCount);
  if (type == Replica.Type.PULL) return (int) (pull - expectedReplicaCount);
  if (type == Replica.Type.TLOG) return (int) (tlog - expectedReplicaCount);
  throw new RuntimeException("NO type");
}
 

private static Map<String, Object> makeReplicaProps(String sliceName, String node, String replicaName, String stateCode, String replicaTypeCode) {
  if (node == null || node.trim().length() == 0) {
    node = "1";
  }

  Replica.State state = Replica.State.ACTIVE;
  if (stateCode != null) {
    switch (stateCode.charAt(0)) {
      case 'S':
        state = Replica.State.ACTIVE;
        break;
      case 'R':
        state = Replica.State.RECOVERING;
        break;
      case 'D':
        state = Replica.State.DOWN;
        break;
      case 'F':
        state = Replica.State.RECOVERY_FAILED;
        break;
      default:
        throw new IllegalArgumentException(
            "Unexpected state for replica: " + stateCode);
    }
  }

  Replica.Type replicaType = Replica.Type.NRT;
  switch (replicaTypeCode)  {
    case "t":
      replicaType = Replica.Type.TLOG;
      break;
    case "p":
      replicaType = Replica.Type.PULL;
      break;
  }

  Map<String,Object> replicaPropMap = new HashMap<>();
  replicaPropMap.put(ZkStateReader.NODE_NAME_PROP, "baseUrl" + node + "_");
  replicaPropMap.put(ZkStateReader.BASE_URL_PROP, "http://baseUrl" + node);
  replicaPropMap.put(ZkStateReader.STATE_PROP, state.toString());
  replicaPropMap.put(ZkStateReader.CORE_NAME_PROP, sliceName + "_" + replicaName);
  replicaPropMap.put(ZkStateReader.REPLICA_TYPE, replicaType.name());
  return replicaPropMap;
}
 

public static List<ReplicaPosition> buildReplicaPositions(SolrCloudManager cloudManager, ClusterState clusterState,
                                                          String collectionName, ZkNodeProps message,
                                                          EnumMap<Replica.Type, Integer> replicaTypeVsCount,
                                                          AtomicReference< PolicyHelper.SessionWrapper> sessionWrapper) throws IOException, InterruptedException {
  boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);
  boolean skipNodeAssignment = message.getBool(SKIP_NODE_ASSIGNMENT, false);
  String sliceName = message.getStr(SHARD_ID_PROP);
  DocCollection collection = clusterState.getCollection(collectionName);

  int numNrtReplicas = replicaTypeVsCount.get(Replica.Type.NRT);
  int numPullReplicas = replicaTypeVsCount.get(Replica.Type.PULL);
  int numTlogReplicas = replicaTypeVsCount.get(Replica.Type.TLOG);
  int totalReplicas = numNrtReplicas + numPullReplicas + numTlogReplicas;

  String node = message.getStr(CoreAdminParams.NODE);
  Object createNodeSetStr = message.get(OverseerCollectionMessageHandler.CREATE_NODE_SET);
  if (createNodeSetStr == null) {
    if (node != null) {
      message.getProperties().put(OverseerCollectionMessageHandler.CREATE_NODE_SET, node);
      createNodeSetStr = node;
    }
  }

  List<ReplicaPosition> positions = null;
  if (!skipCreateReplicaInClusterState && !skipNodeAssignment) {

    positions = Assign.getNodesForNewReplicas(clusterState, collection.getName(), sliceName, numNrtReplicas,
                  numTlogReplicas, numPullReplicas, createNodeSetStr, cloudManager);
    sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
  }

  if (positions == null)  {
    assert node != null;
    if (node == null) {
      // in case asserts are disabled
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
          "A node should have been identified to add replica but wasn't. Please inform solr developers at SOLR-9317");
    }
    // it is unlikely that multiple replicas have been requested to be created on
    // the same node, but we've got to accommodate.
    positions = new ArrayList<>(totalReplicas);
    int i = 0;
    for (Map.Entry<Replica.Type, Integer> entry : replicaTypeVsCount.entrySet()) {
      for (int j = 0; j < entry.getValue(); j++) {
        positions.add(new ReplicaPosition(sliceName, i++, entry.getKey(), node));
      }
    }
  }
  return positions;
}
 

public static String buildSolrCoreName(String collectionName, String shard, Replica.Type type, int replicaNum) {
  // TODO: Adding the suffix is great for debugging, but may be an issue if at some point we want to support a way to change replica type
  return String.format(Locale.ROOT, "%s_%s_replica_%s%s", collectionName, shard, type.name().substring(0,1).toLowerCase(Locale.ROOT), replicaNum);
}
 

private int compareTypes(Replica.Type t1, Replica.Type t2) {
  if (t1 == null && t2 == null) return 0;
  if (t1 != null && t2 == null) return -1;
  if (t1 == null) return 1;
  return 0;
}
 

@Override
protected Replica.Type computeReplicaType() {
  // can't use cloudDesc since this is called by super class, before the constructor instantiates cloudDesc.
  return req.getCore().getCoreDescriptor().getCloudDescriptor().getReplicaType();
}
 

/**
 *
 * @return the replica type of the collection.
 */
protected Replica.Type computeReplicaType() {
  return Replica.Type.NRT;
}