Java Code Examples for org.apache.solr.common.cloud.Replica#getStr()

protected SocketProxy getProxyForReplica(Replica replica) throws Exception {
  String replicaBaseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
  assertNotNull(replicaBaseUrl);

  List<JettySolrRunner> runners = new ArrayList<>(jettys);
  runners.add(controlJetty);
  
  for (JettySolrRunner j : runners) {
    if (replicaBaseUrl.replaceAll("/$", "").equals(j.getProxyBaseUrl().toExternalForm().replaceAll("/$", ""))) {
      return j.getProxy();
    }
  }
  
  printLayout();

  fail("No proxy found for " + replicaBaseUrl + "!");
  return null;
}
 

@Override
public String get() {
  CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor();
  if (cd == null) {
    return null;
  }
  ClusterState state = core.getCoreContainer().getZkController().getClusterState();
  DocCollection collection = state.getCollection(core.getCoreDescriptor().getCollectionName());
  Replica replica = collection.getLeader(core.getCoreDescriptor().getCloudDescriptor().getShardId());
  if (replica == null) {
    log.warn("No leader for {}/{}", collection.getName(), core.getCoreDescriptor().getCloudDescriptor().getShardId());
    return null;
  }
  String baseUrl = replica.getStr("base_url");
  if (baseUrl == null) {
    log.warn("No base_url for replica {}", replica);
  }
  return baseUrl;
}
 

public static String getAssignedCoreNodeName(DocCollection collection, String forNodeName, String forCoreName) {
  Collection<Slice> slices = collection != null ? collection.getSlices() : null;
  if (slices != null) {
    for (Slice slice : slices) {
      for (Replica replica : slice.getReplicas()) {
        String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
        String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);

        if (nodeName.equals(forNodeName) && core.equals(forCoreName)) {
          return replica.getName();
        }
      }
    }
  }
  return null;
}
 

private List<Replica> getReplacedSharedFsReplicas(String collection, ZkStateReader zkStateReader, String lostNodeName) {
  List<Replica> replacedHdfsReplicas = new ArrayList<>();
  for (Replica replica : zkStateReader.getClusterState().getCollection(collection).getReplicas()) {
    String dataDir = replica.getStr("dataDir");
    if (replica.getNodeName().equals(lostNodeName) && dataDir != null) {
      replacedHdfsReplicas.add(replica);
    }
  }

  return replacedHdfsReplicas;
}
 

private String getRandomBaseUrl(Collection<Slice> slices) {
  String coreUrl = null;
  if (slices != null) {
    for (Slice slice : slices) {
      if (!slice.getReplicas().isEmpty()) {
        Replica replica = slice.getReplicas().iterator().next();
        coreUrl = replica.getStr("base_url");
        if (coreUrl != null) {
          break;
        }
      }
    }
  }
  return coreUrl;
}
 

private void checkSharedFsReplicasMovedCorrectly(List<Replica> replacedHdfsReplicas, ZkStateReader zkStateReader, String collection){
  DocCollection docCollection = zkStateReader.getClusterState().getCollection(collection);
  for (Replica replica :replacedHdfsReplicas) {
    boolean found = false;
    String dataDir = replica.getStr("dataDir");
    String ulogDir = replica.getStr("ulogDir");
    for (Replica replica2 : docCollection.getReplicas()) {
      if (dataDir.equals(replica2.getStr("dataDir")) && ulogDir.equals(replica2.getStr("ulogDir"))) {
        found = true;
        break;
      }
    }
    if (!found) fail("Can not found a replica with same dataDir and ulogDir as " + replica + " from:" + docCollection.getReplicas());
  }
}
 

protected JettySolrRunner getJettyForReplica(Replica replica) throws Exception {
  String replicaBaseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
  assertNotNull(replicaBaseUrl);
  URL baseUrl = new URL(replicaBaseUrl);

  JettySolrRunner proxy = jettys.get(baseUrl.toURI());
  assertNotNull("No proxy found for " + baseUrl + "!", proxy);
  return proxy;
}
 

@Test
public void doBlobHandlerTest() throws Exception {

  try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
    CollectionAdminResponse response1;
    CollectionAdminRequest.Create createCollectionRequest = CollectionAdminRequest.createCollection(".system",1,2);
    response1 = createCollectionRequest.process(client);
    assertEquals(0, response1.getStatus());
    assertTrue(response1.isSuccess());
    DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
    Replica replica = sysColl.getActiveSlicesMap().values().iterator().next().getLeader();

    String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
    String url = baseUrl + "/.system/config/requestHandler";
    MapWriter map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
    assertNotNull(map);
    assertEquals("solr.BlobHandler", map._get(asList(
        "config",
        "requestHandler",
        "/blob",
        "class"),null));
    map = TestSolrConfigHandlerConcurrent.getAsMap(baseUrl + "/.system/schema/fields/blob", cloudClient);
    assertNotNull(map);
    assertEquals("blob", map._get(asList(
        "field",
        "name"),null));
    assertEquals("bytes", map._get( asList(
        "field",
        "type"),null));

    checkBlobPost(baseUrl, cloudClient);
    checkBlobPostMd5(baseUrl, cloudClient);
  }
}
 

public ZkWriteCommand deleteReplicaProperty(ClusterState clusterState, ZkNodeProps message) {
  if (checkKeyExistence(message, ZkStateReader.COLLECTION_PROP) == false ||
      checkKeyExistence(message, ZkStateReader.SHARD_ID_PROP) == false ||
      checkKeyExistence(message, ZkStateReader.REPLICA_PROP) == false ||
      checkKeyExistence(message, ZkStateReader.PROPERTY_PROP) == false) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        "Overseer DELETEREPLICAPROP requires " +
            ZkStateReader.COLLECTION_PROP + " and " + ZkStateReader.SHARD_ID_PROP + " and " +
            ZkStateReader.REPLICA_PROP + " and " + ZkStateReader.PROPERTY_PROP + " no action taken.");
  }
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
  String replicaName = message.getStr(ZkStateReader.REPLICA_PROP);
  String property = message.getStr(ZkStateReader.PROPERTY_PROP).toLowerCase(Locale.ROOT);
  if (StringUtils.startsWith(property, OverseerCollectionMessageHandler.COLL_PROP_PREFIX) == false) {
    property = OverseerCollectionMessageHandler.COLL_PROP_PREFIX + property;
  }

  DocCollection collection = clusterState.getCollection(collectionName);
  Replica replica = collection.getReplica(replicaName);

  if (replica == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection/slice/replica " +
        collectionName + "/" + sliceName + "/" + replicaName + " no action taken.");
  }

  log.info("Deleting property {} for collection: {} slice: {} replica: {}", property, collectionName, sliceName, replicaName);
  log.debug("Full message: {}", message);
  String curProp = replica.getStr(property);
  if (curProp == null) return ZkStateWriter.NO_OP; // not there anyway, nothing to do.

  Slice slice = collection.getSlice(sliceName);
  DocCollection newCollection = SliceMutator.updateReplica(collection,
      slice, replicaName, unsetProperty(replica, property));
  return new ZkWriteCommand(collectionName, newCollection);
}
 

public static CreateReplica assignReplicaDetails(SolrCloudManager cloudManager, ClusterState clusterState,
                                               ZkNodeProps message, ReplicaPosition replicaPosition) {
  boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);

  String collection = message.getStr(COLLECTION_PROP);
  String node = replicaPosition.node;
  String shard = message.getStr(SHARD_ID_PROP);
  String coreName = message.getStr(CoreAdminParams.NAME);
  String coreNodeName = message.getStr(CoreAdminParams.CORE_NODE_NAME);
  Replica.Type replicaType = replicaPosition.type;

  if (StringUtils.isBlank(coreName)) {
    coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
  }

  log.info("Node Identified {} for creating new replica of shard {} for collection {}", node, shard, collection);
  if (!clusterState.liveNodesContain(node)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
  }
  DocCollection coll = clusterState.getCollection(collection);
  if (coreName == null) {
    coreName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), coll, shard, replicaType);
  } else if (!skipCreateReplicaInClusterState) {
    //Validate that the core name is unique in that collection
    for (Slice slice : coll.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        String replicaCoreName = replica.getStr(CORE_NAME_PROP);
        if (coreName.equals(replicaCoreName)) {
          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" +
              " for this collection");
        }
      }
    }
  }
  log.info("Returning CreateReplica command.");
  return new CreateReplica(collection, shard, node, replicaType, coreName, coreNodeName);
}
 

@Override
public boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
  if (collectionState == null) { // collection has been deleted - don't wait
    latch.countDown();
    return true;
  }
  Slice slice = collectionState.getSlice(shardId);
  if (slice == null) { // shard has been removed - don't wait
    latch.countDown();
    return true;
  }
  for (Replica replica : slice.getReplicas()) {
    // check if another replica exists - doesn't have to be the one we're moving
    // as long as it's active and can become a leader, in which case we don't have to wait
    // for recovery of specifically the one that we've just added
    if (!replica.getName().equals(replicaId)) {
      if (replica.getType().equals(Replica.Type.PULL)) { // not eligible for leader election
        continue;
      }
      // check its state
      String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
      if (targetCore != null && !targetCore.equals(coreName)) {
        continue;
      }
      if (replica.isActive(liveNodes)) { // recovered - stop waiting
        latch.countDown();
        return true;
      }
    }
  }
  // set the watch again to wait for the new replica to recover
  return false;
}
 

void deleteCoreNode(String collectionName, String replicaName, Replica replica, String core) throws Exception {
  ZkNodeProps m = new ZkNodeProps(
      Overseer.QUEUE_OPERATION, OverseerAction.DELETECORE.toLower(),
      ZkStateReader.CORE_NAME_PROP, core,
      ZkStateReader.NODE_NAME_PROP, replica.getStr(ZkStateReader.NODE_NAME_PROP),
      ZkStateReader.COLLECTION_PROP, collectionName,
      ZkStateReader.CORE_NODE_NAME_PROP, replicaName,
      ZkStateReader.BASE_URL_PROP, replica.getStr(ZkStateReader.BASE_URL_PROP));
  overseer.offerStateUpdate(Utils.toJSON(m));
}
 

private void waitForCoreNodeName(CoreDescriptor descriptor) {
  int retryCount = 320;
  log.debug("look for our core node name");
  while (retryCount-- > 0) {
    final DocCollection docCollection = zkStateReader.getClusterState()
        .getCollectionOrNull(descriptor.getCloudDescriptor().getCollectionName());
    if (docCollection != null && docCollection.getSlicesMap() != null) {
      final Map<String, Slice> slicesMap = docCollection.getSlicesMap();
      for (Slice slice : slicesMap.values()) {
        for (Replica replica : slice.getReplicas()) {
          // TODO: for really large clusters, we could 'index' on this

          String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
          String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);

          String msgNodeName = getNodeName();
          String msgCore = descriptor.getName();

          if (msgNodeName.equals(nodeName) && core.equals(msgCore)) {
            descriptor.getCloudDescriptor()
                .setCoreNodeName(replica.getName());
            getCoreContainer().getCoresLocator().persist(getCoreContainer(), descriptor);
            return;
          }
        }
      }
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }
  }
}
 

private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {
  String fromReplica = null;

  String nodeName = zkController.getNodeName();
  for (Slice slice : zkController.getClusterState().getCollection(fromIndex).getActiveSlicesArr()) {
    if (fromReplica != null)
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "SolrCloud join: To join with a sharded collection, use method=crossCollection.");

    for (Replica replica : slice.getReplicas()) {
      if (replica.getNodeName().equals(nodeName)) {
        fromReplica = replica.getStr(ZkStateReader.CORE_NAME_PROP);
        // found local replica, but is it Active?
        if (replica.getState() != Replica.State.ACTIVE)
          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
              "SolrCloud join: "+fromIndex+" has a local replica ("+fromReplica+
                  ") on "+nodeName+", but it is "+replica.getState());

        break;
      }
    }
  }

  if (fromReplica == null)
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        "SolrCloud join: To join with a collection that might not be co-located, use method=crossCollection.");

  return fromReplica;
}
 

private String getCoreUrl(String collectionName,
                          String origCorename, ClusterState clusterState, List<Slice> slices,
                          boolean byCoreName, boolean activeReplicas) {
  String coreUrl;
  Set<String> liveNodes = clusterState.getLiveNodes();
  Collections.shuffle(slices, random);

  for (Slice slice : slices) {
    List<Replica> randomizedReplicas = new ArrayList<>(slice.getReplicas());
    Collections.shuffle(randomizedReplicas, random);

    for (Replica replica : randomizedReplicas) {
      if (!activeReplicas || (liveNodes.contains(replica.getNodeName())
          && replica.getState() == Replica.State.ACTIVE)) {

        if (byCoreName && !origCorename.equals(replica.getStr(CORE_NAME_PROP))) {
          // if it's by core name, make sure they match
          continue;
        }
        if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
          // don't count a local core
          continue;
        }

        if (origCorename != null) {
          coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
        } else {
          coreUrl = replica.getCoreUrl();
          if (coreUrl.endsWith("/")) {
            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
          }
        }

        return coreUrl;
      }
    }
  }
  return null;
}
 

/**
 * See if coreNodeName has been taken over by another baseUrl and unload core
 * + throw exception if it has been.
 */
public static void checkSharedFSFailoverReplaced(CoreContainer cc, CoreDescriptor desc) {
  if (!cc.isSharedFs(desc)) return;

  ZkController zkController = cc.getZkController();
  String thisCnn = zkController.getCoreNodeName(desc);
  String thisBaseUrl = zkController.getBaseUrl();

  log.debug("checkSharedFSFailoverReplaced running for coreNodeName={} baseUrl={}", thisCnn, thisBaseUrl);

  // if we see our core node name on a different base url, unload
  final DocCollection docCollection = zkController.getClusterState().getCollectionOrNull(desc.getCloudDescriptor().getCollectionName());
  if (docCollection != null && docCollection.getSlicesMap() != null) {
    Map<String,Slice> slicesMap = docCollection.getSlicesMap();
    for (Slice slice : slicesMap.values()) {
      for (Replica replica : slice.getReplicas()) {

        String cnn = replica.getName();
        String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
        log.debug("compare against coreNodeName={} baseUrl={}", cnn, baseUrl);

        if (thisCnn != null && thisCnn.equals(cnn)
            && !thisBaseUrl.equals(baseUrl)) {
          if (cc.getLoadedCoreNames().contains(desc.getName())) {
            cc.unload(desc.getName());
          }

          try {
            FileUtils.deleteDirectory(desc.getInstanceDir().toFile());
          } catch (IOException e) {
            SolrException.log(log, "Failed to delete instance dir for core:"
                + desc.getName() + " dir:" + desc.getInstanceDir());
          }
          log.error("{}",
              new SolrException(ErrorCode.SERVER_ERROR, "Will not load SolrCore " + desc.getName()
                  + " because it has been replaced due to failover.")); // logOk
          throw new SolrException(ErrorCode.SERVER_ERROR,
              "Will not load SolrCore " + desc.getName()
                  + " because it has been replaced due to failover.");
        }
      }
    }
  }
}
 

protected static void ensureAllReplicasAreActive(String testCollectionName, int shards, int rf, int maxWaitSecs)
    throws Exception {
  long startMs = System.currentTimeMillis();

  ZkStateReader zkr = cloudSolrClient.getZkStateReader();
  zkr.updateClusterState(); // force the state to be fresh

  ClusterState cs = zkr.getClusterState();
  Collection<Slice> slices = cs.getActiveSlices(testCollectionName);
  assertTrue(slices.size() == shards);
  boolean allReplicasUp = false;
  long waitMs = 0L;
  long maxWaitMs = maxWaitSecs * 1000L;
  Replica leader = null;
  while (waitMs < maxWaitMs && !allReplicasUp) {
    // refresh state every 2 secs
    if (waitMs % 2000 == 0) {
      log.info("Updating ClusterState");
      cloudSolrClient.getZkStateReader().updateClusterState();
    }

    cs = cloudSolrClient.getZkStateReader().getClusterState();
    assertNotNull(cs);
    allReplicasUp = true; // assume true
    for (Slice shard : cs.getActiveSlices(testCollectionName)) {
      String shardId = shard.getName();
      assertNotNull("No Slice for " + shardId, shard);
      Collection<Replica> replicas = shard.getReplicas();
      assertTrue(replicas.size() == rf);
      leader = shard.getLeader();
      assertNotNull(leader);
      log.info("Found " + replicas.size() + " replicas and leader on " +
          leader.getNodeName() + " for " + shardId + " in " + testCollectionName);

      // ensure all replicas are "active"
      for (Replica replica : replicas) {
        String replicaState = replica.getStr(ZkStateReader.STATE_PROP);
        if (!"active".equals(replicaState)) {
          log.info("Replica " + replica.getName() + " for shard " + shardId + " is currently " + replicaState);
          allReplicasUp = false;
        }
      }
    }

    if (!allReplicasUp) {
      try {
        Thread.sleep(500L);
      } catch (Exception ignoreMe) {
      }
      waitMs += 500L;
    }
  } // end while

  if (!allReplicasUp)
    fail("Didn't see all replicas for " + testCollectionName +
        " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo(testCollectionName));

  long diffMs = (System.currentTimeMillis() - startMs);
  log.info("Took " + diffMs + " ms to see all replicas become active for " + testCollectionName);
}
 

/**
 * Package local for unit tests only please do not use elsewhere
 */
ByteBuffer fetchBlob(String key) {
  Replica replica = getSystemCollReplica();
  String url = replica.getStr(BASE_URL_PROP) + "/" + CollectionAdminParams.SYSTEM_COLL + "/blob/" + key + "?wt=filestream";
  return fetchFromUrl(key, url);
}
 

@SuppressWarnings({"unchecked"})
private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception {
  String backupName = request.getStr(NAME);
  String asyncId = request.getStr(ASYNC);
  String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
  ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(ocmh.overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient());

  String commitName = request.getStr(CoreAdminParams.COMMIT_NAME);
  Optional<CollectionSnapshotMetaData> snapshotMeta = Optional.empty();
  if (commitName != null) {
    SolrZkClient zkClient = ocmh.zkStateReader.getZkClient();
    snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
    if (!snapshotMeta.isPresent()) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
          + " does not exist for collection " + collectionName);
    }
    if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName
          + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
    }
  }

  log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
      backupPath);

  Collection<String> shardsToConsider = Collections.emptySet();
  if (snapshotMeta.isPresent()) {
    shardsToConsider = snapshotMeta.get().getShards();
  }

  final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
  for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
    Replica replica = null;

    if (snapshotMeta.isPresent()) {
      if (!shardsToConsider.contains(slice.getName())) {
        log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.",
            slice.getName(), collectionName, snapshotMeta.get().getName());
        continue;
      }
      replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
    } else {
      // Note - Actually this can return a null value when there is no leader for this shard.
      replica = slice.getLeader();
      if (replica == null) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
      }
    }

    String coreName = replica.getStr(CORE_NAME_PROP);

    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
    params.set(NAME, slice.getName());
    params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
    params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
    params.set(CORE_NAME_PROP, coreName);
    if (snapshotMeta.isPresent()) {
      params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
    }

    shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);
    log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
  }
  log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);

  shardRequestTracker.processResponses(results, shardHandler, true, "Could not backup all shards");
}
 

static ReplicaInfo create(Replica replica) {
  return new AutoValue_SolrIO_ReplicaInfo(
      replica.getStr(ZkStateReader.CORE_NAME_PROP),
      replica.getCoreUrl(),
      replica.getStr(ZkStateReader.BASE_URL_PROP));
}