Java Code Examples for org.apache.solr.client.solrj.request.CollectionAdminRequest#MoveReplica

The following examples show how to use org.apache.solr.client.solrj.request.CollectionAdminRequest#MoveReplica . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestPolicy.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testScheduledTriggerFailure() throws Exception {
  @SuppressWarnings({"rawtypes"})
  Map jsonObj = (Map) loadFromResource("testScheduledTriggerFailure.json");
  SolrCloudManager cloudManager = createCloudManager(jsonObj);
  Suggester suggester = createSuggester(cloudManager, jsonObj, null);
  int count = 0;
  while (count < 10) {
    CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion();
    if (op == null) break;
    count++;
    if (log.isInfoEnabled()) {
      log.info("OP:{}", op.getParams());
    }
    suggester = createSuggester(cloudManager, jsonObj, suggester);
  }

  assertEquals(0, count);
}
 
Example 2
Source File: TestPolicy.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testUtilizeNodeFailure() throws Exception {
  @SuppressWarnings({"rawtypes"})
  Map jsonObj = (Map) loadFromResource("testUtilizeNodeFailure.json"); //(Map) Utils.fromJSONString(state);
  SolrCloudManager cloudManager = createCloudManager(jsonObj);
  Suggester suggester = createSuggester(cloudManager, jsonObj, null);
  int count = 0;
  while (count < 100) {
    CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion();
    if (op == null) break;
    count++;
    if (log.isInfoEnabled()) {
      log.info("OP:{}", op.getParams());
    }
    suggester = createSuggester(cloudManager, jsonObj, suggester);
  }

  assertEquals("count = " + count, 0, count);
}
 
Example 3
Source File: TestPolicy.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testUtilizeNodeFailure2() throws Exception {
  @SuppressWarnings({"rawtypes"})
  Map jsonObj = (Map) loadFromResource("testUtilizeNodeFailure2.json");
  SolrCloudManager cloudManager = createCloudManager(jsonObj);
  Suggester suggester = createSuggester(cloudManager, jsonObj, null);
  int count = 0;
  while (count < 100) {
    CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion();
    if (op == null) break;
    count++;
    if (log.isInfoEnabled()) {
      log.info("OP:{}", op.getParams());
    }
    suggester = createSuggester(cloudManager, jsonObj, suggester);
  }

  assertEquals("count = " + count, 1, count);
}
 
Example 4
Source File: TestSnapshotCloudManager.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testSimulatorFromSnapshot() throws Exception {
  Path tmpPath = createTempDir();
  File tmpDir = tmpPath.toFile();
  SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(realManager, null);
  snapshotCloudManager.saveSnapshot(tmpDir, true, false);
  SnapshotCloudManager snapshotCloudManager1 = SnapshotCloudManager.readSnapshot(tmpDir);
  try (SimCloudManager simCloudManager = SimCloudManager.createCluster(snapshotCloudManager1, null, TimeSource.get("simTime:50"))) {
    SimSolrCloudTestCase.assertClusterStateEquals(snapshotCloudManager.getClusterStateProvider().getClusterState(), simCloudManager.getClusterStateProvider().getClusterState());
    assertNodeStateProvider(snapshotCloudManager, simCloudManager, "freedisk");
    assertDistribStateManager(snapshotCloudManager.getDistribStateManager(), simCloudManager.getDistribStateManager());
    ClusterState state = simCloudManager.getClusterStateProvider().getClusterState();
    Replica r = state.getCollection(CollectionAdminParams.SYSTEM_COLL).getReplicas().get(0);
    // get another node
    String target = null;
    for (String node : simCloudManager.getClusterStateProvider().getLiveNodes()) {
      if (!node.equals(r.getNodeName())) {
        target = node;
        break;
      }
    }
    if (target == null) {
      fail("can't find suitable target node for replica " + r + ", liveNodes=" + simCloudManager.getClusterStateProvider().getLiveNodes());
    }
    CollectionAdminRequest.MoveReplica moveReplica = CollectionAdminRequest
        .moveReplica(CollectionAdminParams.SYSTEM_COLL, r.getName(), target);
    log.info("################");
    simCloudManager.simGetSolrClient().request(moveReplica);
  }
}
 
Example 5
Source File: MoveReplicaSuggester.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({"rawtypes"})
SolrRequest tryEachNode(boolean strict) {
  //iterate through elements and identify the least loaded
  List<Violation> leastSeriousViolation = null;
  Row bestSrcRow = null;
  Row bestTargetRow = null;
  ReplicaInfo sourceReplicaInfo = null;
  List<Pair<ReplicaInfo, Row>> validReplicas = getValidReplicas(true, true, -1);
  validReplicas.sort(leaderLast);
  for (int i1 = 0; i1 < validReplicas.size(); i1++) {
    lastBestDeviation = null;
    Pair<ReplicaInfo, Row> fromReplica = validReplicas.get(i1);
    Row fromRow = fromReplica.second();
    ReplicaInfo ri = fromReplica.first();
    if (ri == null) continue;
    final int i = session.indexOf(fromRow.node);
    int stopAt = force ? 0 : i;
    Row targetRow = null;
    for (int j = session.matrix.size() - 1; j >= stopAt; j--) {
      targetRow = session.matrix.get(j);
      if (targetRow.node.equals(fromRow.node)) continue;
      if (!isNodeSuitableForReplicaAddition(targetRow, fromRow)) continue;
      targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first
      Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node
      List<Violation> errs = testChangedMatrix(strict, srcRowModified.session);
      Policy.Session tmpSession = srcRowModified.session;

      if (!containsNewErrors(errs) &&
          isLessSerious(errs, leastSeriousViolation) &&
          (force || (tmpSession.indexOf(srcRowModified.node) < tmpSession.indexOf(targetRow.node)))) {

        int result = -1;
        if (!force && srcRowModified.isLive && targetRow.isLive)  {
          result = tmpSession.getPolicy().getClusterPreferences().get(0).compare(srcRowModified, tmpSession.getNode(targetRow.node), true);
          if (result == 0) result = tmpSession.getPolicy().getClusterPreferences().get(0).compare(srcRowModified, tmpSession.getNode(targetRow.node), false);
        }

        if (result <= 0) {
          leastSeriousViolation = errs;
          bestSrcRow = srcRowModified;
          sourceReplicaInfo = ri;
          bestTargetRow = targetRow;
        }
      }
    }
  }
  if (bestSrcRow != null) {
    this.session = bestSrcRow.session;
    return new CollectionAdminRequest.MoveReplica(
        sourceReplicaInfo.getCollection(),
        sourceReplicaInfo.getName(),
        bestTargetRow.node);
  }
  return null;
}
 
Example 6
Source File: TestWithCollection.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Test
  public void testMoveReplicaMainCollection() throws Exception {
    String prefix = "testMoveReplicaMainCollection";
    String xyz = prefix + "_xyz";
    String abc = prefix + "_abc";

    CloudSolrClient solrClient = cluster.getSolrClient();

    String setClusterPolicyCommand = "{" +
        " 'set-cluster-policy': [" +
        "      {'cores':'<10', 'node':'#ANY'}," +
        "      {'replica':'<2', 'node':'#ANY'}," +
        "    ]" +
        "}";
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
    solrClient.request(req);

    String chosenNode = cluster.getRandomJetty(random()).getNodeName();
    log.info("Chosen node {} for collection {}", chosenNode, abc);
    CollectionAdminRequest.createCollection(abc, 1, 1)
        .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
        .process(solrClient);
    CollectionAdminRequest.createCollection(xyz, 1, 1)
        .setWithCollection(abc)
        .process(solrClient);

    String otherNode = null;
    for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
      if (!chosenNode.equals(jettySolrRunner.getNodeName())) {
        otherNode = jettySolrRunner.getNodeName();
      }
    }

    DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz);
    DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc);
    assertNull(collection.getReplicas(otherNode)); // sanity check
    assertNull(withCollection.getReplicas(otherNode)); // sanity check

    CollectionAdminRequest.MoveReplica moveReplica = new CollectionAdminRequest.MoveReplica(xyz, collection.getReplicas().iterator().next().getName(), otherNode);
    moveReplica.setWaitForFinalState(true);
    moveReplica.process(solrClient);
//    zkClient().printLayoutToStdOut();
    collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); // refresh
    DocCollection withCollectionRefreshed = solrClient.getZkStateReader().getClusterState().getCollection(abc); // refresh
    assertTrue(collection.getReplicas().stream().noneMatch(
        replica -> withCollectionRefreshed.getReplicas(replica.getNodeName()) == null
            || withCollectionRefreshed.getReplicas(replica.getNodeName()).isEmpty()));
  }
 
Example 7
Source File: MoveReplicaTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 17-Mar-2018 This JIRA is fixed, but this test still fails
//17-Aug-2018 commented  @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
// commented out on: 17-Feb-2019   @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018
public void testFailedMove() throws Exception {
  String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove;
  int REPLICATION = 2;

  CloudSolrClient cloudClient = cluster.getSolrClient();

  // random create tlog or pull type replicas with nrt
  boolean isTlog = random().nextBoolean();
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);
  create.setAutoAddReplicas(false);
  cloudClient.request(create);

  addDocs(coll, 100);

  NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
  String overseerLeader = (String) overSeerStatus.get("leader");

  // don't kill overseer in this test
  Replica replica;
  int count = 10;
  do {
    replica = getRandomReplica(coll, cloudClient);
  } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0);
  assertNotNull("could not find non-overseer replica???", replica);
  Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
  ArrayList<String> l = new ArrayList<>(liveNodes);
  Collections.shuffle(l, random());
  String targetNode = null;
  for (String node : liveNodes) {
    if (!replica.getNodeName().equals(node) && !overseerLeader.equals(node)) {
      targetNode = node;
      break;
    }
  }
  assertNotNull(targetNode);
  CollectionAdminRequest.MoveReplica moveReplica = createMoveReplicaRequest(coll, replica, targetNode);
  moveReplica.setInPlaceMove(inPlaceMove);
  // start moving
  String asyncId = IdUtils.randomId();
  moveReplica.processAsync(asyncId, cloudClient);
  // shut down target node
  for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
    if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) {
      JettySolrRunner j = cluster.stopJettySolrRunner(i);
      cluster.waitForJettyToStop(j);
      break;
    }
  }
  CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus(asyncId);
  // wait for async request success
  boolean success = true;
  for (int i = 0; i < 200; i++) {
    CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
    assertNotSame(rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED);
    if (rsp.getRequestStatus() == RequestStatusState.FAILED) {
      success = false;
      break;
    }
    Thread.sleep(500);
  }
  assertFalse(success);

  if (log.isInfoEnabled()) {
    log.info("--- current collection state: {}", cloudClient.getZkStateReader().getClusterState().getCollection(coll));
  }
  assertEquals(100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
}
 
Example 8
Source File: MoveReplicaTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(String coll, Replica replica, String targetNode, String shardId) {
  return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName());
}
 
Example 9
Source File: MoveReplicaTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(String coll, Replica replica, String targetNode) {
  return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode);
}
 
Example 10
Source File: TestSimExecutePlanAction.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Test
  // commented out on: 24-Dec-2018   @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018
  public void testExecute() throws Exception {
    SolrClient solrClient = cluster.simGetSolrClient();
    String collectionName = "testExecute";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
        "conf", 1, 2);
    create.setMaxShardsPerNode(1);
    create.process(solrClient);

    if (log.isInfoEnabled()) {
      log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 120, TimeUnit.SECONDS,
          CloudUtil.clusterShape(1, 2, false, true)));
    }

    String sourceNodeName = cluster.getSimClusterStateProvider().simGetRandomNode();
    ClusterState clusterState = cluster.getClusterStateProvider().getClusterState();
    DocCollection docCollection = clusterState.getCollection(collectionName);
    List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
    assertNotNull(replicas);
    assertFalse(replicas.isEmpty());

    List<String> otherNodes = cluster.getClusterStateProvider().getLiveNodes().stream()
        .filter(node -> !node.equals(sourceNodeName)).collect(Collectors.toList());
    assertFalse(otherNodes.isEmpty());
    String survivor = otherNodes.get(0);

    try (ExecutePlanAction action = new ExecutePlanAction()) {
      action.configure(cluster.getLoader(), cluster, Collections.singletonMap("name", "execute_plan"));

      // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
      AtomicBoolean znodeCreated = new AtomicBoolean(false);

      CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor);
      CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
        @Override
        public void setAsyncId(String asyncId) {
          super.setAsyncId(asyncId);
          String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
          try {
            if (cluster.getDistribStateManager().hasData(parentPath)) {
              java.util.List<String> children = cluster.getDistribStateManager().listData(parentPath);
              if (!children.isEmpty()) {
                String child = children.get(0);
                VersionedData data = cluster.getDistribStateManager().getData(parentPath + "/" + child);
                @SuppressWarnings({"rawtypes"})
                Map m = (Map) Utils.fromJSON(data.getData());
                if (m.containsKey("requestid")) {
                  znodeCreated.set(m.get("requestid").equals(asyncId));
                }
              }
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }

        }
      };
      List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
      NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST,
          "mock_trigger_name", Collections.singletonList(SIM_TIME_SOURCE.getTimeNs()),
          Collections.singletonList(sourceNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
      ActionContext actionContext = new ActionContext(cluster, null,
          new HashMap<>(Collections.singletonMap("operations", operations)));
      action.process(nodeLostEvent, actionContext);

//      assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
      @SuppressWarnings({"unchecked"})
      List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
      assertNotNull(responses);
      assertEquals(2, responses.size());
      NamedList<Object> response = responses.get(0);
      assertNull(response.get("failure"));
      assertNotNull(response.get("success"));
    }

    if (log.isInfoEnabled()) {
      log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 300, TimeUnit.SECONDS,
          CloudUtil.clusterShape(1, 2, false, true)));
    }
  }
 
Example 11
Source File: ExecutePlanActionTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Test
  public void testExecute() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "testExecute";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
        "conf", 1, 2);
    create.setMaxShardsPerNode(1);
    create.process(solrClient);
    
    cluster.waitForActiveCollection(collectionName, 1, 2);

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));

    JettySolrRunner sourceNode = cluster.getRandomJetty(random());
    String sourceNodeName = sourceNode.getNodeName();
    ClusterState clusterState = solrClient.getZkStateReader().getClusterState();
    DocCollection docCollection = clusterState.getCollection(collectionName);
    List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
    assertNotNull(replicas);
    assertFalse(replicas.isEmpty());

    List<JettySolrRunner> otherJetties = cluster.getJettySolrRunners().stream()
        .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList());
    assertFalse(otherJetties.isEmpty());
    JettySolrRunner survivor = otherJetties.get(0);

    try (ExecutePlanAction action = new ExecutePlanAction()) {
      action.configure(loader, cloudManager, Collections.singletonMap("name", "execute_plan"));

      // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
      AtomicBoolean znodeCreated = new AtomicBoolean(false);

      CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
      CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
        @Override
        public void setAsyncId(String asyncId) {
          super.setAsyncId(asyncId);
          String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
          try {
            if (zkClient().exists(parentPath, true)) {
              java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
              if (!children.isEmpty()) {
                String child = children.get(0);
                byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true);
                @SuppressWarnings({"rawtypes"})
                Map m = (Map) Utils.fromJSON(data);
                if (m.containsKey("requestid")) {
                  znodeCreated.set(m.get("requestid").equals(asyncId));
                }
              }
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }

        }
      };
      List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
      NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent
        (TriggerEventType.NODELOST, "mock_trigger_name",
         Collections.singletonList(cloudManager.getTimeSource().getTimeNs()),
         Collections.singletonList(sourceNodeName),
         CollectionParams.CollectionAction.MOVEREPLICA.toLower());
      ActionContext actionContext = new ActionContext(survivor.getCoreContainer().getZkController().getSolrCloudManager(), null,
          new HashMap<>(Collections.singletonMap("operations", operations)));
      action.process(nodeLostEvent, actionContext);

//      assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
      @SuppressWarnings({"unchecked"})
      List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
      assertNotNull(responses);
      assertEquals(2, responses.size());
      NamedList<Object> response = responses.get(0);
      assertNull(response.get("failure"));
      assertNotNull(response.get("success"));
    }

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));
  }