Java Code Examples for org.apache.solr.core.CoreContainer#isZooKeeperAware()

public void loadClusterReporters(PluginInfo[] pluginInfos, CoreContainer cc) {
  // don't load for non-cloud instances
  if (!cc.isZooKeeperAware()) {
    return;
  }
  Map<String, String> attrs = new HashMap<>();
  attrs.put("name", "clusterDefault");
  attrs.put("group", SolrInfoBean.Group.cluster.toString());
  Map<String, Object> initArgs = new HashMap<>();
  initArgs.put("period", DEFAULT_CLOUD_REPORTER_PERIOD);
  List<PluginInfo> infos = prepareCloudPlugins(pluginInfos, SolrInfoBean.Group.cluster.toString(),
      attrs, initArgs);
  String registryName = getRegistryName(SolrInfoBean.Group.cluster);
  for (PluginInfo info : infos) {
    try {
      loadReporter(registryName, cc, info);
    } catch (Exception e) {
      log.warn("Could not load cluster reporter, pluginInfo={}", info, e);
    }
  }
}
 

public static void verifyRuleParams(CoreContainer cc, Map<String, Object> m) {
  @SuppressWarnings({"rawtypes"})
  List l = (List) m.get(RULE);
  if (l != null) {
    for (Object o : l) {
      @SuppressWarnings({"rawtypes"})
      Map map = (Map) o;
      try {
        new Rule(map);
      } catch (Exception e) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error in rule " + m, e);
      }
    }
  }
  if (cc != null && cc.isZooKeeperAware())
    ReplicaAssigner.verifySnitchConf(cc.getZkController().getSolrCloudManager(), (List) m.get(SNITCH));
}
 

@SuppressWarnings({"rawtypes"})
public void inform(SolrCore core) {
  String defaultCollection;
  String defaultZkhost;
  CoreContainer coreContainer = core.getCoreContainer();
  this.solrClientCache = coreContainer.getSolrClientCache();
  this.coreName = core.getName();
  String cacheKey = this.getClass().getName() + "_" + coreName + "_";
  this.objectCache = coreContainer.getObjectCache().computeIfAbsent(cacheKey + "objectCache",
      ConcurrentHashMap.class, k-> new ConcurrentHashMap());
  if (coreContainer.isZooKeeperAware()) {
    defaultCollection = core.getCoreDescriptor().getCollectionName();
    defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress();
    streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost);
    streamFactory.withDefaultZkHost(defaultZkhost);
    modelCache = coreContainer.getObjectCache().computeIfAbsent(cacheKey + "modelCache",
        ModelCache.class,
        k -> new ModelCache(250, defaultZkhost, solrClientCache));
  }
  streamFactory.withSolrResourceLoader(core.getResourceLoader());

  // This pulls all the overrides and additions from the config
  addExpressiblePlugins(streamFactory, core);
}
 

/**
 * Shuts down the test harness and nulls out the values setup by {@link #initCore}
 */
public static void deleteCore() {
  if (h != null) {
    log.info("###deleteCore" );
    // If the test case set up Zk, it should still have it as available,
    // otherwise the core close will just be unnecessarily delayed.
    CoreContainer cc = h.getCoreContainer();
    if (! cc.getCores().isEmpty() && cc.isZooKeeperAware()) {
      try {
        cc.getZkController().getZkClient().exists("/", false);
      } catch (KeeperException e) {
        log.error("Testing connectivity to ZK by checking for root path failed", e);
        fail("Trying to tear down a ZK aware core container with ZK not reachable");
      } catch (InterruptedException ignored) {}
    }

    h.close();
  }

  if (factoryProp == null) {
    System.clearProperty("solr.directoryFactory");
  }

  if (System.getProperty(UPDATELOG_SYSPROP) != null) {
    // clears the updatelog sysprop at the end of the test run
    System.clearProperty(UPDATELOG_SYSPROP);
  }
  
  solrConfig = null;
  h = null;
  lrf = null;
  configString = schemaString = null;
  initCoreDataDir = null;
  hdfsDataDir = null;
}
 

@Override
public void init(PluginInfo pluginInfo, CoreContainer cc) {
  super.init(pluginInfo, cc);
  if (reporter != null) {
    reporter.close();;
  }
  if (!enabled) {
    log.info("Reporter disabled for registry {}", registryName);
    return;
  }
  // start reporter only in cloud mode
  if (!cc.isZooKeeperAware()) {
    log.warn("Not ZK-aware, not starting...");
    return;
  }
  if (period < 1) { // don't start it
    log.info("Turning off node reporter, period={}", period);
    return;
  }
  HttpClient httpClient = cc.getUpdateShardHandler().getDefaultHttpClient();
  ZkController zk = cc.getZkController();
  String reporterId = zk.getNodeName();
  reporter = SolrReporter.Builder.forReports(metricManager, reports)
      .convertRatesTo(TimeUnit.SECONDS)
      .convertDurationsTo(TimeUnit.MILLISECONDS)
      .withHandler(handler)
      .withReporterId(reporterId)
      .setCompact(true)
      .cloudClient(false) // we want to send reports specifically to a selected leader instance
      .skipAggregateValues(true) // we don't want to transport details of aggregates
      .skipHistograms(true) // we don't want to transport histograms
      .build(httpClient, new OverseerUrlSupplier(zk));

  reporter.start(period, TimeUnit.SECONDS);
}
 

/**
 * Returns an String with the name of a core.
 * <p>
 * This method searches the core with fromIndex name in the core's container.
 * If fromIndex isn't name of collection or alias it's returns fromIndex without changes.
 * If fromIndex is name of alias but if the alias points to multiple collections it's throw
 * SolrException.ErrorCode.BAD_REQUEST because multiple shards not yet supported.
 *
 * @param  fromIndex name of the index
 * @param  container the core container for searching the core with fromIndex name or alias
 * @return      the string with name of core
 */
public static String getCoreName(final String fromIndex, CoreContainer container) {
  if (container.isZooKeeperAware()) {
    ZkController zkController = container.getZkController();
    final String resolved = resolveAlias(fromIndex, zkController);
    // TODO DWS: no need for this since later, clusterState.getCollection will throw a reasonable error
    if (!zkController.getClusterState().hasCollection(resolved)) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
          "SolrCloud join: Collection '" + fromIndex + "' not found!");
    }
    return findLocalReplicaForFromIndex(zkController, resolved);
  }
  return fromIndex;
}
 

@Override
public void inform(SolrCore core) {
  super.inform(core);
  String defaultCollection;
  String defaultZkhost;
  CoreContainer coreContainer = core.getCoreContainer();
  this.solrClientCache = coreContainer.getSolrClientCache();
  this.coreName = core.getName();

  if (coreContainer.isZooKeeperAware()) {
    defaultCollection = core.getCoreDescriptor().getCollectionName();
    defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress();
    streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost);
    streamFactory.withDefaultZkHost(defaultZkhost);
    modelCache = new ModelCache(250,
        defaultZkhost,
        solrClientCache);
  }
  streamFactory.withSolrResourceLoader(core.getResourceLoader());
  StreamHandler.addExpressiblePlugins(streamFactory, core);
  initialStreamContext = new StreamContext();
  initialStreamContext.setStreamFactory(streamFactory);
  initialStreamContext.setSolrClientCache(solrClientCache);
  initialStreamContext.setModelCache(modelCache);
  initialStreamContext.setObjectCache(objectCache);
  initialStreamContext.put("core", this.coreName);
  initialStreamContext.put("solr-core", core);
}
 

public void inform(SolrCore core) {
  CoreContainer coreContainer = core.getCoreContainer();

  if(coreContainer.isZooKeeperAware()) {
    defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress();
    defaultWorkerCollection = core.getCoreDescriptor().getCollectionName();
    isCloud = true;
  }
}
 

@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
  // Make sure the cores is enabled
  CoreContainer cores = getCoreContainer();
  if (cores == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        "Core container instance missing");
  }

  // Make sure that the core is ZKAware
  if (!cores.isZooKeeperAware()) {
    throw new SolrException(ErrorCode.BAD_REQUEST,
        "Solr instance is not running in SolrCloud mode.");
  }

  // Pick the action
  SolrParams params = req.getParams();
  String a = params.get(CoreAdminParams.ACTION);
  if (a != null) {
    CollectionAction action = CollectionAction.get(a);
    if (action == null) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown action: " + a);
    }
    CollectionOperation operation = CollectionOperation.get(action);
    if (log.isInfoEnabled()) {
      log.info("Invoked Collection Action :{} with params {} and sendToOCPQueue={}"
          , action.toLower(), req.getParamString(), operation.sendToOCPQueue);
    }
    MDCLoggingContext.setCollection(req.getParams().get(COLLECTION));
    invokeAction(req, rsp, cores, action, operation);
  } else {
    throw new SolrException(ErrorCode.BAD_REQUEST, "action is a required param");
  }
  rsp.setHttpCaching(false);
}
 

@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
    throws InterruptedException, KeeperException, IOException {

  CoreContainer coreContainer = req.getCore().getCoreContainer();
  if (coreContainer.isZooKeeperAware()) {
    showFromZooKeeper(req, rsp, coreContainer);
  } else {
    showFromFileSystem(req, rsp);
  }
}
 

private ShardHandler getAndPrepShardHandler(SolrQueryRequest req, ResponseBuilder rb) {
  ShardHandler shardHandler = null;

  CoreContainer cc = req.getCore().getCoreContainer();
  boolean isZkAware = cc.isZooKeeperAware();
  rb.isDistrib = req.getParams().getBool(DISTRIB, isZkAware);
  if (!rb.isDistrib) {
    // for back compat, a shards param with URLs like localhost:8983/solr will mean that this
    // search is distributed.
    final String shards = req.getParams().get(ShardParams.SHARDS);
    rb.isDistrib = ((shards != null) && (shards.indexOf('/') > 0));
  }
  
  if (rb.isDistrib) {
    shardHandler = shardHandlerFactory.getShardHandler();
    shardHandler.prepDistributed(rb);
    if (!rb.isDistrib) {
      shardHandler = null; // request is not distributed after all and so the shard handler is not needed
    }
  }

  if (isZkAware) {
    String shardsTolerant = req.getParams().get(ShardParams.SHARDS_TOLERANT);
    boolean requireZkConnected = shardsTolerant != null && shardsTolerant.equals(ShardParams.REQUIRE_ZK_CONNECTED);
    ZkController zkController = cc.getZkController();
    boolean zkConnected = zkController != null && ! zkController.getZkClient().getConnectionManager().isLikelyExpired();
    if (requireZkConnected && false == zkConnected) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ZooKeeper is not connected");
    } else {
      NamedList<Object> headers = rb.rsp.getResponseHeader();
      if (headers != null) {
        headers.add("zkConnected", zkConnected);
      }
    }
  }

  return shardHandler;
}
 

private ShardHandler getAndPrepShardHandler(SolrQueryRequest req, ResponseBuilder rb, ShardHandlerFactory shardHandlerFactory) {
  ShardHandler shardHandler = null;

  boolean isZkAware = false;
  CoreContainer cc = null;
  if (req.getCore() != null) {
    cc = req.getCore().getCoreContainer();
    isZkAware = cc.isZooKeeperAware();
  } 
  
  rb.isDistrib = req.getParams().getBool("distrib", isZkAware);
  if (!rb.isDistrib) {
    // for back compat, a shards param with URLs like localhost:8983/solr will mean that this
    // search is distributed.
    final String shards = req.getParams().get(ShardParams.SHARDS);
    rb.isDistrib = ((shards != null) && (shards.indexOf('/') > 0));
  }
  
  if (rb.isDistrib) {
    shardHandler = shardHandlerFactory.getShardHandler();
    shardHandler.prepDistributed(rb);
    if (!rb.isDistrib) {
      shardHandler = null; // request is not distributed after all and so the shard handler is not needed
    }
  }

  if(isZkAware) {
    ZkController zkController = cc.getZkController();
    NamedList<Object> headers = rb.rsp.getResponseHeader();
    if(headers != null) {
      headers.add("zkConnected", 
          zkController != null 
        ? !zkController.getZkClient().getConnectionManager().isLikelyExpired() 
        : false);
    }
    
  }

  return shardHandler;
}
 

@SuppressWarnings({"unchecked"})
public void inform(SolrCore core) {
  String defaultCollection;
  String defaultZkhost;
  CoreContainer coreContainer = core.getCoreContainer();
  this.coreName = core.getName();
  this.solrClientCache = coreContainer.getSolrClientCache();

  if(coreContainer.isZooKeeperAware()) {
    defaultCollection = core.getCoreDescriptor().getCollectionName();
    defaultZkhost = core.getCoreContainer().getZkController().getZkServerAddress();
    streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost);
    streamFactory.withDefaultZkHost(defaultZkhost);
  }

  // This pulls all the overrides and additions from the config
  StreamHandler.addExpressiblePlugins(streamFactory, core);

  // Check deprecated approach.
  Object functionMappingsObj = initArgs.get("streamFunctions");
  if(null != functionMappingsObj){
    log.warn("solrconfig.xml: <streamFunctions> is deprecated for adding additional streaming functions to GraphHandler.");
    NamedList<?> functionMappings = (NamedList<?>)functionMappingsObj;
    for(Entry<String,?> functionMapping : functionMappings) {
      String key = functionMapping.getKey();
      PluginInfo pluginInfo = new PluginInfo(key, Collections.singletonMap("class", functionMapping.getValue()));

      if (pluginInfo.pkgName == null) {
        Class<? extends Expressible> clazz = core.getResourceLoader().findClass((String) functionMapping.getValue(),
            Expressible.class);
        streamFactory.withFunctionName(key, clazz);
      } else {
        @SuppressWarnings("resource")
        StreamHandler.ExpressibleHolder holder = new StreamHandler.ExpressibleHolder(pluginInfo, core, SolrConfig.classVsSolrPluginInfo.get(Expressible.class.getName()));
        streamFactory.withFunctionName(key, () -> holder.getClazz());
      }

    }

  }
}
 

@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {

  CoreContainer cores = getCoreContainer();
  rsp.setHttpCaching(false);

  // Core container should not be null and active (redundant check)
  if(cores == null || cores.isShutDown()) {
    rsp.setException(new SolrException(SolrException.ErrorCode.SERVER_ERROR, "CoreContainer is either not initialized or shutting down"));
    return;
  }
  if(!cores.isZooKeeperAware()) {
    //TODO: Support standalone instances
    rsp.setException(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Health check is only available when running in SolrCloud mode"));
    return;
  }
  if (log.isDebugEnabled()) {
    log.debug("Invoked HealthCheckHandler on [{}]", coreContainer.getZkController().getNodeName());
  }
  ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
  ClusterState clusterState = zkStateReader.getClusterState();
  // Check for isConnected and isClosed
  if(zkStateReader.getZkClient().isClosed() || !zkStateReader.getZkClient().isConnected()) {
    rsp.add(STATUS, FAILURE);
    rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Host Unavailable: Not connected to zk"));
    return;
  }

  // Fail if not in live_nodes
  if (!clusterState.getLiveNodes().contains(cores.getZkController().getNodeName())) {
    rsp.add(STATUS, FAILURE);
    rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Host Unavailable: Not in live nodes as per zk"));
    return;
  }

  // Optionally require that all cores on this node are active if param 'requireHealthyCores=true'
  if (req.getParams().getBool(PARAM_REQUIRE_HEALTHY_CORES, false)) {
    Collection<CloudDescriptor> coreDescriptors = cores.getCores().stream()
        .map(c -> c.getCoreDescriptor().getCloudDescriptor()).collect(Collectors.toList());
    long unhealthyCores = findUnhealthyCores(coreDescriptors, clusterState);
    if (unhealthyCores > 0) {
        rsp.add(STATUS, FAILURE);
        rsp.add("num_cores_unhealthy", unhealthyCores);
        rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, unhealthyCores + " out of "
            + cores.getAllCoreNames().size() + " replicas are currently initializing or recovering"));
        return;
    }
    rsp.add("message", "All cores are healthy");
  }

  // All lights green, report healthy
  rsp.add(STATUS, OK);
}
 

public static boolean maybeProxyToNodes(SolrQueryRequest req, SolrQueryResponse rsp, CoreContainer container)
    throws IOException, SolrServerException, InterruptedException {
  String nodeNames = req.getParams().get(PARAM_NODES);
  if (nodeNames == null || nodeNames.isEmpty()) {
    return false; // local request
  }

  if (!container.isZooKeeperAware()) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Parameter " + PARAM_NODES + " only supported in Cloud mode");
  }
  
  Set<String> nodes;
  String pathStr = req.getPath();
  
  @SuppressWarnings({"unchecked"})
  Map<String,String> paramsMap = req.getParams().toMap(new HashMap<>());
  paramsMap.remove(PARAM_NODES);
  SolrParams params = new MapSolrParams(paramsMap);
  Set<String> liveNodes = container.getZkController().zkStateReader.getClusterState().getLiveNodes();
  
  if (nodeNames.equals("all")) {
    nodes = liveNodes;
    log.debug("All live nodes requested");
  } else {
    nodes = new HashSet<>(Arrays.asList(nodeNames.split(",")));
    for (String nodeName : nodes) {
      if (!nodeName.matches("^[^/:]+:\\d+_[\\w/]+$")) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Parameter " + PARAM_NODES + " has wrong format");
      }

      if (!liveNodes.contains(nodeName)) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Requested node " + nodeName + " is not part of cluster");
      }
    }       
    log.debug("Nodes requested: {}", nodes);
  }
  if (log.isDebugEnabled()) {
    log.debug("{} parameter {} specified on {} request", PARAM_NODES, nodeNames, pathStr);
  }
  
  Map<String, Pair<Future<NamedList<Object>>, SolrClient>> responses = new HashMap<>();
  for (String node : nodes) {
    responses.put(node, callRemoteNode(node, pathStr, params, container.getZkController()));
  }
  
  for (Map.Entry<String, Pair<Future<NamedList<Object>>, SolrClient>> entry : responses.entrySet()) {
    try {
      NamedList<Object> resp = entry.getValue().first().get(10, TimeUnit.SECONDS);
      entry.getValue().second().close();
      rsp.add(entry.getKey(), resp);
    } catch (ExecutionException ee) {
      log.warn("Exception when fetching result from node {}", entry.getKey(), ee);
    } catch (TimeoutException te) {
      log.warn("Timeout when fetching result from node {}", entry.getKey(), te);
    }
  }
  if (log.isInfoEnabled()) {
    log.info("Fetched response from {} nodes: {}", responses.keySet().size(), responses.keySet());
  }
  return true;
}