org.apache.solr.client.solrj.SolrServerException Java Examples

@Override
protected void setup(Context context) throws IOException, InterruptedException {
  verifyPartitionAssignment(context);    
  SolrRecordWriter.addReducerContext(context);

  /*
   * Note that ReflectionUtils.newInstance() above also implicitly calls
   * resolver.configure(context.getConfiguration()) if the resolver
   * implements org.apache.hadoop.conf.Configurable
   */

  this.exceptionHandler = new FaultTolerance(
      context.getConfiguration().getBoolean(FaultTolerance.IS_PRODUCTION_MODE, false), 
      context.getConfiguration().getBoolean(FaultTolerance.IS_IGNORING_RECOVERABLE_EXCEPTIONS, false),
      context.getConfiguration().get(FaultTolerance.RECOVERABLE_EXCEPTION_CLASSES, SolrServerException.class.getName()));
  
  this.heartBeater = new HeartBeater(context);
}
 

public void createDocumentShrinkers() throws SolrServerException, InterruptedException, ExecutionException {
  ExecutorService service = Executors.newFixedThreadPool(threadPoolSize);
  CompletionService<String> completionService = new ExecutorCompletionService<String>(service);
  HttpSolrServer thothServer = new HttpSolrServer(thothIndexUrl + realTimeCore);
  HttpSolrServer thothShrankServer = new HttpSolrServer(thothIndexUrl + shrankCore);
  ArrayList<ServerDetail> listOfServers = new ThothServers().getList(thothServer);

  for (ServerDetail serverDetail: listOfServers){
    LOG.info("Shrinking docs for server("+serverDetail.getName()+"):("+serverDetail.getPort()+") ");
    completionService.submit(new DocumentShrinker(serverDetail, nowMinusTimeToShrink, thothServer, thothShrankServer));
  }

  // Wait for all the executors to finish
  for(int i = 0; i < listOfServers.size(); i++){
    completionService.take().get();
  }
  LOG.info("Done Shrinking.");
}
 

private SolrInputDocument fetchExistingOrCreateNewSolrDoc(String id) throws SolrServerException, IOException {
  Map<String, String> p = new HashMap<String, String>();
  p.put("q", PHRASE + ":\"" + ClientUtils.escapeQueryChars(id) + "\"");
  
  SolrParams params = new MapSolrParams(p);
  
  QueryResponse res = solrAC.query(params);
  
  if (res.getResults().size() == 0) {
    return new SolrInputDocument();
  } else if (res.getResults().size() == 1) {
    SolrDocument doc = res.getResults().get(0);
    SolrInputDocument tmp = new SolrInputDocument();
    
    for (String fieldName : doc.getFieldNames()) {
      tmp.addField(fieldName, doc.getFieldValue(fieldName));
    }
    return tmp;
  } else {
    throw new IllegalStateException("Query with params : " + p + " returned more than 1 hit!");
  }
}
 

protected List<Map<String, Object>> getIndexFields(String index)
    throws IOException, SolrServerException {
  List<Map<String, Object>> indexFields = new ArrayList<>();

  // Get all the fields in use, including dynamic fields
  LukeRequest lukeRequest = new LukeRequest();
  LukeResponse lukeResponse = lukeRequest.process(client, index);
  for (Entry<String, LukeResponse.FieldInfo> field : lukeResponse.getFieldInfo().entrySet()) {
    Map<String, Object> fieldData = new HashMap<>();
    fieldData.put("name", field.getValue().getName());
    fieldData.put("type", field.getValue().getType());
    indexFields.add(fieldData);

  }

  // Get all the schema fields
  SchemaRepresentation schemaRepresentation = new SchemaRequest().process(client, index)
      .getSchemaRepresentation();
  indexFields.addAll(schemaRepresentation.getFields());

  return indexFields;
}
 

@Before
public void init() throws IOException, SolrServerException {
    MockitoAnnotations.initMocks(this);
    when(solrClient.ping()).thenReturn(solrPingResponse);
    when(solrPingResponse.getStatus()).thenReturn(0);
    when(solrPingResponse.getQTime()).thenReturn(10);

    when(solrClient.query(any(), any(SolrRequest.METHOD.class))).thenReturn(response);
    when(response.getResults()).thenReturn(new SolrDocumentList());
    when(response.getResponse()).thenReturn(responseObject);
    when(responseObject.get("responseHeader")).thenReturn(responseObject);
    when(responseObject.get("params")).thenReturn(responseObject);
    when(responseObject.get("suggestion.field")).thenReturn("category");

    when(solrClient.add(org.mockito.Matchers.<Collection<SolrInputDocument>>any())).thenReturn(iResponse);
    when(solrClient.add(any(SolrInputDocument.class))).thenReturn(iResponse);
    when(iResponse.getQTime()).thenReturn(10);
    when(iResponse.getElapsedTime()).thenReturn(15l);

    //we use the protected constructor to avoid schema checking
    server = new SolrSearchServerTestClass(solrClient);
}
 

/**
 * Write a record. This method accumulates records in to a batch, and when
 * {@link #batchSize} items are present flushes it to the indexer. The writes
 * can take a substantial amount of time, depending on {@link #batchSize}. If
 * there is heavy disk contention the writes may take more than the 600 second
 * default timeout.
 */
@Override
public void write(K key, V value) throws IOException {
  heartBeater.needHeartBeat();
  try {
    try {
      SolrInputDocumentWritable sidw = (SolrInputDocumentWritable) value;
      batch.add(sidw.getSolrInputDocument());
      if (batch.size() >= batchSize) {
        batchWriter.queueBatch(batch);
        numDocsWritten += batch.size();
        if (System.nanoTime() >= nextLogTime) {
          LOG.info("docsWritten: {}", numDocsWritten);
          nextLogTime += TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);
        }
        batch.clear();
      }
    } catch (SolrServerException e) {
      throw new IOException(e);
    }
  } finally {
    heartBeater.cancelHeartBeat();
  }

}
 

@Override
public void run() {
  while (!finished) {
    if (rand.nextBoolean()) {
      doc.setField(authField, authFieldValue0);
    } else {
      doc.setField(authField, authFieldValue1);
    }
    try {
      server.add(doc);
    } catch (SolrServerException sse) {
      throw new RuntimeException(sse);
    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }
  }
}
 

@Test
public void testRedirect() throws Exception {
  final String clientUrl = jetty.getBaseUrl().toString() + "/redirect/foo";
  try (HttpSolrClient client = getHttpSolrClient(clientUrl)) {
    SolrQuery q = new SolrQuery("*:*");
    // default = false
    SolrServerException e = expectThrows(SolrServerException.class, () -> client.query(q));
    assertTrue(e.getMessage().contains("redirect"));

    client.setFollowRedirects(true);
    client.query(q);

    //And back again:
    client.setFollowRedirects(false);
    e = expectThrows(SolrServerException.class, () -> client.query(q));
    assertTrue(e.getMessage().contains("redirect"));
  }

}
 

private boolean waitForRuleToExpire(String collection, String shard, String splitKey, long finishTime) throws KeeperException, InterruptedException, SolrServerException, IOException {
  DocCollection state;
  Slice slice;
  boolean ruleRemoved = false;
  long expiryTime = finishTime + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS);
  while (System.nanoTime() < expiryTime) {
    cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collection);
    state = getCollectionState(collection);
    slice = state.getSlice(shard);
    Map<String,RoutingRule> routingRules = slice.getRoutingRules();
    if (routingRules == null || routingRules.isEmpty() || !routingRules.containsKey(splitKey)) {
      ruleRemoved = true;
      break;
    }
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", splitKey + random().nextInt());
    cluster.getSolrClient().add(collection, doc);
    Thread.sleep(1000);
  }
  return ruleRemoved;
}
 

@Override
public OntologyEntryBean findOntologyEntryByUri(String uri) throws SearchEngineException {
	OntologyEntryBean retVal = null;
	
	try {
		SolrQuery query = new SolrQuery(uri);
		query.setRequestHandler(config.getOntologyNodeRequestHandler());
		
		QueryResponse response = server.query(query);
		List<OntologyEntryBean> annotations = response.getBeans(OntologyEntryBean.class);
		if (annotations.size() > 0) {
			retVal = annotations.get(0);
		}
	} catch (SolrServerException e) {
		throw new SearchEngineException(e);
	}
	
	return retVal;
}
 

private void initialStream(SolrQuery query, SolrClient connection) {
    try {
        SolrQuery solrQuery = query.getCopy();
        solrQuery.setRows(nrOfTimeSeriesPerBatch);
        solrQuery.setStart(currentDocumentCount);

        solrStreamingHandler.init(nrOfTimeSeriesPerBatch, currentDocumentCount);
        QueryResponse response = connection.queryAndStreamResponse(solrQuery, solrStreamingHandler);

        nrOfAvailableTimeSeries = response.getResults().getNumFound();
        queryStart = 0;//(long) response.getResponseHeader().get(ChronixSolrStorageConstants.QUERY_START_LONG);
        queryEnd = Long.MAX_VALUE;//(long) response.getResponseHeader().get(ChronixSolrStorageConstants.QUERY_END_LONG);

        needStream = false;
    } catch (SolrServerException | IOException e) {
        LOGGER.error("SolrServerException occurred while querying server.", e);
    }
}
 

protected void indexJATEDocuments(Path file, EmbeddedSolrServer solrServer, JATEProperties jateProp, boolean commit) throws JATEException {
    if (file == null) {
        return;
    }

    try {
        JATEDocument jateDocument = JATEUtil.loadJATEDocument(file);

        if (isNotEmpty(jateDocument))
            JATEUtil.addNewDoc(solrServer, jateDocument.getId(),
                    jateDocument.getId(), jateDocument.getContent(), jateProp, commit);
    } catch (FileNotFoundException ffe) {
        throw new JATEException(ffe.toString());
    } catch (IOException ioe) {
        throw new JATEException(String.format("failed to index [%s]", file.toString()) + ioe.toString());
    } catch (SolrServerException sse) {
        throw new JATEException(String.format("failed to index [%s] ", file.toString()) + sse.toString());
    }
}
 

protected MCROAISolrResult solrQuery(Optional<String> cursor) throws SolrServerException, IOException {
    SolrQuery query = getBaseQuery(CommonParams.Q);

    // set support
    if (this.set != null) {
        String setId = this.set.getSetId();
        MCROAISetConfiguration<SolrQuery, SolrDocument, String> setConfig = getSetManager().getConfig(setId);
        setConfig.getHandler().apply(this.set, query);
    }
    // from & until
    if (this.from != null || this.until != null) {
        String fromUntilCondition = buildFromUntilCondition(this.from, this.until);
        query.add(CommonParams.FQ, fromUntilCondition);
    }

    // cursor
    query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursor.orElse(CursorMarkParams.CURSOR_MARK_START));
    query.set(CommonParams.ROWS, String.valueOf(getPartitionSize()));
    query.set(CommonParams.SORT, "id asc");

    // do the query
    SolrClient solrClient = MCRSolrClientFactory.getMainSolrClient();
    QueryResponse response = solrClient.query(query);
    Collection<MCROAISetResolver<String, SolrDocument>> setResolver = getSetResolver(response.getResults());
    return new MCROAISolrResult(response, d -> toHeader(d, setResolver));
}
 

@Test
public void validate_indexing() {
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set("q", "*:*");

    try {
        QueryResponse qResp = server.query(params);
        SolrDocumentList docList = qResp.getResults();

        assert (docList.getNumFound() == 2000);
    } catch (SolrServerException e) {
        e.printStackTrace();
    } catch (IOException ioe) {
        ioe.printStackTrace();
    }
}
 

/**
 * Has the same behavior as {@link SolrClient#deleteByQuery(String)}.
 * 
 * @param deleteQuery delete query to be executed
 */
@Override
public void deleteByQuery(String deleteQuery) throws SolrServerException, IOException {
    try {
        solrServer.deleteByQuery(deleteQuery);
    } catch (SolrException e) {
        if (isDocumentIssue(e)) {
            documentDeleteErrorMeter.mark(1);
        } else {
            solrDeleteErrorMeter.mark(1);
            throw e;
        }
    } catch (SolrServerException sse) {
        solrDeleteErrorMeter.mark(1);
        throw sse;
    }
}
 

public void doTestAssertTagStreaming(BiFunction<ModifiableSolrParams,String,QueryRequest> newQueryRequest) throws IOException, SolrServerException {
  ModifiableSolrParams params = params();
  String input = "foo boston bar";//just one tag;
  QueryRequest req = newQueryRequest.apply(params, input);
  req.setPath("/tag");

  final AtomicReference<SolrDocument> refDoc = new AtomicReference<>();
  req.setStreamingResponseCallback(new StreamingResponseCallback() {
    @Override
    public void streamSolrDocument(SolrDocument doc) {
      refDoc.set(doc);
    }

    @Override
    public void streamDocListInfo(long numFound, long start, Float maxScore) {

    }
  });
  QueryResponse rsp = req.process(solrServer);
  assertNotNull(rsp.getResponse().get("tags"));
  assertNotNull(refDoc.get());
  assertEquals("Boston", ((Field)refDoc.get().getFieldValue("name")).stringValue());
}
 

public void indexMap(String id, Map<String, List<String>> objectMap)
    throws IOException, SolrServerException {
  SolrInputDocument document = new SolrInputDocument();
  document.addField("id", (trimId ? id.trim() : id));
  for (Map.Entry<String, List<String>> entry : objectMap.entrySet()) {
    String key = entry.getKey();
    Object value = entry.getValue();
    if (value != null) {
      if (!key.endsWith("_sni") && !key.endsWith("_ss"))
        key += "_ss";
      document.addField(key, value);
    }
  }

  try {
    UpdateResponse response = solr.add(document);
  } catch (HttpSolrClient.RemoteSolrException ex) {
    System.err.printf("document: %s", document);
    System.err.printf("Commit exception: %s%n", ex.getMessage());
  }
}
 

@Override
public UpdateResponse delete(SolrDataQuery query) {
	Assert.notNull(query, "Query must not be 'null'.");

	final String queryString = this.queryParsers.getForClass(query.getClass()).getQueryString(query);

	return execute(new SolrCallback<UpdateResponse>() {
		@Override
		public UpdateResponse doInSolr(SolrClient solrClient) throws SolrServerException, IOException {
			return solrClient.deleteByQuery(queryString);
		}
	});
}
 

@Override
public void rollback() {
	execute(new SolrCallback<UpdateResponse>() {
		@Override
		public UpdateResponse doInSolr(SolrClient solrClient) throws SolrServerException, IOException {
			return solrClient.rollback();
		}
	});
}
 

private SolrServer createServerAndTableWithSimpleTestDoc(String table) throws BlurException, TException, IOException,
    SolrServerException {
  createTable(table);
  SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table);
  SolrInputDocument doc;

  doc = createSimpleTestDoc();

  server.add(doc);
  return server;
}
 

private long fetch() throws IOException, SolrServerException {
	final SolrQuery query = new SolrQuery(filter);

	query.setRows(rows);
	query.setStart((int) start);

	// Only request the fields to be copied and the ID.
	query.setFields(idField);

	if (null != fields) {
		fields.forEach(query::addField);
	}

	logger.info(String.format("Fetching up to %d documents, skipping %d.", rows, start));
	client.queryAndStreamResponse(query, this);

	final long fetched = this.fetched;

	// Stop if there are no more results.
	// Instruct transformers to stop by sending a poison pill.
	if (fetched < rows) {
		stopped = true;
	}

	// Reset for the next run.
	this.fetched = 0;
	return fetched;
}
 

@Test
public void testSetPropertyValidationOfCluster() throws IOException, SolrServerException {
  @SuppressWarnings({"rawtypes"})
  NamedList resp = cluster.getSolrClient().request(
    new V2Request.Builder("/cluster").withMethod(SolrRequest.METHOD.POST).withPayload("{set-property: {name: autoAddReplicas, val:false}}").build());
  assertTrue(resp.toString().contains("status=0"));
  resp = cluster.getSolrClient().request(
      new V2Request.Builder("/cluster").withMethod(SolrRequest.METHOD.POST).withPayload("{set-property: {name: autoAddReplicas, val:null}}").build());
  assertTrue(resp.toString().contains("status=0"));
}
 

public void testGetRawFile() throws SolrServerException, IOException {
  SolrClient client = getSolrClient();
  //assertQ(req("qt", "/admin/file")); TODO file bug that SolrJettyTestBase extends SolrTestCaseJ4
  QueryRequest request = new QueryRequest(params("file", "managed-schema"));
  request.setPath("/admin/file");
  final AtomicBoolean readFile = new AtomicBoolean();
  request.setResponseParser(new ResponseParser() {
    @Override
    public String getWriterType() {
      return "mock";//unfortunately this gets put onto params wt=mock but it apparently has no effect
    }

    @Override
    public NamedList<Object> processResponse(InputStream body, String encoding) {
      try {
        if (body.read() >= 0)
          readFile.set(true);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
      return null;
    }

    @Override
    public NamedList<Object> processResponse(Reader reader) {
      throw new UnsupportedOperationException("TODO unimplemented");//TODO
    }
  });

  client.request(request);//runs request
  //request.process(client); but we don't have a NamedList response
  assertTrue(readFile.get());
}
 

private SolrClient getClient(StratioStreamingMessage message) throws IOException, SolrServerException, URISyntaxException, TransformerException, SAXException, ParserConfigurationException {
    String core = message.getStreamName();
    if (solrClients.containsKey(core)) {
        //we have a client for this core
        return solrClients.get(core);
    } else {
        SolrClient solrClient = getSolrclient(core);
        solrClients.put(core, solrClient);
        return solrClient;
    }
}
 

/**
 * Get the {@link CoreStatus} data for a {@link Replica}
 * <p>
 * This assumes that the replica is hosted on a live node.
 */
protected static CoreStatus getCoreStatus(Replica replica) throws IOException, SolrServerException {
  JettySolrRunner jetty = cluster.getReplicaJetty(replica);
  try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString(), cluster.getSolrClient().getHttpClient())) {
    return CoreAdminRequest.getCoreStatus(replica.getCoreName(), client);
  }
}
 

/**
 * Returns a {@link NamedList} containing server
 * response deserialization is based on the {@code responseParser}
 */
private NamedList<Object> queryClient(SolrClient solrClient, final ModifiableSolrParams params,
                                      ResponseParser responseParser) throws SolrServerException, IOException {
  QueryRequest queryRequest = new QueryRequest(params);
  queryRequest.setResponseParser(responseParser);
  return solrClient.request(queryRequest);
}
 

@Test
@Ignore("This test requires a missing resource.")
public void testLoadFullTaxon() throws Exception {
	ParserWrapper pw = new ParserWrapper();
	final OWLGraphWrapper g = pw.parseToOWLGraph(taxonFile);
	
	GafSolrDocumentLoaderIntegrationRunner.printMemoryStats();
	GafSolrDocumentLoaderIntegrationRunner.gc();
	GafSolrDocumentLoaderIntegrationRunner.printMemoryStats();
	
	ConfigManager configManager = new ConfigManager();
	configManager.add("src/test/resources/test-ont-config.yaml");
	FlexCollection c = new FlexCollection(configManager, g);
	
	GafSolrDocumentLoaderIntegrationRunner.printMemoryStats();
	GafSolrDocumentLoaderIntegrationRunner.gc();
	GafSolrDocumentLoaderIntegrationRunner.printMemoryStats();
	
	FlexSolrDocumentLoader loader = new FlexSolrDocumentLoader((SolrServer)null, c) {

		@Override
		protected void addToServer(Collection<SolrInputDocument> docs)
				throws SolrServerException, IOException {
			solrCounter += docs.size();
			GafSolrDocumentLoaderIntegrationRunner.printMemoryStats();
			System.out.println("Cache size: "+g.getCurrentEdgesAdvancedCacheSize());
		}
		
	};
	loader.load();
	assertTrue(solrCounter > 0);
}
 

private static void createCollectionIfNotExists(CloudSolrClient client, Configuration config, String collection)
        throws IOException, SolrServerException, KeeperException, InterruptedException {
    if (!checkIfCollectionExists(client, collection)) {
        final Integer numShards = config.get(NUM_SHARDS);
        final Integer maxShardsPerNode = config.get(MAX_SHARDS_PER_NODE);
        final Integer replicationFactor = config.get(REPLICATION_FACTOR);


        // Ideally this property used so a new configset is not uploaded for every single
        // index (collection) created in solr.
        // if a generic configSet is not set, make the configset name the same as the collection.
        // This was the default behavior before a default configSet could be specified
        final String  genericConfigSet = config.has(SOLR_DEFAULT_CONFIG) ? config.get(SOLR_DEFAULT_CONFIG):collection;

        final CollectionAdminRequest.Create createRequest = CollectionAdminRequest.createCollection(collection, genericConfigSet, numShards, replicationFactor);
        createRequest.setMaxShardsPerNode(maxShardsPerNode);

        final CollectionAdminResponse createResponse = createRequest.process(client);
        if (createResponse.isSuccess()) {
            logger.trace("Collection {} successfully created.", collection);
        } else {
            throw new SolrServerException(Joiner.on("\n").join(createResponse.getErrorMessages()));
        }
    }

    waitForRecoveriesToFinish(client, collection);
}
 

public void scriptRequest(String collection) throws SolrServerException, IOException {
  SolrClient client = solrCluster.getSolrClient();
  SolrInputDocument doc = sdoc("id", "4055", "subject", "Solr");
  client.add(collection, doc);
  client.commit(collection);

  assertEquals("42", client.query(collection, params("q", "*:*")).getResults().get(0).get("script_added_i"));
}
 

protected static QueryResponse queryRandomShard(ModifiableSolrParams params) throws SolrServerException, IOException
{
    Random r = SOLR_RANDOM_SUPPLIER.getRandomGenerator();
    int which = r.nextInt(clientShards.size());
    SolrClient client = clientShards.get(which);
    return client.query(params);
}