org.elasticsearch.action.search.SearchAction Java Examples

/**
 * Perform a test search request to validate the element prior to storing it.
 *
 * @param validation validation info
 * @param element the element stored
 * @param task the parent task
 * @param listener the action listener to write to
 * @param onSuccess action ro run when the validation is successfull
 */
private void validate(FeatureValidation validation,
                      StorableElement element,
                      Task task,
                      ActionListener<FeatureStoreResponse> listener,
                      Runnable onSuccess) {
    ValidatingLtrQueryBuilder ltrBuilder = new ValidatingLtrQueryBuilder(element,
            validation, factory);
    SearchRequestBuilder builder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
    builder.setIndices(validation.getIndex());
    builder.setQuery(ltrBuilder);
    builder.setFrom(0);
    builder.setSize(20);
    // Bail out early and don't score the whole index.
    builder.setTerminateAfter(1000);
    builder.request().setParentTask(clusterService.localNode().getId(), task.getId());
    builder.execute(wrap((r) -> {
            if (r.getFailedShards() > 0) {
                ShardSearchFailure failure = r.getShardFailures()[0];
                throw new IllegalArgumentException("Validating the element caused " + r.getFailedShards() +
                        " shard failures, see root cause: " + failure.reason(), failure.getCause());
            }
            onSuccess.run();
        },
        (e) -> listener.onFailure(new IllegalArgumentException("Cannot store element, validation failed.", e))));
}
 

protected void executeCardinalityRequest(final NodeTaskContext context, final NodeTaskReporter reporter) {
  logger.debug("Executing async cardinality action");
  final SearchRequest cardinalityRequest = this.getCardinalityRequest(context.getNode(), context.getVisitor().getParentRequest());
  context.getClient().execute(SearchAction.INSTANCE, cardinalityRequest, new ActionListener<SearchResponse>() {

    @Override
    public void onResponse(SearchResponse searchResponse) {
      Cardinality c = searchResponse.getAggregations().get(context.getNode().getLookupPath());
      context.getNode().setCardinality(c.getValue());
      reporter.success(context);
    }

    @Override
    public void onFailure(Throwable e) {
      reporter.failure(e);
    }

  });
}
 

public Long mostRecentDocument(String index) {
    if (client() == null) {
        return null;
    }
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE);
    SortBuilder sort = SortBuilders.fieldSort("_timestamp").order(SortOrder.DESC);
    SearchResponse searchResponse = searchRequestBuilder.setIndices(index).addField("_timestamp").setSize(1).addSort(sort).execute().actionGet();
    if (searchResponse.getHits().getHits().length == 1) {
        SearchHit hit = searchResponse.getHits().getHits()[0];
        if (hit.getFields().get("_timestamp") != null) {
            return hit.getFields().get("_timestamp").getValue();
        } else {
            return 0L;
        }
    }
    return null;
}
 

public HttpInvoker(Settings settings, ThreadPool threadPool, Headers headers, URL url) {
    super(settings, threadPool, headers);
    this.contexts = new HashMap<>();
    this.bootstrap = new ClientBootstrap(new NioClientSocketChannelFactory(
            Executors.newCachedThreadPool(),
            Executors.newCachedThreadPool()));
    bootstrap.setPipelineFactory(new HttpInvoker.HttpClientPipelineFactory());
    bootstrap.setOption("tcpNoDelay", true);

    registerAction(BulkAction.INSTANCE, HttpBulkAction.class);
    registerAction(CreateIndexAction.INSTANCE, HttpCreateIndexAction.class);
    registerAction(RefreshAction.INSTANCE, HttpRefreshIndexAction.class);
    registerAction(ClusterUpdateSettingsAction.INSTANCE, HttpClusterUpdateSettingsAction.class);
    registerAction(UpdateSettingsAction.INSTANCE, HttpUpdateSettingsAction.class);
    registerAction(SearchAction.INSTANCE, HttpSearchAction.class);

    this.url = url;
}
 

/**
 * Submit a search query to Elasticsearch.
 *
 * @param indexName The name of the index to search.
 * @param query     The query to search.
 * @return The search response returned by Elasticsearch.
 * @throws IOException
 */
private SearchResponse search(String indexName, String query) throws IOException {
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
    try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(new NamedXContentRegistry(searchModule
            .getNamedXContents()), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, query)) {
        searchSourceBuilder.parseXContent(parser);
    }
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(indexName).setSource(searchSourceBuilder);
    if (this.searchAllowPartialSearchResults != null)
        searchRequestBuilder.setAllowPartialSearchResults(this.searchAllowPartialSearchResults);
    if (this.searchBatchedReduceSize != null)
        searchRequestBuilder.setBatchedReduceSize(this.searchBatchedReduceSize);
    if (this.searchMaxConcurrentShardRequests != null)
        searchRequestBuilder.setMaxConcurrentShardRequests(this.searchMaxConcurrentShardRequests);
    if (this.searchPreFilterShardSize != null)
        searchRequestBuilder.setPreFilterShardSize(this.searchPreFilterShardSize);
    if (this.searchPreference != null)
        searchRequestBuilder.setPreference(this.searchPreference);
    if (this.searchRequestCache != null)
        searchRequestBuilder.setRequestCache(this.searchRequestCache);
    if (this.maxTimePerQuery != null)
        searchRequestBuilder.setTimeout(TimeValue.parseTimeValue(this.maxTimePerQuery, "timeout"));
    return searchRequestBuilder.execute().actionGet();
}
 

public void testLangDetectBinary() throws Exception {
    try {
        CreateIndexRequestBuilder createIndexRequestBuilder =
                new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex("test");
        createIndexRequestBuilder.addMapping("someType", jsonBuilder()
                .startObject()
                .startObject("properties")
                .startObject("content")
                .field("type", "binary")
                .startObject("fields")
                .startObject("language")
                .field("type", "langdetect")
                .field("binary", true)
                .endObject()
                .endObject()
                .endObject()
                .endObject()
                .endObject());
        createIndexRequestBuilder.execute().actionGet();
        IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client(), IndexAction.INSTANCE)
                .setIndex("test").setType("someType").setId("1")
                //\"God Save the Queen\" (alternatively \"God Save the King\"
                .setSource("content", "IkdvZCBTYXZlIHRoZSBRdWVlbiIgKGFsdGVybmF0aXZlbHkgIkdvZCBTYXZlIHRoZSBLaW5nIg==");
        indexRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute().actionGet();
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE)
                .setIndices("test")
                .setQuery(QueryBuilders.termQuery("content.language", "en"))
                .addStoredField("content.language");
        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
        assertEquals(1L, searchResponse.getHits().getTotalHits());
        assertEquals("en", searchResponse.getHits().getAt(0).field("content.language").getValue());
    } finally {
        DeleteIndexRequestBuilder deleteIndexRequestBuilder =
                new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, "test");
        deleteIndexRequestBuilder.execute().actionGet();
    }
}
 

@Test
public void testDuplicateDocIDs() throws Exception {
    long numactions = NUM_ACTIONS;
    final BulkNodeClient client = ClientBuilder.builder()
            .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
            .setMetric(new LongAdderIngestMetric())
            .toBulkNodeClient(client("1"));
    try {
        client.newIndex("test");
        for (int i = 0; i < NUM_ACTIONS; i++) {
            client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
        }
        client.flushIngest();
        client.waitForResponses(TimeValue.timeValueSeconds(30));
        client.refreshIndex("test");
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
                .setIndices("test")
                .setTypes("test")
                .setQuery(matchAllQuery());
        long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
        logger.info("hits = {}", hits);
        assertTrue(hits < NUM_ACTIONS);
    } catch (NoNodeAvailableException e) {
        logger.warn("skipping, no node available");
    } finally {
        client.shutdown();
        assertEquals(numactions, client.getMetric().getSucceeded().getCount());
        if (client.hasThrowable()) {
            logger.error("error", client.getThrowable());
        }
        assertFalse(client.hasThrowable());
    }
}
 

@Test
public void testDuplicateDocIDs() throws Exception {
    long numactions = NUM_ACTIONS;
    final BulkTransportClient client = ClientBuilder.builder()
            .put(getSettings())
            .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
            .setMetric(new LongAdderIngestMetric())
            .toBulkTransportClient();
    try {
        client.newIndex("test");
        for (int i = 0; i < NUM_ACTIONS; i++) {
            client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
        }
        client.flushIngest();
        client.waitForResponses(TimeValue.timeValueSeconds(30));
        client.refreshIndex("test");
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
                .setIndices("test")
                .setTypes("test")
                .setQuery(matchAllQuery());
        long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
        logger.info("hits = {}", hits);
        assertTrue(hits < NUM_ACTIONS);
    } catch (NoNodeAvailableException e) {
        logger.warn("skipping, no node available");
    } finally {
        client.shutdown();
        assertEquals(numactions, client.getMetric().getSucceeded().getCount());
        if (client.hasThrowable()) {
            logger.error("error", client.getThrowable());
        }
        assertFalse(client.hasThrowable());
    }
}
 

@Override
public <Request extends ActionRequest, Response extends ActionResponse> void apply(final Task task, final String action,
        final Request request, final ActionListener<Response> listener, final ActionFilterChain<Request, Response> chain) {
    if (!SearchAction.INSTANCE.name().equals(action)) {
        chain.proceed(task, action, request, listener);
        return;
    }

    final SearchRequest searchRequest = (SearchRequest) request;
    final ActionListener<Response> wrappedListener = DynamicRanker.getInstance().wrapActionListener(action, searchRequest, listener);
    chain.proceed(task, action, request, wrappedListener == null ? listener : wrappedListener);
}
 

/**
 * The ids in index function will take as arguments the index name and the document type and will return a list of all the ids in the index.
 */
@Override
public final List<String> getIdsInIndex(String indexName, String documentType)
{
    // Create an array list for storing the ids
    List<String> idList = new ArrayList<>();

    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.query(QueryBuilders.matchAllQuery());
    // Create a search request and set the scroll time and scroll size
    final SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(indexName).setTypes(documentType).setScroll(new TimeValue(ELASTIC_SEARCH_SCROLL_KEEP_ALIVE_TIME))
        .setSize(ELASTIC_SEARCH_SCROLL_PAGE_SIZE).setSource(searchSourceBuilder);

    // Retrieve the search response
    final Search.Builder searchBuilder = new Search.Builder(searchRequestBuilder.toString()).addIndex(indexName);

    searchBuilder.setParameter(Parameters.SIZE, ELASTIC_SEARCH_SCROLL_PAGE_SIZE);
    searchBuilder.setParameter(Parameters.SCROLL, new TimeValue(ELASTIC_SEARCH_SCROLL_KEEP_ALIVE_TIME).toString());

    JestResult jestResult = jestClientHelper.execute(searchBuilder.build());

    // While there are hits available, page through the results and add them to the id list
    while (jestResult.getSourceAsStringList().size() != 0)
    {
        for (String jsonString : jestResult.getSourceAsStringList())
        {
            JsonElement root = new JsonParser().parse(jsonString);
            idList.add(root.getAsJsonObject().get("id").getAsString());
        }
        String scrollId = jestResult.getJsonObject().get("_scroll_id").getAsString();
        SearchScroll scroll = new SearchScroll.Builder(scrollId, new TimeValue(ELASTIC_SEARCH_SCROLL_KEEP_ALIVE_TIME).toString()).build();
        jestResult = jestClientHelper.execute(scroll);

    }
    return idList;
}
 

public static SearchRequestBuilder constructSearchRequestBuilder(String query, Pageable p, int numberOfFragments, Client client) {

        return new SearchRequestBuilder(client, SearchAction.INSTANCE)
            .setHighlighterEncoder("html")
            .setHighlighterFragmentSize(150)
            .setHighlighterPreTags("<mark>")
            .setHighlighterPostTags("</mark>")
            .addHighlightedField("content")
            .setQuery(queryBuilder(query))
            .setFetchSource(null, "content");
    }
 

@Override
public void execute(ActionListener<CountResponse> listener) {
    CountRequest countRequest = beforeExecute(request);
    client.execute(SearchAction.INSTANCE, countRequest.toSearchRequest(), new DelegatingActionListener<SearchResponse, CountResponse>(listener) {
        @Override
        protected CountResponse getDelegatedFromInstigator(SearchResponse response) {
            return new CountResponse(response);
        }
    });
}
 

@Override
	public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
        Hint scrollHint = null;
        for (Hint hint : select.getHints()) {
            if (hint.getType() == HintType.USE_SCROLL) {
                scrollHint = hint;
                break;
            }
        }
        if (scrollHint != null && scrollHint.getParams()[0] instanceof String) {
            return new SqlElasticSearchRequestBuilder(new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE, (String) scrollHint.getParams()[0]).setScroll(new TimeValue((Integer) scrollHint.getParams()[1])));
        }

		/*
		zhongshu-comment 6.1.1.5这个版本和elastic6.1.1这个分支用的是这一行代码
		但是在本地调试时我的client没有实例化，并没有去连es，所以这行代码会报空指针
		那就将这行注释掉吧，以后就用下面那行
		 */
//		this.request = client.prepareSearch();

		/*
		zhongshu-comment  6.2.4.1这个版本和master_zhongshu_dev_01用的是这一行代码，虽然client为null，但是下面这行代码并不会报空指针
							为了在本地调试、执行下文的那些代码获得es的dsl，所以就使用这行代码，暂时将上面哪一行注释掉，上线的时候记得替换掉
		变量request是es搜索请求对象，调用的是es的api，SearchRequestBuilder是es的原生api
		 */
        this.request = new SearchRequestBuilder(client, SearchAction.INSTANCE);
		setIndicesAndTypes();

		//zhongshu-comment 将Select对象中封装的sql token信息转换并传到成员变量es搜索请求对象request中
		setFields(select.getFields());

		setWhere(select.getWhere());
		setSorts(select.getOrderBys());
		setLimit(select.getOffset(), select.getRowCount());

        //
        if (scrollHint != null) {
            if (!select.isOrderdSelect())
                request.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC);
            request.setSize((Integer) scrollHint.getParams()[0]).setScroll(new TimeValue((Integer) scrollHint.getParams()[1]));
        } else {
            request.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
        }
        updateRequestWithIndexAndRoutingOptions(select, request);
		updateRequestWithHighlight(select, request);
		updateRequestWithCollapse(select, request);
		updateRequestWithPostFilter(select, request);
		updateRequestWithStats(select, request);
		updateRequestWithPreference(select, request);
		updateRequestWithTrackTotalHits(select, request);
		updateRequestWithTimeout(select, request);
		updateRequestWithIndicesOptions(select, request);
		updateRequestWithMinScore(select, request);
		SqlElasticSearchRequestBuilder sqlElasticRequestBuilder = new SqlElasticSearchRequestBuilder(request);

		return sqlElasticRequestBuilder;
	}
 

public void testChineseLanguageCode() throws Exception {
    try {
        XContentBuilder builder = jsonBuilder()
                .startObject()
                .startObject("properties")
                .startObject("content")
                .field("type", "text")
                .startObject("fields")
                .startObject("language")
                .field("type", "langdetect")
                .array("languages", "zh-cn", "en", "de")
                .endObject()
                .endObject()
                .endObject()
                .endObject()
                .endObject();
        CreateIndexRequestBuilder createIndexRequestBuilder =
                new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE);
        createIndexRequestBuilder.setIndex("test").addMapping("someType", builder).execute().actionGet();
        String source = "位于美国首都华盛顿都会圈的希望中文学校５日晚举办活动庆祝建立２０周年。" +
                "从中国大陆留学生为子女学中文而自发建立的学习班，到学生规模在全美名列前茅的中文学校，" +
                "这个平台的发展也折射出美国的中文教育热度逐步提升。\n" +
                "希望中文学校是大华盛顿地区最大中文学校，现有７个校区逾４０００名学生，" +
                "规模在美国东部数一数二。" +
                "不过，见证了希望中文学校２０年发展的人们起初根本无法想象这个小小的中文教育平台能发展到今日之规模。";
        IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client(), IndexAction.INSTANCE)
                .setIndex("test").setType("someType").setId("1")
                .setSource("content", source);
        indexRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute().actionGet();
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE)
                .setIndices("test")
                .setQuery(QueryBuilders.termQuery("content.language", "zh-cn"))
                .addStoredField("content.language");
        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
        assertEquals(1L, searchResponse.getHits().getTotalHits());
        assertEquals("zh-cn", searchResponse.getHits().getAt(0).field("content.language").getValue());
    } finally {
        DeleteIndexRequestBuilder deleteIndexRequestBuilder =
                new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, "test");
        deleteIndexRequestBuilder.execute().actionGet();
    }
}
 

@Override
public ActionResponse search(String[] indices, String[] types, String query, int size) {
  final SearchRequestBuilder reqBuilder = new SearchRequestBuilder(
      client, SearchAction.INSTANCE);
  reqBuilder.setIndices();

  if (indices != null) {
    reqBuilder.setIndices(indices);
  }
  if (types != null) {
    reqBuilder.setTypes(types);
  }

  if (!StringUtils.isEmpty(query)) {
    // The query can be either JSON-formatted, nor a Lucene query
    // So, try to parse as a JSON => if there is an error, consider the query a Lucene one
    try {
      @SuppressWarnings("rawtypes")
      final Map source = gson.fromJson(query, Map.class);
      reqBuilder.setExtraSource(source);
    } catch (final JsonSyntaxException e) {
      // This is not a JSON (or maybe not well formatted...)
      reqBuilder.setQuery(QueryBuilders.queryStringQuery(query).analyzeWildcard(true));
    }
  }

  reqBuilder.setSize(size);

  final SearchResponse searchResp = reqBuilder.get();

  final ActionResponse actionResp = new ActionResponse()
      .succeeded(true)
      .totalHits(searchResp.getHits().getTotalHits());

  if (searchResp.getAggregations() != null) {
    setAggregations(searchResp.getAggregations(), actionResp);
  } else {
    for (final SearchHit hit: searchResp.getHits()) {
      // Fields can be found either in _source, or in fields (it depends on the query)
      // => specific for elasticsearch's version < 5
      //
      String src = hit.getSourceAsString();
      if (src == null) {
        final Map<String, Object> hitFields = new HashMap<>();
        for (final SearchHitField hitField : hit.getFields().values()) {
          hitFields.put(hitField.getName(), hitField.getValues());
        }
        src = gson.toJson(hitFields);
      }
      actionResp.addHit(new HitWrapper(hit.getIndex(), hit.getType(), hit.getId(), src));
    }
  }

  return actionResp;
}
 

public void testGermanLanguageCode() throws Exception {
    try {
        XContentBuilder builder = jsonBuilder()
                .startObject()
                .startObject("properties")
                .startObject("content")
                .field("type", "text")
                .startObject("fields")
                .startObject("language")
                .field("type", "langdetect")
                .array("languages", "zh-cn", "en", "de")
                .endObject()
                .endObject()
                .endObject()
                .endObject()
                .endObject();
        CreateIndexRequestBuilder createIndexRequestBuilder =
                new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE);
        createIndexRequestBuilder.setIndex("test").addMapping("someType", builder).execute().actionGet();
        String source = "Einigkeit und Recht und Freiheit\n" +
                "für das deutsche Vaterland!\n" +
                "Danach lasst uns alle streben\n" +
                "brüderlich mit Herz und Hand!";
        IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client(), IndexAction.INSTANCE)
                .setIndex("test").setType("someType").setId("1")
                .setSource("content", source);
        indexRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                .execute().actionGet();
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE)
                .setIndices("test")
                .setQuery(QueryBuilders.termQuery("content.language", "de"))
                .addStoredField("content.language");
        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
        assertEquals(1L, searchResponse.getHits().getTotalHits());
        assertEquals("de", searchResponse.getHits().getAt(0).field("content.language").getValue());
    } finally {
        DeleteIndexRequestBuilder deleteIndexRequestBuilder =
                new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, "test");
        deleteIndexRequestBuilder.execute().actionGet();
    }
}
 

public CoordinateSearchRequestBuilder(final ElasticsearchClient client) {
  // hack to be able to subclass SearchRequestBuilder: the action instance is only used in #execute which we overwrite
  super(client, SearchAction.INSTANCE);
}
 

@Test
public void testThreadedRandomDocs() throws Exception {
    int maxthreads = Runtime.getRuntime().availableProcessors();
    long maxactions = MAX_ACTIONS;
    final long maxloop = NUM_ACTIONS;
    logger.info("HttpBulkNodeClient max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop);
    final HttpBulkNodeClient client = ClientBuilder.builder()
            .put("host", "127.0.0.1")
            .put("port", 9200)
            .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions)
            .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
            .setMetric(new LongAdderIngestMetric())
            .toHttpBulkNodeClient();
    try {
        client.newIndex("test")
                .startBulk("test", -1, 1000);
        ThreadPoolExecutor pool = EsExecutors.newFixed("http-bulk-nodeclient-test", maxthreads, 30,
                EsExecutors.daemonThreadFactory("http-bulk-nodeclient-test"));
        final CountDownLatch latch = new CountDownLatch(maxthreads);
        for (int i = 0; i < maxthreads; i++) {
            pool.execute(new Runnable() {
                public void run() {
                    for (int i = 0; i < maxloop; i++) {
                        client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
                    }
                    latch.countDown();
                }
            });
        }
        logger.info("waiting for max 30 seconds...");
        latch.await(30, TimeUnit.SECONDS);
        logger.info("flush...");
        client.flushIngest();
        client.waitForResponses(TimeValue.timeValueSeconds(30));
        logger.info("got all responses, thread pool shutdown...");
        pool.shutdown();
        logger.info("pool is shut down");
    } catch (NoNodeAvailableException e) {
        logger.warn("skipping, no node available");
    } finally {
        client.stopBulk("test");
        assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
        if (client.hasThrowable()) {
            logger.error("error", client.getThrowable());
        }
        assertFalse(client.hasThrowable());
        client.refreshIndex("test");
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
                .setQuery(QueryBuilders.matchAllQuery()).setSize(0);
        assertEquals(maxthreads * maxloop,
                searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
        client.shutdown();
    }
}
 

@Test
public void testThreadedRandomDocsIngestClient() throws Exception {
    int maxthreads = Runtime.getRuntime().availableProcessors();
    long maxactions = MAX_ACTIONS;
    final long maxloop = NUM_ACTIONS;
    Settings settings = Settings.settingsBuilder()
            .put("index.number_of_shards", 2)
            .put("index.number_of_replicas", 1)
            .build();
    final IngestTransportClient ingest = ClientBuilder.builder()
            .put(getSettings())
            .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions)
            .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
            .setMetric(new LongAdderIngestMetric())
            .toIngestTransportClient();
    try {
        ingest.newIndex("test", settings, null)
                .startBulk("test", -1, 1000);
        ThreadPoolExecutor pool =
                EsExecutors.newFixed("ingestclient-test", maxthreads, 30, EsExecutors.daemonThreadFactory("ingestclient-test"));
        final CountDownLatch latch = new CountDownLatch(maxthreads);
        for (int i = 0; i < maxthreads; i++) {
            pool.execute(new Runnable() {
                public void run() {
                    for (int i = 0; i < maxloop; i++) {
                        ingest.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
                    }
                    latch.countDown();
                }
            });
        }
        logger.info("waiting for max 30 seconds...");
        latch.await(30, TimeUnit.SECONDS);
        logger.info("client flush ...");
        ingest.flushIngest();
        ingest.waitForResponses(TimeValue.timeValueSeconds(30));
        logger.info("thread pool to be shut down ...");
        pool.shutdown();
        logger.info("thread pool shut down");
    } catch (NoNodeAvailableException e) {
        logger.warn("skipping, no node available");
    } finally {
        ingest.stopBulk("test");
        assertEquals(maxthreads * maxloop, ingest.getMetric().getSucceeded().getCount());
        if (ingest.hasThrowable()) {
            logger.error("error", ingest.getThrowable());
        }
        assertFalse(ingest.hasThrowable());
        ingest.refreshIndex("test");
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(ingest.client(), SearchAction.INSTANCE)
                .setIndices("_all") // to avoid NPE
                .setQuery(QueryBuilders.matchAllQuery())
                .setSize(0);
        assertEquals(maxthreads * maxloop,
                searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
        ingest.shutdown();
    }
}
 

@Test
public void testDuplicateDocIDs() throws Exception {
    long numactions = NUM_ACTIONS;
    final IngestTransportClient client = ClientBuilder.builder()
            .put(getSettings())
            .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
            .setMetric(new LongAdderIngestMetric())
            .toIngestTransportClient();
    try {
        client.newIndex("test");
        client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30));
        for (int i = 0; i < NUM_ACTIONS; i++) {
            client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
        }
        client.flushIngest();
        logger.info("flushed, waiting for responses");
        client.waitForResponses(TimeValue.timeValueSeconds(30));
        logger.info("refreshing");
        client.refreshIndex("test");
        logger.info("searching");
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
                .setIndices("test")
                .setTypes("test")
                .setQuery(matchAllQuery());
        long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
        logger.info("hits = {}", hits);
        assertTrue(hits < NUM_ACTIONS);
    } catch (NoNodeAvailableException e) {
        logger.warn("skipping, no node available");
    } catch (Throwable t) {
        logger.error("oops", t);
    } finally {
        logger.info("shutting down client");
        client.shutdown();
        assertEquals(numactions, client.getMetric().getSucceeded().getCount());
        if (client.hasThrowable()) {
            logger.error("error", client.getThrowable());
        }
        assertFalse(client.hasThrowable());
        logger.info("done");
    }
}
 

@Test
public void testThreadedRandomDocsBulkClient() throws Exception {
    int maxthreads = Runtime.getRuntime().availableProcessors();
    long maxactions = MAX_ACTIONS;
    final long maxloop = NUM_ACTIONS;

    Settings settingsForIndex = Settings.settingsBuilder()
            .put("index.number_of_shards", 2)
            .put("index.number_of_replicas", 1)
            .build();

    final BulkTransportClient client = ClientBuilder.builder()
            .put(getSettings())
            .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions)
            .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) // = disable autoflush for this test
            .setMetric(new LongAdderIngestMetric())
            .toBulkTransportClient();
    try {
        client.newIndex("test", settingsForIndex, null)
                .startBulk("test", -1, 1000);
        ThreadPoolExecutor pool =
                EsExecutors.newFixed("bulkclient-test", maxthreads, 30, EsExecutors.daemonThreadFactory("bulkclient-test"));
        final CountDownLatch latch = new CountDownLatch(maxthreads);
        for (int i = 0; i < maxthreads; i++) {
            pool.execute(new Runnable() {
                public void run() {
                    for (int i = 0; i < maxloop; i++) {
                        client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
                    }
                    latch.countDown();
                }
            });
        }
        logger.info("waiting for max 30 seconds...");
        latch.await(30, TimeUnit.SECONDS);
        logger.info("client flush ...");
        client.flushIngest();
        client.waitForResponses(TimeValue.timeValueSeconds(30));
        logger.info("thread pool to be shut down ...");
        pool.shutdown();
        logger.info("poot shut down");
    } catch (NoNodeAvailableException e) {
        logger.warn("skipping, no node available");
    } finally {
        client.stopBulk("test");
        assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
        if (client.hasThrowable()) {
            logger.error("error", client.getThrowable());
        }
        assertFalse(client.hasThrowable());
        client.refreshIndex("test");
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
                .setIndices("_all") // to avoid NPE at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:580)
                .setQuery(QueryBuilders.matchAllQuery())
                .setSize(0);
        assertEquals(maxthreads * maxloop,
                searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
        client.shutdown();
    }
}
 

@Test
public void testThreadedRandomDocsNodeClient() throws Exception {
    int maxthreads = Runtime.getRuntime().availableProcessors();
    Long maxactions = MAX_ACTIONS;
    final Long maxloop = NUM_ACTIONS;
    logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop);
    final BulkNodeClient client = ClientBuilder.builder()
            .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions)
            .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))// disable auto flush for this test
            .setMetric(new LongAdderIngestMetric())
            .toBulkNodeClient(client("1"));
    try {
        client.newIndex("test")
                .startBulk("test", -1, 1000);
        ThreadPoolExecutor pool = EsExecutors.newFixed("bulk-nodeclient-test", maxthreads, 30,
                EsExecutors.daemonThreadFactory("bulk-nodeclient-test"));
        final CountDownLatch latch = new CountDownLatch(maxthreads);
        for (int i = 0; i < maxthreads; i++) {
            pool.execute(new Runnable() {
                public void run() {
                    for (int i = 0; i < maxloop; i++) {
                        client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
                    }
                    latch.countDown();
                }
            });
        }
        logger.info("waiting for max 30 seconds...");
        latch.await(30, TimeUnit.SECONDS);
        logger.info("flush...");
        client.flushIngest();
        client.waitForResponses(TimeValue.timeValueSeconds(30));
        logger.info("got all responses, thread pool shutdown...");
        pool.shutdown();
        logger.info("pool is shut down");
    } catch (NoNodeAvailableException e) {
        logger.warn("skipping, no node available");
    } finally {
        client.stopBulk("test");
        assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
        if (client.hasThrowable()) {
            logger.error("error", client.getThrowable());
        }
        assertFalse(client.hasThrowable());
        client.refreshIndex("test");
        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
                .setQuery(QueryBuilders.matchAllQuery()).setSize(0);
        assertEquals(maxthreads * maxloop,
                searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
        client.shutdown();
    }
}