Java Code Examples for org.elasticsearch.action.search.SearchResponse#getScrollId()

/**
     * 处理scroll结果
     * @Author lihaodong
     * @Description
     * @Date 20:18 2018/12/21
     * @Param [response, highlightField]
     * @return java.util.List<java.util.Map<java.lang.String,java.lang.Object>>
     **/
    private static   List<Map<String, Object>>   disposeScrollResult(SearchResponse response ,String highlightField){
        List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>();
        //使用scrollId迭代查询
        while (response.getHits().getHits().length > 0) {
            String scrollId = response.getScrollId();
            response = client.prepareSearchScroll(scrollId)
                    .setScroll(TimeValue.timeValueMinutes(1))//设置查询context的存活时间
                    .execute()
                    .actionGet();
            SearchHits hits = response.getHits();
            for (SearchHit hit : hits.getHits()) {
                Map<String, Object> resultMap =getResultMap(hit, highlightField);
                sourceList.add(resultMap);
//                System.out.println(JSON.toJSONString(resultMap));
            }
        }
        ClearScrollRequest request = new ClearScrollRequest();
        request.addScrollId(response.getScrollId());
        client.clearScroll(request);
        return sourceList;
    }
 

@Override
public <T> AggregatedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pageable pageable) {
    List<T> results = new ArrayList<>();
    for (SearchHit hit : response.getHits()) {
        if (hit != null) {
            T result = null;
            if (StringUtils.hasText(hit.getSourceAsString())) {
                result = JSONObject.parseObject(hit.getSourceAsString(), clazz);
            }
            // 高亮查询
            for (HighlightField field : hit.getHighlightFields().values()) {
                try {
                    PropertyUtils.setProperty(result, field.getName(), concat(field.fragments()));
                } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
                    log.error("设置高亮字段异常：{}", e.getMessage(), e);
                }
            }
            results.add(result);
        }
    }
    return new AggregatedPageImpl<T>(results, pageable, response.getHits().getTotalHits(), response.getAggregations(), response.getScrollId());
}
 

@Override
public <T> AggregatedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pageable pageable) {

	long totalHits = response.getHits().getTotalHits();
	float maxScore = response.getHits().getMaxScore();

	List<Torrent> results = new ArrayList<>();
	for (SearchHit hit : response.getHits().getHits()) {
		if (hit == null)
			continue;
		Torrent result;
		result = JSONUtil.parseObject(hit.getSourceAsString(), Torrent.class);
		result.setInfoHash(hit.getId());
		if (hit.getHighlightFields().containsKey("fileName"))
			result.setFileName(hit.getHighlightFields().get("fileName").fragments()[0].toString());
		else
			result.setFileName((String) hit.getSourceAsMap().get("fileName"));
		results.add(result);
	}
	return new AggregatedPageImpl<>((List<T>) results, pageable, totalHits, response.getAggregations(), response.getScrollId(),
			maxScore);
}
 

@Override
public Stream<? extends Entity> scroll(int numSlice, int nbSlices) throws IOException {
    sourceBuilder.query(boolQuery);
    if (nbSlices > 1) {
        sourceBuilder.slice(new SliceBuilder(numSlice, nbSlices));
    }
    SearchResponse search;
    if (scrollId == null) {
        SearchRequest searchRequest = new SearchRequest(new String[]{indexName}, sourceBuilder).scroll(KEEP_ALIVE);
        searchRequest.types(config.indexType);
        search = client.search(searchRequest);
        scrollId = search.getScrollId();
        totalHits = search.getHits().totalHits;
    } else {
        search = client.searchScroll(new SearchScrollRequest(scrollId).scroll(KEEP_ALIVE));
        scrollId = search.getScrollId();
    }
    return resultStream(this.cls, () -> search.getHits().iterator());
}
 

@Override
public <T> AggregatedPage<T> mapResults(SearchResponse response, Class<T> aClass,
    Pageable pageable) {
  List<GlobalTransactionDocument> result = new ArrayList<>();
  for (SearchHit hit : response.getHits()) {
    if (response.getHits().getHits().length <= 0) {
      return new AggregatedPageImpl<T>(Collections.EMPTY_LIST, pageable,
          response.getHits().getTotalHits(), response.getScrollId());
    }
    GlobalTransactionDocument globalTransactionDocument = null;
    try {
      globalTransactionDocument = mapper.readValue(hit.getSourceAsString(),
          GlobalTransactionDocument.class);
    } catch (IOException e) {
      throw new RuntimeException(e.getMessage(), e);
    }
    result.add(globalTransactionDocument);
  }
  if (result.isEmpty()) {
    return new AggregatedPageImpl<T>(Collections.EMPTY_LIST, pageable,
        response.getHits().getTotalHits(), response.getScrollId());
  }
  return new AggregatedPageImpl<T>((List<T>) result, pageable,
      response.getHits().getTotalHits(), response.getScrollId());
}
 

protected void delegateBulkRequest(final ConditionBean cb, Function<SearchHits, Boolean> handler) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForCursor).setSize(sizeForCursor);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        builder.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            if (!handler.apply(searchHits)) {
                break;
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
            scrollId = response.getScrollId();
        }
    } finally {
        deleteScrollContext(scrollId);
    }
}
 

@Override
public AsyncFuture<Void> transform(final SearchResponse response) {
    final SearchHit[] hits = response.getHits().getHits();

    final Set<T> batch = new HashSet<>();

    for (final SearchHit hit : hits) {
        if (limit.isGreaterOrEqual(size)) {
            break;
        }

        batch.add(converter.apply(hit));
        size += 1;
    }

    if (hits.length == 0 || limit.isGreaterOrEqual(size)) {
        return seriesFunction.apply(batch);
    }

    final String scrollId = response.getScrollId();
    if (scrollId == null) {
        return seriesFunction.apply(batch);
    }

    // Fetch the next page in the scrolling search. The result will be handled by a new call
    // to this method.
    return seriesFunction
        .apply(batch)
        .lazyTransform(v -> searchFactory.apply(scrollId).get().lazyTransform(this));
}
 

public void emit() {
    // 构建查询
    Scroll esScroll = new Scroll(new TimeValue(scroll, TimeUnit.MINUTES));
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setScroll(esScroll).setQuery(query).setSize(size);
    if (StringUtils.isNotEmpty(type)) {
        searchRequestBuilder.setTypes(type);
    }

    if (logger.isDebugEnabled()) {
        logger.debug(searchRequestBuilder.toString());
    }

    String scrollId = null;
    try {
        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
        scrollId = searchResponse.getScrollId();

        long total = searchResponse.getHits().getTotalHits();
        // 处理第一次search查询
        int messageCount = submitMessage(searchResponse);

        // 计算剩余的批次
        long remainTotal = total - messageCount;
        long batchNum = remainTotal % size == 0 ? remainTotal / size : (remainTotal / size) + 1;

        for (long i = 0; !stop && i < batchNum; i++) {
            // 按批查询数据
            SearchScrollRequestBuilder scrollRequestBuilder = client.prepareSearchScroll(scrollId).setScroll(esScroll);
            searchResponse = scrollRequestBuilder.execute().actionGet();
            submitMessage(searchResponse);
        }
    } catch (Exception e) {
        logger.error("query error", e);
    } finally {
        if (StringUtils.isNotEmpty(scrollId)) {
            client.prepareClearScroll().addScrollId(scrollId).execute().actionGet();
        }
    }
}
 

public RetrievedResult retrieve(RequestContext requestContext) {
    final JsonNode requestBody = requestContext.getRequestBody();
    boolean setScroll = JsonHelpers.getOptionalBoolean(requestBody, setScrollKey, false);
    if (setScroll && scrollComplete) {
        return new RetrievedResult(new ArrayList<>(), 0);
    }
    JsonNode elasticSearchRequest;
    if (requestBody.has(elasticSearchReqKey)) {
        elasticSearchRequest = JsonHelpers.getRequiredJson(requestBody, elasticSearchReqKey);
    } else if (matchFields.length > 0 && requestBody.has(queryKey)) {
        String query = JsonHelpers.getRequiredString(requestBody, queryKey);
        elasticSearchRequest = Json.parse(QueryBuilders.multiMatchQuery(query, matchFields).toString());
    } else if (defaultMatchAll || setScroll) {
        elasticSearchRequest = Json.parse(QueryBuilders.matchAllQuery().toString());
    } else {
        return new RetrievedResult(new ArrayList<>(), 0);
    }
    Integer size = JsonHelpers.getOptionalInt(elasticSearchRequest, "size", null);
    Integer from = JsonHelpers.getOptionalInt(elasticSearchRequest, "from", null);
    SearchResponse searchResponse = elasticSearchService
            .search(elasticSearchIndex,
                    retrieveType,
                    elasticSearchRequest,
                    retrieveFields,
                    setScroll,
                    scrollId, size, from);
    RetrievedResult initialResult = ESRetrieverUtilities.parse(elasticSearchScoreName, requestContext,
            searchResponse, expanders, retrieveFields);
    if (setScroll) {
        scrollId = searchResponse.getScrollId();
        if (initialResult.getEntityList().size() == 0) {
            elasticSearchService.resetScroll(scrollId);
            scrollComplete = true;
        }
    }
    return initialResult;
}
 

protected void delegateBulkRequest(final ConditionBean cb, Function<SearchHits, Boolean> handler) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForCursor).setSize(sizeForCursor);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        builder.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            if (!handler.apply(searchHits)) {
                break;
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
            scrollId = response.getScrollId();
        }
    } finally {
        deleteScrollContext(scrollId);
    }
}
 

/**
 * 开始滚动数据
 *
 * @param queryBuilder 查询句柄
 * @return 滚动id和当前的一批数据
 */
public Pair<String, List<Webpage>> startScroll(QueryBuilder queryBuilder, int size) {
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(queryBuilder)
            .setSize(size)
            .setScroll(TimeValue.timeValueMinutes(SCROLL_TIMEOUT));
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    return new MutablePair<>(response.getScrollId(), warpHits2List(response.getHits()));
}
 

/**
 * 开始滚动数据
 *
 * @param queryBuilder 查询句柄
 * @return 滚动id和当前的一批数据
 */
public Pair<String, List<Webpage>> startScroll(QueryBuilder queryBuilder, int size) {
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(queryBuilder)
            .setSize(size)
            .setScroll(TimeValue.timeValueMinutes(SCROLL_TIMEOUT));
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    return new MutablePair<>(response.getScrollId(), warpHits2List(response.getHits()));
}
 

@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}
 

@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}
 

@Test
public void scrollQuery() throws IOException {
    QueryCondition queryCondition = new QueryCondition();
    queryCondition.setQueryBuilder(QueryBuilders.matchAllQuery());
    queryCondition.setSize(10);

    SearchResponse scrollResponse = elasticsearchTemplate.executeQuery("es_test", queryCondition, "type");

    String scrollId = scrollResponse.getScrollId();
    long total = scrollResponse.getHits().totalHits;
    log.info("\nscrollId is:{}\n", scrollId);
    log.info("\ntotal is:{}\n", total);

    List<Spu> content;
    if (total < 10) {
        content = elasticsearchTemplate.analyzeSearchResponse(Spu.class, scrollResponse);
        assertThat(content.size(), is(3));
    } else {
        for (int i = 0, sum = 0; sum < total; i++) {
            content = elasticsearchTemplate.analyzeSearchResponse(Spu.class, esClient.prepareSearchScroll(scrollId)
                .setScroll(TimeValue.timeValueMinutes(8)).execute().actionGet());

            sum += 10;
            log.info("\n总量{} 已经查到{}", total, sum);
            assertThat(content.size(), is(10));
        }
    }
}
 

public long updateByQuery(final String index, final Function<SearchRequestBuilder, SearchRequestBuilder> option,
        final BiFunction<UpdateRequestBuilder, SearchHit, UpdateRequestBuilder> builder) {

    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    SearchResponse response =
            option.apply(
                    client.prepareSearch(index).setScroll(scrollForUpdate).setSize(sizeForUpdate)
                            .setPreference(Constants.SEARCH_PREFERENCE_LOCAL)).execute()
                    .actionGet(fessConfig.getIndexScrollSearchTimeout());

    int count = 0;
    String scrollId = response.getScrollId();
    try {
        while (scrollId != null) {
            final SearchHits searchHits = response.getHits();
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                final UpdateRequestBuilder requestBuilder =
                        builder.apply(client.prepareUpdate().setIndex(index).setId(hit.getId()), hit);
                if (requestBuilder != null) {
                    bulkRequest.add(requestBuilder);
                }
                count++;
            }
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(fessConfig.getIndexBulkTimeout());
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response =
                    client.prepareSearchScroll(scrollId).setScroll(scrollForUpdate).execute()
                            .actionGet(fessConfig.getIndexBulkTimeout());
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
            scrollId = response.getScrollId();
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}
 

public <T> long scrollSearch(final String index, final SearchCondition<SearchRequestBuilder> condition,
        final EntityCreator<T, SearchResponse, SearchHit> creator, final BooleanFunction<T> cursor) {
    long count = 0;

    final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setScroll(scrollForSearch);
    if (condition.build(searchRequestBuilder)) {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();

        String scrollId = null;
        try {
            if (logger.isDebugEnabled()) {
                logger.debug("Query DSL:\n{}", searchRequestBuilder);
            }
            SearchResponse response = searchRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexSearchTimeout());

            scrollId = response.getScrollId();
            while (scrollId != null) {
                final SearchHits searchHits = response.getHits();
                final SearchHit[] hits = searchHits.getHits();
                if (hits.length == 0) {
                    break;
                }

                for (final SearchHit hit : hits) {
                    count++;
                    if (!cursor.apply(creator.build(response, hit))) {
                        if (scrollId != null) {
                            client.prepareClearScroll().addScrollId(scrollId)
                                    .execute(ActionListener.wrap(res -> {}, e1 -> logger.warn("Failed to clear scrollId.", e1)));
                        }
                        break;
                    }
                }

                response =
                        client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute()
                                .actionGet(fessConfig.getIndexBulkTimeout());
                scrollId = response.getScrollId();
            }
        } catch (final SearchPhaseExecutionException e) {
            if (scrollId != null) {
                client.prepareClearScroll().addScrollId(scrollId)
                        .execute(ActionListener.wrap(res -> {}, e1 -> logger.warn("Failed to clear scrollId.", e1)));
            }
            throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY),
                    "Invalid query: " + searchRequestBuilder, e);
        }
    }

    return count;
}
 

@Override
public AsyncFuture<SearchTransformResult<T>> transform(final SearchResponse response) {
    final SearchHit[] hits = response.getHits().getHits();
    final String scrollId = response.getScrollId();

    SearchHit lastHit = null;
    reporter.failedShards(response.getFailedShards());

    for (final SearchHit hit : hits) {

        final T convertedHit = converter.apply(hit);

        if (!results.add(convertedHit)) {
            duplicates += 1;
        } else {
            size += 1;
        }

        if (limit.isGreater(size)) {
            results.remove(convertedHit);
            return async.resolved(
                new SearchTransformResult<>(limit.limitSet(results), true, scrollId));
        }

        lastHit = hit;
    }

    if (hits.length == 0) {
        return async.resolved(new SearchTransformResult<>(results, false, scrollId));
    }

    if (scrolling && scrollId == null) {
        return async.resolved(SearchTransformResult.of());
    }

    // Fetch the next page in the scrolling search. The result will be handled by a new call
    // to this method.
    final Supplier<AsyncFuture<SearchResponse>> scroller =
        searchFactory.apply(scrollId, lastHit);
    return scroller.get().lazyTransform(this);
}
 

public JdbcSearchResponse parseScrollSearchResponse(SearchResponse response,Map<String,String> aliasMap) {
    JdbcSearchResponse searchResponse = parseSearchResponse(response,aliasMap);
    String scrollId = response.getScrollId();
    return new JdbcScrollSearchResponse(searchResponse, scrollId);
}
 

private void reset(SearchResponse response)
{
    scrollId = response.getScrollId();
    searchHits = response.getHits();
    currentPosition = 0;
}