org.elasticsearch.search.Scroll Java Examples

public CrateSearchContext(long id,
                          final long nowInMillis,
                          SearchShardTarget shardTarget,
                          Engine.Searcher engineSearcher,
                          IndexService indexService,
                          final IndexShard indexShard,
                          ScriptService scriptService,
                          PageCacheRecycler pageCacheRecycler,
                          BigArrays bigArrays,
                          Counter timeEstimateCounter,
                          Optional<Scroll> scroll) {
    super(id, new CrateSearchShardRequest(nowInMillis, scroll, indexShard),
            shardTarget, engineSearcher, indexService,
            indexShard, scriptService, pageCacheRecycler,
            bigArrays, timeEstimateCounter, ParseFieldMatcher.STRICT, SearchService.NO_TIMEOUT);
    this.engineSearcher = engineSearcher;
}
 

@Override
public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
    String scrollId = request.param("scroll_id");
    SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
    searchScrollRequest.scrollId(scrollId);
    String scroll = request.param("scroll");
    if (scroll != null) {
        searchScrollRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
    }

    if (RestActions.hasBodyContent(request)) {
        XContentType type = XContentFactory.xContentType(RestActions.getRestContent(request));
        if (type == null) {
            if (scrollId == null) {
                scrollId = RestActions.getRestContent(request).toUtf8();
                searchScrollRequest.scrollId(scrollId);
            }
        } else {
            // NOTE: if rest request with xcontent body has request parameters, these parameters override xcontent values
            buildFromContent(RestActions.getRestContent(request), searchScrollRequest);
        }
    }
    client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<SearchResponse>(channel));
}
 

public static void buildFromContent(BytesReference content, SearchScrollRequest searchScrollRequest) {
    try (XContentParser parser = XContentHelper.createParser(content)) {
        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
            throw new IllegalArgumentException("Malforrmed content, must start with an object");
        } else {
            XContentParser.Token token;
            String currentFieldName = null;
            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                if (token == XContentParser.Token.FIELD_NAME) {
                    currentFieldName = parser.currentName();
                } else if ("scroll_id".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                    searchScrollRequest.scrollId(parser.text());
                } else if ("scroll".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                    searchScrollRequest.scroll(new Scroll(TimeValue.parseTimeValue(parser.text(), null, "scroll")));
                } else {
                    throw new IllegalArgumentException("Unknown parameter [" + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] ");
                }
            }
        }
    } catch (IOException e) {
        throw new IllegalArgumentException("Failed to parse request body", e);
    }
}
 

public CrateSearchContext createContext(
        int jobSearchContextId,
        IndexShard indexshard,
        Engine.Searcher engineSearcher,
        WhereClause whereClause) {

    ShardId shardId = indexshard.shardId();
    SearchShardTarget searchShardTarget = new SearchShardTarget(
            clusterService.state().nodes().localNodeId(),
            shardId.getIndex(),
            shardId.id()
    );
    IndexService indexService = indexshard.indexService();
    CrateSearchContext searchContext = new CrateSearchContext(
            jobSearchContextId,
            System.currentTimeMillis(),
            searchShardTarget,
            engineSearcher,
            indexService,
            indexshard,
            scriptService,
            pageCacheRecycler,
            bigArrays,
            threadPool.estimatedTimeInMillisCounter(),
            Optional.<Scroll>absent()
    );
    LuceneQueryBuilder.Context context = luceneQueryBuilder.convert(
            whereClause,  indexService.mapperService(), indexService.fieldData(), indexService.cache());
    searchContext.parsedQuery(new ParsedQuery(context.query(), EMPTY_NAMED_FILTERS));

    Float minScore = context.minScore();
    if (minScore != null) {
        searchContext.minimumScore(minScore);
    }

    return searchContext;
}
 

private CrateSearchShardRequest(long nowInMillis, Optional<Scroll> scroll,
                                IndexShard indexShard) {
    this.nowInMillis = nowInMillis;
    this.scroll = scroll.orNull();
    this.index = indexShard.indexService().index().name();
    this.shardId = indexShard.shardId().id();
}
 

/**
 * 获取符合条件的文档ID集合
 *
 * @param conditions 筛选条件
 * @param esObject   ES基础请求参数
 * @return 文档ID集合
 * @throws ExecutionException   es 执行异常
 * @throws InterruptedException es 执行异常
 */
private List<String> getAccordConditionDocIds(List<SearchCondition> conditions, BaseESObject esObject)
        throws ExecutionException, InterruptedException {

    QueryESObject queryESObject = new QueryESObject(esObject.getSystemName(), esObject.getIndexName(),
            esObject.getTypeName());
    queryESObject.setSearchConditions(conditions);

    final SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch().setIndices(esObject.getIndexName())
            .setTypes(esObject.getTypeName()).setExplain(explain).setSize(perShardSize).setFetchSource(Boolean.TRUE);

    queryConditionBuilder.build(searchRequestBuilder, queryESObject);
    searchRequestBuilder.setScroll(new Scroll(TimeValue.timeValueSeconds(scrollTime)));

    SearchLogger.log(searchRequestBuilder);
    final SearchResponse searchResponse = searchRequestBuilder.execute().get();
    SearchLogger.log(searchResponse);
    final SearchHits hits = searchResponse.getHits();
    if (hits.getHits().length == 0) {
        return null;
    }

    List<String> docIds = new ArrayList<>();
    for (SearchHit hit : hits) {
        docIds.add(hit.getId());
    }
    scrollSearch(searchResponse.getScrollId(), docIds, new ArrayList<>());
    return docIds;
}
 

public void emit() {
    // 构建查询
    Scroll esScroll = new Scroll(new TimeValue(scroll, TimeUnit.MINUTES));
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setScroll(esScroll).setQuery(query).setSize(size);
    if (StringUtils.isNotEmpty(type)) {
        searchRequestBuilder.setTypes(type);
    }

    if (logger.isDebugEnabled()) {
        logger.debug(searchRequestBuilder.toString());
    }

    String scrollId = null;
    try {
        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
        scrollId = searchResponse.getScrollId();

        long total = searchResponse.getHits().getTotalHits();
        // 处理第一次search查询
        int messageCount = submitMessage(searchResponse);

        // 计算剩余的批次
        long remainTotal = total - messageCount;
        long batchNum = remainTotal % size == 0 ? remainTotal / size : (remainTotal / size) + 1;

        for (long i = 0; !stop && i < batchNum; i++) {
            // 按批查询数据
            SearchScrollRequestBuilder scrollRequestBuilder = client.prepareSearchScroll(scrollId).setScroll(esScroll);
            searchResponse = scrollRequestBuilder.execute().actionGet();
            submitMessage(searchResponse);
        }
    } catch (Exception e) {
        logger.error("query error", e);
    } finally {
        if (StringUtils.isNotEmpty(scrollId)) {
            client.prepareClearScroll().addScrollId(scrollId).execute().actionGet();
        }
    }
}
 

public MoreLikeThisRequestBuilder<JsonInput, JsonOutput> searchScroll(String keepAlive) {
    request.searchScroll(new Scroll(TimeValue.parseTimeValue(keepAlive, null)));
    return this;
}
 

protected ElasticsearchDataModel createDataModel(final Client client,
        final IndexInfo indexInfo,
        final Map<String, Object> modelInfoSettings) {
    if (StringUtils.isBlank(indexInfo.getUserIndex())) {
        throw new TasteException("User Index is blank.");
    }
    if (StringUtils.isBlank(indexInfo.getPreferenceIndex())) {
        throw new TasteException("Preference Index is blank.");
    }
    if (StringUtils.isBlank(indexInfo.getItemIndex())) {
        throw new TasteException("Item Index is blank.");
    }

    final String className = SettingsUtils
            .get(modelInfoSettings, "class",
                    "org.codelibs.elasticsearch.taste.model.ElasticsearchDataModel");

    try {
        final Class<?> clazz = Class.forName(className);
        final ElasticsearchDataModel model = (ElasticsearchDataModel) clazz
                .newInstance();
        model.setClient(client);
        model.setPreferenceIndex(indexInfo.getPreferenceIndex());
        model.setPreferenceType(indexInfo.getPreferenceType());
        model.setUserIndex(indexInfo.getUserIndex());
        model.setUserType(indexInfo.getUserType());
        model.setItemIndex(indexInfo.getItemIndex());
        model.setItemType(indexInfo.getItemType());
        model.setUserIdField(indexInfo.getUserIdField());
        model.setItemIdField(indexInfo.getItemIdField());
        model.setValueField(indexInfo.getValueField());
        model.setTimestampField(indexInfo.getTimestampField());

        final Map<String, Object> scrollSettings = SettingsUtils.get(
                modelInfoSettings, "scroll");
        model.setScrollSize(((Number) SettingsUtils.get(scrollSettings,
                "size", 1000)).intValue());
        model.setScrollKeepAlive(new Scroll(TimeValue
                .timeValueSeconds(((Number) SettingsUtils.get(
                        scrollSettings, "keep_alive", 60L)).longValue())));

        final Map<String, Object> querySettings = SettingsUtils.get(
                modelInfoSettings, "query");
        final String userQuery = SettingsUtils.get(querySettings, "user");
        if (StringUtils.isNotBlank(userQuery)) {
            model.setUserQueryBuilder(QueryBuilders.queryStringQuery(userQuery));
        }
        final String itemQuery = SettingsUtils.get(querySettings, "item");
        if (StringUtils.isNotBlank(itemQuery)) {
            model.setUserQueryBuilder(QueryBuilders.queryStringQuery(itemQuery));
        }

        final Map<String, Object> cacheSettings = SettingsUtils.get(
                modelInfoSettings, "cache");
        final Object weight = SettingsUtils.get(cacheSettings, "weight");
        if (weight instanceof Number) {
            model.setMaxCacheWeight(((Integer) weight).longValue());
        } else {
            final long weightSize = parseWeight(weight.toString());
            if (weightSize > 0) {
                model.setMaxCacheWeight(weightSize);
            }
        }

        final Object maxPreferenceSize = SettingsUtils
                .get(modelInfoSettings, "max_preference_size");
        if (maxPreferenceSize instanceof Number) {
            final int size = ((Number) maxPreferenceSize).intValue();
            if (size > 0) {
                model.setMaxPreferenceSize(size);
            }
        }

        return model;
    } catch (ClassNotFoundException | InstantiationException
            | IllegalAccessException e) {
        throw new TasteException("Could not create an instance of "
                + className);
    }
}
 

public void setScrollKeepAlive(final Scroll scrollKeepAlive) {
    this.scrollKeepAlive = scrollKeepAlive;
}
 

/**
 * If set, will enable scrolling of the search request for the specified timeout.
 */
public SearchRequest scroll(String keepAlive) {
    return scroll(new Scroll(TimeValue.parseTimeValue(keepAlive, null, getClass().getSimpleName() + ".Scroll.keepAlive")));
}
 

/**
 * If set, will enable scrolling of the search request for the specified timeout.
 */
public SearchRequest scroll(TimeValue keepAlive) {
    return scroll(new Scroll(keepAlive));
}
 

/**
 * If set, will enable scrolling of the search request.
 */
public Scroll scroll() {
    return scroll;
}
 

/**
 * If set, will enable scrolling of the search request for the specified timeout.
 */
public SearchScrollRequest scroll(String keepAlive) {
    return scroll(new Scroll(TimeValue.parseTimeValue(keepAlive, null, getClass().getSimpleName() + ".keepAlive")));
}
 

/**
 * If set, will enable scrolling of the search request for the specified timeout.
 */
public SearchScrollRequest scroll(TimeValue keepAlive) {
    return scroll(new Scroll(keepAlive));
}
 

/**
 * If set, will enable scrolling of the search request.
 */
public Scroll scroll() {
    return scroll;
}
 

/**
 * If set, will enable scrolling of the search request.
 */
public SearchRequestBuilder setScroll(Scroll scroll) {
    request.scroll(scroll);
    return this;
}
 

/**
 * If set, will enable scrolling of the search request.
 */
public SearchScrollRequestBuilder setScroll(Scroll scroll) {
    request.scroll(scroll);
    return this;
}
 

public static void parseSearchRequest(SearchRequest searchRequest, RestRequest request, ParseFieldMatcher parseFieldMatcher,
        BytesReference bodyContent) {
    searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
    // get the content, and put it in the body
    // add content/source as template if template flag is set
    boolean isTemplateRequest = request.path().endsWith("/template");
    if (bodyContent == null) {
        if (RestActions.hasBodyContent(request)) {
            bodyContent = RestActions.getRestContent(request);
        }
    }
    if (bodyContent != null) {
        if (isTemplateRequest) {
            searchRequest.templateSource(bodyContent);
        } else {
            searchRequest.source(bodyContent);
        }
    }

    // do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types
    // from the REST layer. these modes are an internal optimization and should
    // not be specified explicitly by the user.
    String searchType = request.param("search_type");
    if (SearchType.fromString(searchType, parseFieldMatcher).equals(SearchType.QUERY_AND_FETCH) ||
            SearchType.fromString(searchType, parseFieldMatcher).equals(SearchType.DFS_QUERY_AND_FETCH)) {
        throw new IllegalArgumentException("Unsupported search type [" + searchType + "]");
    } else {
        searchRequest.searchType(searchType);
    }

    searchRequest.extraSource(parseSearchSource(request));
    searchRequest.requestCache(request.paramAsBoolean("request_cache", null));

    String scroll = request.param("scroll");
    if (scroll != null) {
        searchRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
    }

    searchRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
    searchRequest.routing(request.param("routing"));
    searchRequest.preference(request.param("preference"));
    searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
}
 

@Override
public Scroll scroll() {
    return shardSearchLocalRequest.scroll();
}
 

@Override
public Scroll scroll() {
    return scroll;
}
 

public Scroll scroll() {
    return scroll;
}
 

@Override
public Scroll scroll() {
    return scroll;
}
 

Scroll scroll();