Java Code Examples for org.elasticsearch.search.builder.SearchSourceBuilder#fetchSource()

private static SearchResponse search(AggregationBuilder aggregation) throws IOException {
    SearchRequest searchRequest = new SearchRequest();
    searchRequest.indices("student");
    searchRequest.types("_doc");
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    //不需要解释
    searchSourceBuilder.explain(false);
    //不需要原始数据
    searchSourceBuilder.fetchSource(false);
    //不需要版本号
    searchSourceBuilder.version(false);
    searchSourceBuilder.aggregation(aggregation);
    logger.info("查询的语句:"+searchSourceBuilder.toString());
    searchRequest.source(searchSourceBuilder);
    // 同步查询
    SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
    return  searchResponse;
}
 

public SearchResponse searchResources(SearchCriteria searchCriteria, List<SearchField> searchFields, List<SortField> sortFields, RestHighLevelClient client, String index) throws IOException {
    // See Also: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-query-builders.html
    QueryAccumulator queryAccumulator = new QueryAccumulator(searchCriteria);
    if (searchCriteria.getParamList() != null) {
        searchFields.stream().forEach(queryAccumulator::addQuery);
        // For now assuming there is only date field where range query will
        // be done. If we there are more than one, then we should create a
        // hashmap for each field name
        if (queryAccumulator.fromDate != null || queryAccumulator.toDate != null) {
            queryAccumulator.queries.add(setDateRange(queryAccumulator.dateFieldName, queryAccumulator.fromDate, queryAccumulator.toDate));
        }
    }
    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
    queryAccumulator.queries.stream().filter(x -> x != null).forEach(boolQueryBuilder::must);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    setSortClause(searchCriteria, sortFields, searchSourceBuilder);
    searchSourceBuilder.from(searchCriteria.getStartIndex());
    searchSourceBuilder.size(searchCriteria.getMaxRows());
    searchSourceBuilder.fetchSource(true);
    SearchRequest query = new SearchRequest();
    query.indices(index);
    query.source(searchSourceBuilder.query(boolQueryBuilder));
    return client.search(query, RequestOptions.DEFAULT);
}
 

static SearchRequest buildSuggestQuery(
    String prefix, OccurrenceSearchParameter parameter, Integer limit, String index) {
  SearchRequest request = new SearchRequest();
  request.indices(index);

  SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
  request.source(searchSourceBuilder);

  OccurrenceEsField esField = SEARCH_TO_ES_MAPPING.get(parameter);

  // create suggest query
  searchSourceBuilder.suggest(
      new SuggestBuilder()
          .addSuggestion(
              esField.getFieldName(),
              SuggestBuilders.completionSuggestion(esField.getFieldName() + ".suggest")
                  .prefix(prefix)
                  .size(limit != null ? limit : SearchConstants.DEFAULT_SUGGEST_LIMIT)
                  .skipDuplicates(true)));

  // add source field
  searchSourceBuilder.fetchSource(esField.getFieldName(), null);

  return request;
}
 

@Override
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
    if (!EnabledSetting.isADPluginEnabled()) {
        throw new IllegalStateException(CommonErrorMessages.DISABLED_ERR_MSG);
    }
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.parseXContent(request.contentOrSourceParamParser());
    searchSourceBuilder.fetchSource(getSourceContext(request));
    searchSourceBuilder.seqNoAndPrimaryTerm(true).version(true);
    SearchRequest searchRequest = new SearchRequest().source(searchSourceBuilder).indices(this.index);
    return channel -> client.search(searchRequest, search(channel, this.clazz));
}
 

protected void setIncludeExcludeFields(SearchSourceBuilder searchSourceBuilder,
                                       com.planet.staccato.dto.api.SearchRequest searchRequest) {
    // If include fieldsExtension were provided, make sure `collection` is present because it's needed for Jackson
    // deserialization to the proper ItemProperties subtype.  If the `collection` field was not requested in the
    // `fields` parameter, it will be removed before the response is returned.  Finally, set the includeFields on
    // the api.

    if (searchRequest.getFields() == null) {
        return;
    }

    Set<String> includeSet = null;
    Set<String> excludeSet = null;

    if (searchRequest.getFields() != null && searchRequest.getFields().getInclude() != null) {
        // don't want to alter the original set as it can be used in filters before the response is returned
        includeSet = new HashSet<>(searchRequest.getFields().getInclude());
    }

    if (searchRequest.getFields() != null && searchRequest.getFields().getExclude() != null) {
        excludeSet = searchRequest.getFields().getExclude();
    }

    if ((includeSet != null && !includeSet.isEmpty()) || (excludeSet != null && !excludeSet.isEmpty())) {
        // include fields takes preference, so remove any fields from the exclude set that are also in include
        if (excludeSet != null && includeSet != null) {
            excludeSet = new HashSet<>(excludeSet);
            excludeSet.removeAll(includeSet);
        }
        String[] include = null;
        String[] exclude = null;

        if (includeSet != null && !includeSet.isEmpty()) {
            includeSet.add("collection");
            include = includeSet.toArray(new String[includeSet.size()]);
        }
        if (excludeSet != null && !excludeSet.isEmpty()) {
            exclude = excludeSet.toArray(new String[excludeSet.size()]);
        }
        searchSourceBuilder.fetchSource(include, exclude);
    }

}
 

public static SearchSourceBuilder buildExperimentalSuggestionQuery(
            ExecutableSuggestionSearch search,
            DocumentFactory factory) {

        final String searchContext = search.getSearchContext();
        final SearchSourceBuilder searchSource = new SearchSourceBuilder();

        final BoolQueryBuilder baseQuery = QueryBuilders.boolQuery();

        final String[] suggestionFieldNames = Stream.of(getSuggestionFieldNames(search, factory, searchContext))
                .map(name -> name.concat("_experimental"))
                .toArray(String[]::new);

        final MultiMatchQueryBuilder suggestionQuery = QueryBuilders
                .multiMatchQuery(search.getInput(),suggestionFieldNames)
                .type(MultiMatchQueryBuilder.Type.BEST_FIELDS)
                .operator(Operator.OR);

        baseQuery.must(suggestionQuery);

//        if(search.getTimeZone() != null) {
//            query.set(CommonParams.TZ,search.getTimeZone());
//        }

        baseQuery.filter(buildFilterQuery(search.getFilter(), factory, searchContext));

        searchSource.query(baseQuery);

        final HighlightBuilder highlighter = new HighlightBuilder().numOfFragments(0);
        Stream.of(suggestionFieldNames)
                .forEach(highlighter::field);

        searchSource.highlighter(highlighter);
        searchSource.trackScores(SearchConfiguration.get(SearchConfiguration.SEARCH_RESULT_SHOW_SCORE, true));
        searchSource.fetchSource(true);

        //TODO if nested document search is implemented

        return searchSource;
    }
 

/**
 * Elasticsearch queries default to 10 records returned.  Some internal queries require that all
 * results are returned.  Rather than setting an arbitrarily high size, this method pages through results
 * and returns them all in a single SearchResponse.
 * @param qb A QueryBuilder that provides the query to be run.
 * @return A SearchResponse containing the appropriate results.
 */
public static  SearchResponse queryAllResults(RestHighLevelClient transportClient,
    QueryBuilder qb,
    String index,
    int pageSize
) throws IOException {
  org.elasticsearch.action.search.SearchRequest request = new org.elasticsearch.action.search.SearchRequest();
  SearchSourceBuilder builder = new SearchSourceBuilder();
  builder.query(qb);
  builder.size(pageSize);
  builder.fetchSource(true);
  builder.storedField("*");
  request.source(builder);
  request.indices(index);

  org.elasticsearch.action.search.SearchResponse esResponse = transportClient.search(request);
  List<SearchResult> allResults = getSearchResults(esResponse);
  long total = esResponse.getHits().getTotalHits();
  if (total > pageSize) {
    int pages = (int) (total / pageSize) + 1;
    for (int i = 1; i < pages; i++) {
      int from = i * pageSize;
      builder.from(from);
      esResponse = transportClient.search(request);
      allResults.addAll(getSearchResults(esResponse));
    }
  }
  SearchResponse searchResponse = new SearchResponse();
  searchResponse.setTotal(total);
  searchResponse.setResults(allResults);
  return searchResponse;
}
 

/**
 * Executes a query and applies the predicate to each result.
 *
 * @param downloadFileWork it's used to determine how to page through the results and the search query to be used
 * @param downloadFileWork it's used to determine how to page through the results and the search query to be used
 * @param resultHandler    predicate that process each result, receives as parameter the occurrence key
 */
public static void processQuery(DownloadFileWork downloadFileWork, Consumer<Occurrence> resultHandler) {

  // Calculates the amount of output records
  int nrOfOutputRecords = downloadFileWork.getTo() - downloadFileWork.getFrom();

  try {

    int recordCount = 0;
    // Creates a search request instance using the search request that comes in the fileJob
    SearchSourceBuilder searchSourceBuilder = createSearchQuery(downloadFileWork.getQuery());


    while (recordCount < nrOfOutputRecords) {

      searchSourceBuilder.size(recordCount + LIMIT > nrOfOutputRecords ? nrOfOutputRecords - recordCount : LIMIT);
      searchSourceBuilder.from(downloadFileWork.getFrom() + recordCount);
      searchSourceBuilder.fetchSource(null, "all"); //All field is not needed in the response
      SearchRequest searchRequest = new SearchRequest().indices(downloadFileWork.getEsIndex()).source(searchSourceBuilder);

      SearchResponse searchResponse = downloadFileWork.getEsClient().search(searchRequest, RequestOptions.DEFAULT);
      consume(searchResponse, resultHandler);

      SearchHit[] searchHits = searchResponse.getHits().getHits();
      recordCount += searchHits.length;

    }
  } catch (IOException ex) {
    throw Throwables.propagate(ex);
  }
}
 

/**
     * 查询文章数据
     */
    public List<Article> queryArticle(String keyword, String category) throws IOException {
        BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        //数据偏移量，下标从0开始，取前5个
        sourceBuilder.from(0);
        sourceBuilder.size(5);
        //指定返回结果的字段，第一个参数为包含的字段，第二个参数为排除的字段
        sourceBuilder.fetchSource(new String[]{"title"}, new String[]{});

        //分词匹配查询

        //且条件组合，必须满足每一项，计算相关性得分
        boolBuilder.must(QueryBuilders.matchQuery("title", keyword));
//        boolBuilder.must(QueryBuilders.matchQuery("content", keyword));

        //且条件组合，必须满足每一项，不计算相关性得分
//        boolBuilder.filter(QueryBuilders.matchQuery("title", keyword));
//        boolBuilder.filter(QueryBuilders.matchQuery("content", keyword));

        //或条件组合，满足任意一项即可
//        boolBuilder.should(QueryBuilders.matchQuery("title", keyword));
//        boolBuilder.should(QueryBuilders.matchQuery("content", keyword));

        //非条件组合，不满足每一项
//        boolBuilder.mustNot(QueryBuilders.matchQuery("title", keyword));
//        boolBuilder.mustNot(QueryBuilders.matchQuery("content", keyword));

        if (StringUtils.isNotBlank(category)) {
            //不分词匹配查询
            boolBuilder.must(QueryBuilders.matchPhraseQuery("category", category));

            //只匹配分词，如字段值为"Quick Foxes"，则只有"Quick"和"Foxes"可以匹配到值，"Quick Foxes"则不行
//            boolBuilder.must(QueryBuilders.termQuery("category", category));
        }

        //范围条件
        RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("publishTime");
        rangeQueryBuilder.gte("2018-12-31T00:00:00Z");
        rangeQueryBuilder.lte("2018-12-31T23:59:59Z");
        boolBuilder.must(rangeQueryBuilder);

        sourceBuilder.query(boolBuilder);
        SearchRequest searchRequest = new SearchRequest();
        //指定索引名称，如果未指定，则会查询所有
        searchRequest.indices(BLOG_INDEX);
        searchRequest.types(ARTICLE_TYPE);
        searchRequest.source(sourceBuilder);

        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);

        logger.info("查询数据 | {} | {} | {}", keyword, category, JSON.toJSONString(searchResponse));

        List<Article> articleList = Lists.newArrayList();
        SearchHit[] searchHits = searchResponse.getHits().getHits();
        for (SearchHit searchHit : searchHits) {
            Article article = new Article();
            article.setId(Long.valueOf(searchHit.getId()));
            article.setTitle((String) searchHit.getSourceAsMap().get("title"));

            articleList.add(article);
        }

        return articleList;
    }
 

/**
 * @return boolean
 * @Author pancm
 * @Description 根据条件查询
 * @Date 2019/3/21
 * @Param []
 **/
public static List<Map<String, Object>> query(String index, String type, EsQueryCondition esQueryCondition, QueryBuilder... queryBuilders) throws IOException {
    if (index == null || type == null) {
        return null;
    }
    List<Map<String, Object>> list = new ArrayList<>();
    try {
        // 查询指定的索引库
        SearchRequest searchRequest = new SearchRequest(index);
        searchRequest.types(type);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

        if (esQueryCondition != null) {
            Integer form = esQueryCondition.getIndex();
            Integer pagesize = esQueryCondition.getPagesize();
            if (form != null && form > 0 && pagesize != null && pagesize > 0) {
                form = (form - 1) * pagesize;
                pagesize = form + pagesize;
                // 设置起止和结束
                sourceBuilder.from(form);
                sourceBuilder.size(pagesize);
            }
            String routing = esQueryCondition.getRouting();
            if (routing != null && routing.length() > 0) {
                // 设置路由
                searchRequest.routing(routing);
            }

            // 设置索引库表达式
            searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen());

            //设置排序
            String order = esQueryCondition.getOrder();
            if (order != null) {
                String[] orderField = esQueryCondition.getOrderField();
                SortOrder order2 = order.equals(SortOrder.DESC) ? SortOrder.DESC : SortOrder.ASC;
                //如果设置了排序字段则用排序的字段进行排序，否则就默认排序
                if (orderField != null) {
                    for (String field : orderField) {
                        sourceBuilder.sort(new FieldSortBuilder(field).order(order2));
                    }
                } else {
                    sourceBuilder.sort(new ScoreSortBuilder().order(order2));
                }
            }
            String[] includeFields = esQueryCondition.getIncludeFields();
            String[] excludeFields = esQueryCondition.getExcludeFields();
            if (includeFields != null && includeFields.length > 0 && excludeFields != null && excludeFields.length > 0) {
                sourceBuilder.fetchSource(includeFields, excludeFields);
            }
            sourceBuilder.fetchSource(esQueryCondition.isCloseSource());
        }
        //设置条件
        if (queryBuilders != null) {
            for (QueryBuilder queryBuilder : queryBuilders) {
                sourceBuilder.query(queryBuilder);
            }
        }

        searchRequest.source(sourceBuilder);
        // 同步查询
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);

        if(queryBuilders != null|| (esQueryCondition != null && esQueryCondition.isQueryData())){
            // 结果
            searchResponse.getHits().forEach(hit -> {
                Map<String, Object> map = hit.getSourceAsMap();
                list.add(map);
            });
        }

          if(esQueryCondition != null && esQueryCondition.isNeedTotal()){
              Map<String, Object> mapTotal = new HashMap<>();
              mapTotal.put("total", searchResponse.getHits().getTotalHits());
              list.add(mapTotal);
          }

    } finally {
        if (isAutoClose) {
            close();
        }
    }
    return list;
}
 

@SuppressWarnings("unchecked")
public static void set(SearchSourceBuilder searchSource, Object value) {
    searchSource.fetchSource(((Collection<String>) value).toArray(new String[0]), new String[0]);
}
 

/**
 * Builds an Elasticsearch search request.
 * @param searchRequest The Metron search request.
 * @param queryBuilder
 * @return An Elasticsearch search request.
 */
private org.elasticsearch.action.search.SearchRequest buildSearchRequest(
    SearchRequest searchRequest,
    QueryBuilder queryBuilder) throws InvalidSearchException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Got search request; request={}", ElasticsearchUtils.toJSON(searchRequest).orElse("???"));
  }
  SearchSourceBuilder searchBuilder = new SearchSourceBuilder()
      .size(searchRequest.getSize())
      .from(searchRequest.getFrom())
      .query(queryBuilder)
      .trackScores(true);
  List<String> fields = searchRequest.getFields();
  // column metadata needed to understand the type of each sort field
  Map<String, FieldType> meta;
  try {
    meta = columnMetadataDao.getColumnMetadata(searchRequest.getIndices());
  } catch(IOException e) {
    throw new InvalidSearchException("Unable to get column metadata", e);
  }

  // handle sort fields
  for(SortField sortField : searchRequest.getSort()) {

    // what type is the sort field?
    FieldType sortFieldType = meta.getOrDefault(sortField.getField(), FieldType.OTHER);

    // sort order - if ascending missing values sorted last. otherwise, missing values sorted first
    org.elasticsearch.search.sort.SortOrder sortOrder = getElasticsearchSortOrder(sortField.getSortOrder());
    String missingSortOrder;
    if(sortOrder == org.elasticsearch.search.sort.SortOrder.DESC) {
      missingSortOrder = SORT_MISSING_LAST;
    } else {
      missingSortOrder = SORT_MISSING_FIRST;
    }

    // sort by the field - missing fields always last
    FieldSortBuilder sortBy = new FieldSortBuilder(sortField.getField())
        .order(sortOrder)
        .missing(missingSortOrder)
        .unmappedType(sortFieldType.getFieldType());
    searchBuilder.sort(sortBy);
  }

  // handle search fields
  if (fields != null) {
    searchBuilder.fetchSource("*", null);
  } else {
    searchBuilder.fetchSource(true);
  }

  List<String> facetFields = searchRequest.getFacetFields();

  // handle facet fields
  if (facetFields != null) {
    // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html
    for(String field : facetFields) {
      String name = getFacetAggregationName(field);
      TermsAggregationBuilder terms = AggregationBuilders.terms( name).field(field);
      // new TermsBuilder(name).field(field);
      searchBuilder.aggregation(terms);
    }
  }

  // return the search request
  String[] indices = wildcardIndices(searchRequest.getIndices());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString());
  }
  return new org.elasticsearch.action.search.SearchRequest()
      .indices(indices)
      .source(searchBuilder);
}
 

/**
 * {@inheritDoc}
 */
@Override
public Object doGet(final DataFetchingEnvironment env) {
    Field field = env.getMergedField().getSingleField();
    String fieldName = field.getName();

    // Get arguments for pagination & sorting
    int offset = Optional.ofNullable(env.<Integer>getArgument(ARG_NAME_OFFSET)).orElse(0);
    int limit = Optional.ofNullable(env.<Integer>getArgument(ARG_NAME_LIMIT)).orElse(defaultLimit);
    String sortBy = Optional.ofNullable(env.<String>getArgument(ARG_NAME_SORT_BY)).orElse(defaultSortField);
    String sortOrder = Optional.ofNullable(env.<String>getArgument(ARG_NAME_SORT_ORDER)).orElse(defaultSortOrder);

    List<String> queryFieldIncludes = new LinkedList<>();
    // Add content-type to includes, we might need it for a GraphQL TypeResolver
    queryFieldIncludes.add(QUERY_FIELD_NAME_CONTENT_TYPE);

    List<Map<String, Object>> items = new LinkedList<>();
    Map<String, Object> result = new HashMap<>(2);
    result.put(FIELD_NAME_ITEMS, items);

    // Setup the ES query
    SearchSourceBuilder source = new SearchSourceBuilder();
    BoolQueryBuilder query = boolQuery();
    source
        .query(query)
        .from(offset)
        .size(limit)
        .sort(sortBy, SortOrder.fromString(sortOrder));

    StopWatch watch = new StopWatch(field.getName() + " - " + field.getAlias());

    watch.start("build filters");

    // Filter by the content-type
    switch (fieldName) {
        case FIELD_NAME_CONTENT_ITEMS:
            query.filter(existsQuery(QUERY_FIELD_NAME_CONTENT_TYPE));
            break;
        case FIELD_NAME_PAGES:
            query.filter(regexpQuery(QUERY_FIELD_NAME_CONTENT_TYPE, CONTENT_TYPE_REGEX_PAGE));
            break;
        case FIELD_NAME_COMPONENTS:
            query.filter(regexpQuery(QUERY_FIELD_NAME_CONTENT_TYPE, CONTENT_TYPE_REGEX_COMPONENT));
            break;
        default:
            // Get the content-type name from the field name
            query.filter(termQuery(QUERY_FIELD_NAME_CONTENT_TYPE, getOriginalName(fieldName)));
            break;
    }

    // Check the selected fields to build the ES query
    Optional<Field> itemsField = field.getSelectionSet().getSelections()
                        .stream()
                        .map(f -> (Field) f)
                        .filter(f -> f.getName().equals(FIELD_NAME_ITEMS))
                        .findFirst();
    if (itemsField.isPresent()) {
        List<Selection> selections = itemsField.get().getSelectionSet().getSelections();
        selections.forEach(selection -> processSelection(StringUtils.EMPTY, selection, query, queryFieldIncludes,
         env));
    }

    // Only fetch the selected fields for better performance
    source.fetchSource(queryFieldIncludes.toArray(new String[0]), new String[0]);
    watch.stop();

    logger.debug("Executing query: {}", source);

    watch.start("searching items");
    SearchResponse response = elasticsearch.search(new SearchRequest().source(source));
    watch.stop();

    watch.start("processing items");
    result.put(FIELD_NAME_TOTAL, response.getHits().totalHits);
    if (response.getHits().totalHits > 0) {
        for(SearchHit hit :  response.getHits().getHits()) {
            items.add(fixItems(hit.getSourceAsMap()));
        }
    }
    watch.stop();

    if (logger.isTraceEnabled()) {
        logger.trace(watch.prettyPrint());
    }

    return result;
}