org.elasticsearch.search.aggregations.bucket.terms.Terms Java Examples

/**
 * 获取query的关联信息
 *
 * @param query 查询queryString
 * @param size  结果集数量
 * @return 相关信息
 */
public Pair<Map<String, List<Terms.Bucket>>, List<Webpage>> relatedInfo(String query, int size) {
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(QueryBuilders.queryStringQuery(query))
            .addSort("gatherTime", SortOrder.DESC)
            .addAggregation(AggregationBuilders.terms("relatedPeople").field("namedEntity.nr"))
            .addAggregation(AggregationBuilders.terms("relatedLocation").field("namedEntity.ns"))
            .addAggregation(AggregationBuilders.terms("relatedInstitution").field("namedEntity.nt"))
            .addAggregation(AggregationBuilders.terms("relatedKeywords").field("keywords"))
            .setSize(size);
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    Map<String, List<Terms.Bucket>> info = Maps.newHashMap();
    info.put("relatedPeople", ((Terms) response.getAggregations().get("relatedPeople")).getBuckets());
    info.put("relatedLocation", ((Terms) response.getAggregations().get("relatedLocation")).getBuckets());
    info.put("relatedInstitution", ((Terms) response.getAggregations().get("relatedInstitution")).getBuckets());
    info.put("relatedKeywords", ((Terms) response.getAggregations().get("relatedKeywords")).getBuckets());
    return Pair.of(info, warpHits2List(response.getHits()));
}
 

@Override
TotalSalesQuery.Result extractResult(SearchResponse searchResponse) {
    TotalSalesQuery.Result.ResultBuilder result = TotalSalesQuery.Result.builder();
    long count = 0;
    BigDecimal amount = BigDecimal.ZERO;
    Filter filterAgg = searchResponse.getAggregations().get("agg_filter");
    Terms products = filterAgg.getAggregations().get("count_by_product");
    for (Terms.Bucket b : products.getBuckets()){
        count += b.getDocCount();
        Sum sum = b.getAggregations().get("total_premium");
        amount = amount.add(BigDecimal.valueOf(sum.getValue()).setScale(2,BigDecimal.ROUND_HALF_UP));
        result.productTotal(b.getKeyAsString(), SalesResult.of(b.getDocCount(),BigDecimal.valueOf(sum.getValue())));
    }
    result.total(SalesResult.of(count,amount));

    return result.build();
}
 

private static ImmutableList<ImmutableMap<String, Object>> getSoyRankingList(
    Terms aggregation, long totalCount) {
    ImmutableList.Builder<ImmutableMap<String, Object>> builder = ImmutableList.builder();
    long otherCount = totalCount;
    for (Terms.Bucket bucket : aggregation.getBuckets()) {
        String key = bucket.getKey();
        long count = bucket.getDocCount();
        otherCount -= count;
        builder.add(ImmutableMap.<String, Object> of(
            "key", key,
            "count", count,
            "proportion", ((double) count) / totalCount));
    }
    if (otherCount > 0) {
        builder.add(ImmutableMap.<String, Object> of(
            "other", true,
            "key", "Other",
            "count", otherCount,
            "proportion", ((double) otherCount / totalCount)));
    }
    return builder.build();
}
 

/**
 * Returns the aggregation that is associated with the specified name. This method also validates that the retrieved aggregation exists.
 *
 * @param searchResponse the response of the search request
 * @param aggregationName the name of the aggregation
 *
 * @return the aggregation
 */
public Terms getAggregation(SearchResponse searchResponse, String aggregationName)
{
    // Retrieve the aggregations from the search response.
    Aggregations aggregations = getAggregationsFromSearchResponse(searchResponse);

    // Retrieve the specified aggregation.
    Terms aggregation = aggregations.get(aggregationName);

    // Fail if retrieved aggregation is null.
    if (aggregation == null)
    {
        // Log the error along with the search response contents.
        LOGGER.error("Failed to retrieve \"{}\" aggregation from the search response. searchResponse={}", aggregationName,
            jsonHelper.objectToJson(searchResponse));

        // Throw an exception.
        throw new IllegalStateException("Invalid search result.");
    }

    return aggregation;
}
 

@Test
public void testGetResultTypeIndexSearchResponseDto()
{
    SearchResponse searchResponse = mock(SearchResponse.class);
    Terms terms = mock(Terms.class);
    Aggregations aggregations = mock(Aggregations.class);
    Terms.Bucket bucket = mock(Terms.Bucket.class);
    List<Terms.Bucket> buckets = Collections.singletonList(bucket);

    when(searchResponse.getAggregations()).thenReturn(aggregations);
    when(aggregations.get(RESULT_TYPE_AGGS)).thenReturn(terms);
    when(terms.getBuckets()).thenReturn(buckets);
    when(bucket.getKeyAsString()).thenReturn(TAG_CODE);
    when(bucket.getDocCount()).thenReturn(TAG_COUNT);

    List<ResultTypeIndexSearchResponseDto> expectedList = new ArrayList<>();
    expectedList.add(new ResultTypeIndexSearchResponseDto(TAG_CODE, TAG_COUNT, TAG_CODE));
    List<ResultTypeIndexSearchResponseDto> resultList = elasticsearchHelper.getResultTypeIndexSearchResponseDto(searchResponse);

    assertEquals(expectedList, resultList);
}
 

@Test
public void groupByTest() throws Exception {
	Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender", TEST_INDEX_ACCOUNT));
	Terms gender = result.get("gender");
	for(Terms.Bucket bucket : gender.getBuckets()) {
		String key = bucket.getKey().toString();
		long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
		if(key.equalsIgnoreCase("m")) {
			Assert.assertEquals(507, count);
		}
		else if(key.equalsIgnoreCase("f")) {
			Assert.assertEquals(493, count);
		}
		else {
			throw new Exception(String.format("Unexpected key. expected: m OR f. found: %s", key));
		}
	}
}
 

@Test
public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath() throws Exception {
    Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/nestedType GROUP BY  nested(message.info),filter('myFilter',message.info = 'a'),reverse_nested(comment.data,'~comment')", TEST_INDEX_NESTED_TYPE));
    InternalNested nested = result.get("message.info@NESTED");
    InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
    Terms infos = filter.getAggregations().get("message.info");
    Assert.assertEquals(1,infos.getBuckets().size());
    for(Terms.Bucket bucket : infos.getBuckets()) {
        InternalReverseNested reverseNested = bucket.getAggregations().get("comment.data@NESTED_REVERSED");
        InternalNested innerNested = reverseNested.getAggregations().get("comment.data@NESTED");
        Terms terms = innerNested.getAggregations().get("comment.data");
        Terms.Bucket internalBucket = terms.getBuckets().get(0);

        long count = ((ValueCount) internalBucket.getAggregations().get("COUNT(*)")).getValue();
        String key = internalBucket.getKey().toString();
        if(key.equalsIgnoreCase("ab")) {
            Assert.assertEquals(2, count);
        }
        else {
            throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
        }
    }
}
 

private List<GroupResult> getGroupResults(GroupRequest groupRequest, int index, Aggregations aggregations, Map<String, FieldType> commonColumnMetadata) {
  List<Group> groups = groupRequest.getGroups();
  String field = groups.get(index).getField();
  List<GroupResult> searchResultGroups = new ArrayList<>();
  if(aggregations != null) {
    Terms terms = aggregations.get(getGroupByAggregationName(field));
    for (Bucket bucket : terms.getBuckets()) {
      GroupResult groupResult = new GroupResult();
      groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field)));
      groupResult.setTotal(bucket.getDocCount());
      Optional<String> scoreField = groupRequest.getScoreField();
      if (scoreField.isPresent()) {
        Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
        groupResult.setScore(score.getValue());
      }
      if (index < groups.size() - 1) {
        groupResult.setGroupedBy(groups.get(index + 1).getField());
        groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata));
      }
      searchResultGroups.add(groupResult);
    }
  }
  return searchResultGroups;
}
 

@Test
public void multipleGroupByTest() throws Exception {
	Set expectedAges = new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));

	Map<String, Set<Integer>> buckets = new HashMap<>();

	Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender,  terms('field'='age','size'=200,'alias'='age')", TEST_INDEX_ACCOUNT));
	Terms gender = result.get("gender");
	for(Terms.Bucket genderBucket : gender.getBuckets()) {
		String genderKey = genderBucket.getKey().toString();
		buckets.put(genderKey, new HashSet<Integer>());
		Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
		for(Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
			buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey().toString()));
		}
	}

	Assert.assertEquals(2, buckets.keySet().size());
	Assert.assertEquals(expectedAges, buckets.get("m"));
	Assert.assertEquals(expectedAges, buckets.get("f"));
}
 

@Test
public void multipleGroupBysWithSize() throws Exception {
    Set expectedAges = new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));

    Map<String, Set<Integer>> buckets = new HashMap<>();

    Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender, terms('alias'='ageAgg','field'='age','size'=3)", TEST_INDEX_ACCOUNT));
    Terms gender = result.get("gender");
    Assert.assertEquals(2,gender.getBuckets().size());
    for(Terms.Bucket genderBucket : gender.getBuckets()) {

        String genderKey = genderBucket.getKey().toString();
        buckets.put(genderKey, new HashSet<Integer>());
        Terms ageBuckets = genderBucket.getAggregations().get("ageAgg");
        Assert.assertEquals(3,ageBuckets.getBuckets().size());

    }


}
 

private void handleSecondPhaseMultiSearchResponse(MultiSearchResponse multiResponse, String table,
                                                  Map<String, EstimationData> estimationDataMap) {
    for(MultiSearchResponse.Item item : multiResponse.getResponses()) {
        SearchResponse response = validateAndGetSearchResponse(item, table);
        if(null == response) {
            continue;
        }
        final long hits = response.getHits()
                .getTotalHits();
        Map<String, Aggregation> output = response.getAggregations().asMap();
        output.forEach((key, value) -> {
            Terms terms = (Terms)output.get(key);
            estimationDataMap.put(key, TermHistogramEstimationData.builder()
                    .count(hits)
                    .termCounts(terms.getBuckets()
                            .stream()
                            .collect(Collectors.toMap(Terms.Bucket::getKeyAsString, Terms.Bucket::getDocCount)))
                    .build());
        });
    }
}
 

private static <A extends Aggregation> void route(Bucket bucket, A a) {
    if (a instanceof Terms) {
        bucket.setBuckets(terms(a));
    } else if (a instanceof Range) {
        bucket.setBuckets(range(a));
    } else if (a instanceof Histogram) {
        bucket.setBuckets(range(a));
    } else if (a instanceof Avg) {
        bucket.setAvg(avg(a));
    } else if (a instanceof Min) {
        bucket.setMin(min(a));
    } else if (a instanceof Max) {
        bucket.setMax(max(a));
    } else if (a instanceof Sum) {
        bucket.setSum(sum(a));
    } else if (a instanceof Stats) {
        stats(bucket, a);
    }  else if (a instanceof ValueCount) {
        bucket.setValueCount(count(a));
    } else {
        throw new UnsupportedOperationException("不支持的聚合类型");
    }
}
 

private static void addCodeDescriptionTfidf(ESIndex index, String[] trainIndexIds, Config config, FeatureList featureList) throws Exception{
    String file = config.getString("train.feature.codeDesc.File");
    List<String> lines = FileUtils.readLines(new File(file));
    String field = config.getString("train.feature.codeDesc.matchField");
    Collection<Terms.Bucket> buckets = index.termAggregation(config.getString("train.label.field"), trainIndexIds);
    Map<String, Long> map = new HashMap<>();
    for(Terms.Bucket bucket: buckets){
        map.put(bucket.getKeyAsString(),bucket.getDocCount());
    }
    Map<String, String> codeMap = new HashMap<>();
    for (String line: lines){
        String[] lineString = line.split("\t");
        String codeDesString = lineString[1].replace("\"", "");
        codeMap.put(lineString[0],codeDesString);
    }
    for(String key:map.keySet()){
        String codeDes = codeMap.get(key);
        int size =map.get(key).intValue();
        CodeDescription codeDescription = new CodeDescription(codeDes, field,size*2,key);
        featureList.add(codeDescription);

    }
}
 

private List<BucketResponse<List<StatsTrendValue>>> buildNestedTrendStats(
        List<String> nesting,
        Aggregations aggregations) {
    final String field = nesting.get(0);
    final List<String> remainingFields = (nesting.size() > 1)
                                         ? nesting.subList(1, nesting.size())
                                         : new ArrayList<>();
    Terms terms = aggregations.get(Utils.sanitizeFieldForAggregation(field));
    List<BucketResponse<List<StatsTrendValue>>> bucketResponses = Lists.newArrayList();
    for (Terms.Bucket bucket : terms.getBuckets()) {
        BucketResponse<List<StatsTrendValue>> bucketResponse = new BucketResponse<>();
        bucketResponse.setKey(String.valueOf(bucket.getKey()));
        if (nesting.size() == 1) {
            bucketResponse.setResult(buildStatsTrendValue(getParameter().getField(), bucket.getAggregations()));
        }
        else {
            bucketResponse.setBuckets(buildNestedTrendStats(remainingFields, bucket.getAggregations()));
        }
        bucketResponses.add(bucketResponse);
    }
    return bucketResponses;
}
 

/**
 * Creates result type facet response dto
 *
 * @param searchResponse search response
 *
 * @return result type facet response dto list
 */
public List<ResultTypeIndexSearchResponseDto> getResultTypeIndexSearchResponseDto(SearchResponse searchResponse)
{
    List<ResultTypeIndexSearchResponseDto> list = new ArrayList<>();
    Terms aggregation = getAggregation(searchResponse, RESULT_TYPE_AGGS);

    for (Terms.Bucket resultTypeEntry : aggregation.getBuckets())
    {
        ResultTypeIndexSearchResponseDto dto = new ResultTypeIndexSearchResponseDto();
        dto.setResultTypeCode(resultTypeEntry.getKeyAsString());
        dto.setResultTypeDisplayName(resultTypeEntry.getKeyAsString());
        dto.setCount(resultTypeEntry.getDocCount());
        list.add(dto);
    }

    return list;
}
 

private void flatten(
        String parentKey,
        List<String> fields,
        List<List<String>> responseList,
        Aggregations aggregations) {
    final String field = fields.get(0);
    final List<String> remainingFields = (fields.size() > 1)
                                         ? fields.subList(1, fields.size())
                                         : new ArrayList<>();
    Terms terms = aggregations.get(Utils.sanitizeFieldForAggregation(field));
    for (Terms.Bucket bucket : terms.getBuckets()) {
        if (fields.size() == 1) {
            responseList.add(getValueList(parentKey, String.valueOf(bucket.getKey())));
        }
        else {
            flatten(getProperKey(parentKey, String.valueOf(bucket.getKey())),
                    remainingFields,
                    responseList,
                    bucket.getAggregations());
        }
    }
}
 

@Test
public void testSimpleSubAggregations() throws  Exception {
	final String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */ * FROM %s/account GROUP BY (gender), (state) ", TEST_INDEX_ACCOUNT);

       SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query);
	SearchResponse response = (SearchResponse) select.get();
	Aggregations result = response.getAggregations();

	Terms gender = result.get("gender");
	for(Terms.Bucket genderBucket : gender.getBuckets()) {
		String genderKey = genderBucket.getKey().toString();
		Assert.assertTrue("Gender should be m or f", genderKey.equals("m") || genderKey.equals("f"));
	}

	Assert.assertEquals(2, gender.getBuckets().size());

	Terms state = result.get("state");
	for(Terms.Bucket stateBucket : state.getBuckets()) {
		if(stateBucket.getKey().toString().equalsIgnoreCase("ak")) {
			Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22);
		}
	}

	Assert.assertEquals(response.getHits().getTotalHits().value, 1000);
	Assert.assertEquals(response.getHits().getHits().length, 10);
}
 

/**
 * @param legend_data2
 * @param type2
 * 方法名： buildGroup
 * 功 能： TODO(这里用一句话描述这个方法的作用)
 * 参 数： @param xAxis_data
 * 参 数： @param terms1
 * 返 回： void
 * 作 者 ： Administrator
 * @throws
 */
private void buildGroup(List<String> xAxis_data, List<String> legend_data, Terms terms1, String type1) {
	if (terms1 == null) {
		return;
	}
	for (Terms.Bucket bucket : terms1.getBuckets()) {
		if (bucket.getKey() == null || "".equals(bucket.getKeyAsString())) {
			continue;
		}
		if (!legend_data.contains(bucket.getKeyAsString())) {
			legend_data.add(bucket.getKeyAsString());
		}
		Terms terms2 = bucket.getAggregations().get(type1 + "_count");
		for (Terms.Bucket bucket2 : terms2.getBuckets()) {
			if (bucket2.getKey() == null || "".equals(bucket2.getKeyAsString())) {
				continue;
			}
			if (!xAxis_data.contains(bucket2.getKeyAsString())) {
				xAxis_data.add(bucket2.getKeyAsString());
			}
		}
	}

}
 

@Test
public void testGetAggregation()
{
    // Create a mock aggregation.
    Terms aggregation = mock(Terms.class);

    // Create mock aggregations.
    Aggregations aggregations = mock(Aggregations.class);
    when(aggregations.get(AGGREGATION_NAME)).thenReturn(aggregation);

    // Create a mock search response.
    SearchResponse searchResponse = mock(SearchResponse.class);
    when(searchResponse.getAggregations()).thenReturn(aggregations);

    // Call the method under test.
    Terms result = elasticsearchHelper.getAggregation(searchResponse, AGGREGATION_NAME);

    // Verify the external calls.
    verifyNoMoreInteractionsHelper();

    // Validate the result.
    assertEquals(aggregation, result);
}
 

private Map<String, Object> getMap(List<String> fields, Aggregations aggregations) {
    final String field = fields.get(0);
    final List<String> remainingFields = (fields.size() > 1)
                                         ? fields.subList(1, fields.size())
                                         : new ArrayList<>();
    Terms terms = aggregations.get(Utils.sanitizeFieldForAggregation(field));
    Map<String, Object> levelCount = Maps.newHashMap();
    for (Terms.Bucket bucket : terms.getBuckets()) {
        if (fields.size() == 1) {
            if (!CollectionUtils.isNullOrEmpty(getParameter().getUniqueCountOn())) {
                String key = Utils.sanitizeFieldForAggregation(getParameter().getUniqueCountOn());
                Cardinality cardinality = bucket.getAggregations()
                        .get(key);
                levelCount.put(String.valueOf(bucket.getKey()), cardinality.getValue());
            }
            else {
                levelCount.put(String.valueOf(bucket.getKey()), bucket.getDocCount());
            }
        }
        else {
            levelCount.put(String.valueOf(bucket.getKey()), getMap(remainingFields, bucket.getAggregations()));
        }
    }
    return levelCount;

}
 

@Test
public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedAndEmptyPath() throws Exception {
    Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/nestedType GROUP BY  nested(message.info),filter('myFilter',message.info = 'a'),reverse_nested(someField,'')", TEST_INDEX_NESTED_TYPE));
    InternalNested nested = result.get("message.info@NESTED");
    InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
    Terms infos = filter.getAggregations().get("message.info");
    Assert.assertEquals(1,infos.getBuckets().size());
    for(Terms.Bucket bucket : infos.getBuckets()) {
        InternalReverseNested reverseNested = bucket.getAggregations().get("someField@NESTED");
        Terms terms = reverseNested.getAggregations().get("someField");
        Terms.Bucket internalBucket = terms.getBuckets().get(0);

        long count = ((ValueCount) internalBucket.getAggregations().get("COUNT(*)")).getValue();
        String key = internalBucket.getKey().toString();
        if(key.equalsIgnoreCase("b")) {
            Assert.assertEquals(2, count);
        }
        else {
            throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
        }
    }
}
 

/**
 * 获取query的关联信息
 *
 * @param query 查询queryString
 * @param size  结果集数量
 * @return 相关信息
 */
public Pair<Map<String, List<Terms.Bucket>>, List<Webpage>> relatedInfo(String query, int size) {
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(QueryBuilders.queryStringQuery(query))
            .addSort("gatherTime", SortOrder.DESC)
            .addAggregation(AggregationBuilders.terms("relatedPeople").field("namedEntity.nr"))
            .addAggregation(AggregationBuilders.terms("relatedLocation").field("namedEntity.ns"))
            .addAggregation(AggregationBuilders.terms("relatedInstitution").field("namedEntity.nt"))
            .addAggregation(AggregationBuilders.terms("relatedKeywords").field("keywords"))
            .setSize(size);
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    Map<String, List<Terms.Bucket>> info = Maps.newHashMap();
    info.put("relatedPeople", ((Terms) response.getAggregations().get("relatedPeople")).getBuckets());
    info.put("relatedLocation", ((Terms) response.getAggregations().get("relatedLocation")).getBuckets());
    info.put("relatedInstitution", ((Terms) response.getAggregations().get("relatedInstitution")).getBuckets());
    info.put("relatedKeywords", ((Terms) response.getAggregations().get("relatedKeywords")).getBuckets());
    return Pair.of(info, warpHits2List(response.getHits()));
}
 

@Test
public void groupByTestWithFilter() throws Exception {
    Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY filter(gender='m'),gender", TEST_INDEX_ACCOUNT));
    InternalFilter filter = result.get("filter(gender = 'm')@FILTER");
    Terms gender = filter.getAggregations().get("gender");

    for(Terms.Bucket bucket : gender.getBuckets()) {
        String key = bucket.getKey().toString();
        long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
        if(key.equalsIgnoreCase("m")) {
            Assert.assertEquals(507, count);
        }
        else {
            throw new Exception(String.format("Unexpected key. expected: only m. found: %s", key));
        }
    }
}
 

private void parseAggregationResult (SearchResponse response, String[] aggreFields, JSONObject resObj) {
    List<Map<String, Object>> aggrs = new ArrayList<Map<String, Object>>();
    for (String aggreField : aggreFields) {
        Terms terms = response.getAggregations().get(aggreField);
        if (terms != null) {
            Map<String, Object> maps = new HashMap<String, Object>();
            for (Terms.Bucket bucket : terms.getBuckets()) {
                maps.put((String)bucket.getKey(), bucket.getDocCount());
            }
            JSONObject jo = new JSONObject();
            jo.put(aggreField, maps);
            aggrs.add(jo);
        }
    }
    resObj.put("aggregations", aggrs);
}
 

@Override
protected void runImpl(LockingTaskExecutor executor, Instant lockAtMostUntil) {
    executor.executeWithLock(() -> {
        try {
            SearchResponse searchResponse = connection.getClient()
                    .prepareSearch(INDEX_V2)
                    .setTypes(TYPE)
                    .setSearchType(SearchType.QUERY_THEN_FETCH)
                    .addAggregation(AggregationBuilders.terms("names")
                                            .field("name.keyword")
                                            .size(1000))
                    .execute()
                    .actionGet();
            Terms agg = searchResponse.getAggregations()
                    .get("names");
            for(Terms.Bucket entry : agg.getBuckets()) {
                deleteOldData(entry.getKeyAsString());
            }
        } catch (Exception e) {
            logger.info("Failed to get aggregations and delete data for index history. {}", e);
        }

    }, new LockConfiguration(consoleHistoryConfig.getJobName(), lockAtMostUntil));
}
 

/**
 * 统计制度域名的网站的文章的词频
 *
 * @param domain 网站域名
 * @return 词-词频
 */
public Map<String, Long> countWordByDomain(String domain) {
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(QueryBuilders.matchQuery("domain", domain))
            .addAggregation(AggregationBuilders.terms("content").field("content").size(200));
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    Terms termsAgg = response.getAggregations().get("content");
    List<Terms.Bucket> list = termsAgg.getBuckets();
    Map<String, Long> count = new HashMap<>();
    list.stream().filter(bucket -> ((String) bucket.getKey()).length() > 1).forEach(bucket -> {
        count.put((String) bucket.getKey(), bucket.getDocCount());
    });
    return count;
}
 

@Override
public int readMetricsValue(final MetricsCondition condition,
                            final String valueColumnName,
                            final Duration duration) throws IOException {
    SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
    buildQuery(sourceBuilder, condition, duration);

    TermsAggregationBuilder entityIdAggregation = AggregationBuilders.terms(Metrics.ENTITY_ID)
                                                                     .field(Metrics.ENTITY_ID)
                                                                     .size(1);
    final Function function = ValueColumnMetadata.INSTANCE.getValueFunction(condition.getName());
    functionAggregation(function, entityIdAggregation, valueColumnName);

    sourceBuilder.aggregation(entityIdAggregation);

    SearchResponse response = getClient().search(condition.getName(), sourceBuilder);

    Terms idTerms = response.getAggregations().get(Metrics.ENTITY_ID);
    for (Terms.Bucket idBucket : idTerms.getBuckets()) {
        switch (function) {
            case Sum:
                Sum sum = idBucket.getAggregations().get(valueColumnName);
                return (int) sum.getValue();
            case Avg:
                Avg avg = idBucket.getAggregations().get(valueColumnName);
                return (int) avg.getValue();
            default:
                avg = idBucket.getAggregations().get(valueColumnName);
                return (int) avg.getValue();
        }
    }
    return ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
}
 

/**
 * Parses an ES aggregation into a set of ResultRows
 * @param agg
 * @return
 * @throws SQLException
 */
public void parseAggregation(Aggregation agg, ESResultSet rs) throws SQLException{
	if(agg instanceof Terms){
		dfsAggregations((Terms)agg, rs, rs.getNewRow());
	}else if (agg instanceof InternalFilter){
		processFilterAgg((InternalFilter)agg, rs);
	}else if (agg instanceof InternalCardinality){
		processCardinalityAgg((InternalCardinality)agg, rs);
	}else throw new SQLException ("Unknown aggregation type "+agg.getClass().getName());
}
 

/**
 * get Tag Type index response
 *
 * @param aggregation aggregation
 *
 * @return list of tag type index search dto
 */
private List<TagTypeIndexSearchResponseDto> getTagTypeIndexSearchResponseDtosFromTerms(Terms aggregation)
{
    List<TagTypeIndexSearchResponseDto> tagTypeIndexSearchResponseDtos = new ArrayList<>();

    for (Terms.Bucket tagTypeCodeEntry : aggregation.getBuckets())
    {
        List<TagIndexSearchResponseDto> tagIndexSearchResponseDtos = new ArrayList<>();

        TagTypeIndexSearchResponseDto tagTypeIndexSearchResponseDto =
            new TagTypeIndexSearchResponseDto(tagTypeCodeEntry.getKeyAsString(), tagIndexSearchResponseDtos, null);
        tagTypeIndexSearchResponseDtos.add(tagTypeIndexSearchResponseDto);

        Terms tagTypeDisplayNameAggs = tagTypeCodeEntry.getAggregations().get(TAGTYPE_NAME_AGGREGATION);
        for (Terms.Bucket tagTypeDisplayNameEntry : tagTypeDisplayNameAggs.getBuckets())
        {
            tagTypeIndexSearchResponseDto.setDisplayName(tagTypeDisplayNameEntry.getKeyAsString());

            Terms tagCodeAggs = tagTypeDisplayNameEntry.getAggregations().get(TAG_CODE_AGGREGATION);
            TagIndexSearchResponseDto tagIndexSearchResponseDto;

            for (Terms.Bucket tagCodeEntry : tagCodeAggs.getBuckets())
            {
                tagIndexSearchResponseDto = new TagIndexSearchResponseDto(tagCodeEntry.getKeyAsString(), tagCodeEntry.getDocCount(), null);
                tagIndexSearchResponseDtos.add(tagIndexSearchResponseDto);

                Terms tagNameAggs = tagCodeEntry.getAggregations().get(TAG_NAME_AGGREGATION);
                for (Terms.Bucket tagNameEntry : tagNameAggs.getBuckets())
                {
                    tagIndexSearchResponseDto.setTagDisplayName(tagNameEntry.getKeyAsString());
                }
            }
        }
    }

    return tagTypeIndexSearchResponseDtos;
}
 

@Test
public void testSubAggregations() throws  Exception {
	Set expectedAges = new HashSet<>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
	final String query = String.format("SELECT /*! DOCS_WITH_AGGREGATION(10) */" +
               " * FROM %s/account GROUP BY (gender, terms('field'='age','size'=200,'alias'='age')), (state) LIMIT 200,200", TEST_INDEX_ACCOUNT);

	Map<String, Set<Integer>> buckets = new HashMap<>();

       SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query);
	SearchResponse response = (SearchResponse) select.get();
	Aggregations result = response.getAggregations();

	Terms gender = result.get("gender");
	for(Terms.Bucket genderBucket : gender.getBuckets()) {
		String genderKey = genderBucket.getKey().toString();
		buckets.put(genderKey, new HashSet<Integer>());
		Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
		for(Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
			buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey().toString()));
		}
	}

	Assert.assertEquals(2, buckets.keySet().size());
	Assert.assertEquals(expectedAges, buckets.get("m"));
	Assert.assertEquals(expectedAges, buckets.get("f"));

	Terms state = result.get("state");
	for(Terms.Bucket stateBucket : state.getBuckets()) {
		if(stateBucket.getKey().toString().equalsIgnoreCase("ak")) {
			Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22);
		}
	}

	Assert.assertEquals(response.getHits().getTotalHits().value, 1000);
	Assert.assertEquals(response.getHits().getHits().length, 10);
}