org.elasticsearch.search.builder.SearchSourceBuilder#aggregation

Source File: ParseUtils.java From anomaly-detection with Apache License 2.0

6 votes

public static String generateInternalFeatureQueryTemplate(AnomalyDetector detector, NamedXContentRegistry xContentRegistry)
    throws IOException {
    RangeQueryBuilder rangeQuery = new RangeQueryBuilder(detector.getTimeField())
        .from("{{" + QUERY_PARAM_PERIOD_START + "}}")
        .to("{{" + QUERY_PARAM_PERIOD_END + "}}");

    BoolQueryBuilder internalFilterQuery = QueryBuilders.boolQuery().must(rangeQuery).must(detector.getFilterQuery());

    SearchSourceBuilder internalSearchSourceBuilder = new SearchSourceBuilder().query(internalFilterQuery);
    if (detector.getFeatureAttributes() != null) {
        for (Feature feature : detector.getFeatureAttributes()) {
            AggregatorFactories.Builder internalAgg = parseAggregators(
                feature.getAggregation().toString(),
                xContentRegistry,
                feature.getId()
            );
            internalSearchSourceBuilder.aggregation(internalAgg.getAggregatorFactories().iterator().next());
        }
    }

    return internalSearchSourceBuilder.toString();
}

Source File: ElasticsearchHighRestFactory.java From database-transform-tool with Apache License 2.0

6 votes

public String selectMatchAll(String indexs,String types,String field,String value){
	try {
		if(StringUtil.isEmpty(indexs))indexs="_all";
		if(xclient==null){
			init();
		}
		SearchSourceBuilder search = new SearchSourceBuilder();
		if(!StringUtil.isEmpty(field)&&!StringUtil.isEmpty(value)&&!(field.matches(regex)||field.matches(value))){
			search.query(QueryBuilders.matchQuery(field, value));
		}
		search.aggregation(AggregationBuilders.terms("data").field(field+".keyword"));
		search.explain(false);
		SearchRequest request = new SearchRequest();
		request.searchType(SearchType.DFS_QUERY_THEN_FETCH);
		request.source(search);
		request.indices(indexs.split(","));
		request.types(types.split(","));
		SearchResponse response = xclient.search(request);
		return response.toString();
	} catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}

Source File: ElasticsearchExtendHighRestFactory.java From database-transform-tool with Apache License 2.0

6 votes

public String selectTermAll(String indexs,String types,String field,String value){
	try {
		if(StringUtil.isEmpty(indexs))indexs="_all";
		if(xclient==null){
			init();
		}
		SearchSourceBuilder search = new SearchSourceBuilder();
		if(!StringUtil.isEmpty(field)&&!StringUtil.isEmpty(value)&&!(field.matches(regex)||field.matches(value))){
			search.query(QueryBuilders.termQuery(field, value));
		}
		search.aggregation(AggregationBuilders.terms("data").field(field+".keyword"));
		search.explain(false);
		SearchRequest request = new SearchRequest();
		request.searchType(SearchType.DFS_QUERY_THEN_FETCH);
		request.source(search);
		request.indices(indexs.split(","));
		request.types(types.split(","));
		SearchResponse response = xclient.search(request);
		return response.toString();
	} catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}

Source File: ProfileThreadSnapshotQueryEsDAO.java From skywalking with Apache License 2.0

6 votes

protected int querySequenceWithAgg(AbstractAggregationBuilder aggregationBuilder, String segmentId, long start, long end) throws IOException {
    SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();

    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
    sourceBuilder.query(boolQueryBuilder);
    List<QueryBuilder> mustQueryList = boolQueryBuilder.must();

    mustQueryList.add(QueryBuilders.termQuery(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId));
    mustQueryList.add(QueryBuilders.rangeQuery(ProfileThreadSnapshotRecord.DUMP_TIME).gte(start).lte(end));
    sourceBuilder.size(0);

    sourceBuilder.aggregation(aggregationBuilder);
    SearchResponse response = getClient().search(ProfileThreadSnapshotRecord.INDEX_NAME, sourceBuilder);
    NumericMetricsAggregation.SingleValue agg = response.getAggregations().get(ProfileThreadSnapshotRecord.SEQUENCE);

    return (int) agg.value();
}

Source File: PercentileRankAggregationMain.java From elasticsearch-pool with Apache License 2.0

6 votes

public static void main(String[] args) throws IOException {
    RestHighLevelClient client = HighLevelClient.getInstance();
    try{
        QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("cmd", "weather_hourforcast");
        PercentileRanksAggregationBuilder aggregationBuilder = AggregationBuilders.percentileRanks("utm_ranks",new double[]{200,500,1000,3000,8000}).field("utm").keyed(false);

        SearchRequest searchRequest = new SearchRequest("serverlog_20180710");//限定index
        searchRequest.types("log");//限定type

        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(matchQueryBuilder);
        searchSourceBuilder.aggregation(aggregationBuilder);
        searchSourceBuilder.size(0);
        searchRequest.source(searchSourceBuilder);

        SearchResponse searchResponse = client.search(searchRequest);
        System.out.println(searchResponse);

    }finally{
        HighLevelClient.close();
    }
}

Source File: PercentilesAggregationMain.java From elasticsearch-pool with Apache License 2.0

6 votes

public static void main(String[] args) throws IOException {
        RestHighLevelClient client = HighLevelClient.getInstance();
        try{
            QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("cmd", "weather_hourforcast");
            PercentilesAggregationBuilder aggregationBuilder = AggregationBuilders.percentiles("utm").field("utm").keyed(false);
//            PercentilesAggregationBuilder aggregationBuilder = AggregationBuilders.percentiles("utm").field("utm").percentiles(95,99,99.9).keyed(false);//自定义百分区间

            SearchRequest searchRequest = new SearchRequest("serverlog_20180710");//限定index
            searchRequest.types("log");//限定type

            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(matchQueryBuilder);
            searchSourceBuilder.aggregation(aggregationBuilder);
            searchSourceBuilder.size(0);
            searchRequest.source(searchSourceBuilder);

            SearchResponse searchResponse = client.search(searchRequest);
            System.out.println(searchResponse);

        }finally{
            HighLevelClient.close();
        }
    }

Source File: TopologyQueryEsDAO.java From skywalking with Apache License 2.0

6 votes

private List<Call.CallDetail> buildServiceRelation(SearchSourceBuilder sourceBuilder, String indexName,
                                                   DetectPoint detectPoint) throws IOException {
    sourceBuilder.aggregation(
        AggregationBuilders
            .terms(Metrics.ENTITY_ID).field(Metrics.ENTITY_ID)
            .subAggregation(
                AggregationBuilders.terms(ServiceRelationServerSideMetrics.COMPONENT_ID)
                                   .field(ServiceRelationServerSideMetrics.COMPONENT_ID))
            .size(1000));

    SearchResponse response = getClient().search(indexName, sourceBuilder);

    List<Call.CallDetail> calls = new ArrayList<>();
    Terms entityTerms = response.getAggregations().get(Metrics.ENTITY_ID);
    for (Terms.Bucket entityBucket : entityTerms.getBuckets()) {
        String entityId = entityBucket.getKeyAsString();
        Terms componentTerms = entityBucket.getAggregations().get(ServiceRelationServerSideMetrics.COMPONENT_ID);
        final int componentId = componentTerms.getBuckets().get(0).getKeyAsNumber().intValue();

        Call.CallDetail call = new Call.CallDetail();
        call.buildFromServiceRelation(entityId, componentId, detectPoint);
        calls.add(call);
    }
    return calls;
}

Source File: StatsAggregationMain.java From elasticsearch-pool with Apache License 2.0

6 votes

public static void main(String[] args) throws IOException {
    RestHighLevelClient client = HighLevelClient.getInstance();
    try{
        QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("cmd", "weather_hourforcast");
        StatsAggregationBuilder aggregationBuilder = AggregationBuilders.stats("utm_stats").field("utm").missing(0);

        SearchRequest searchRequest = new SearchRequest("serverlog_20180710");//限定index
        searchRequest.types("log");//限定type

        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(matchQueryBuilder);
        searchSourceBuilder.aggregation(aggregationBuilder);
        searchSourceBuilder.size(0);
        searchRequest.source(searchSourceBuilder);

        SearchResponse searchResponse = client.search(searchRequest);
        System.out.println(searchResponse);

    }finally{
        HighLevelClient.close();
    }
}

Source File: EsHelper.java From occurrence with Apache License 2.0

6 votes

/**
 * Finds all the indexes of the alias where the given datasetKey is indexed.
 *
 * @param esClient client to connect to ES
 * @param datasetKey datasetKey to look for
 * @param aliases index or alias where we are looking for indexes
 * @return indexes found
 */
public static Set<String> findExistingIndexesInAliases(
    final RestHighLevelClient esClient, String datasetKey, String[] aliases) {
  Objects.requireNonNull(esClient);
  Preconditions.checkArgument(!Strings.isNullOrEmpty(datasetKey), "datasetKey is required");
  Preconditions.checkArgument(aliases != null && aliases.length > 0, "aliases are required");

  SearchRequest esRequest = new SearchRequest();
  esRequest.indices(aliases);

  SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
  searchSourceBuilder.size(0);
  esRequest.source(searchSourceBuilder);

  // add match query to filter by datasetKey
  searchSourceBuilder.query(QueryBuilders.matchQuery(DATASET_KEY_FIELD, datasetKey));
  // add aggs by index
  searchSourceBuilder.aggregation(AggregationBuilders.terms(AGG_BY_INDEX).field("_index"));

  try {
    return parseFindExistingIndexesInAliasResponse(esClient.search(esRequest, HEADERS.get()));
  } catch (IOException e) {
    throw new SearchException("Could not find indexes that contain the dataset " + datasetKey, e);
  }
}

Source File: ChildrenAggregationMain.java From elasticsearch-pool with Apache License 2.0

6 votes

public static void main(String[] args) throws IOException {
    RestHighLevelClient client = HighLevelClient.getInstance();
    try{

        ChildrenAggregationBuilder aggregationBuilder = new ChildrenAggregationBuilder("utm","histogram");
        aggregationBuilder.field("utm");

        SearchRequest searchRequest = new SearchRequest("serverlog_20180710");//限定index
        searchRequest.types("log");//限定type

        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggregationBuilder);
        searchRequest.source(searchSourceBuilder);

        SearchResponse searchResponse = client.search(searchRequest);
        System.out.println(searchResponse);
        Aggregations aggregations = searchResponse.getAggregations();
        List<Aggregation> aggregationList = aggregations.asList();
        for(Aggregation each: aggregationList){
            System.out.println(each);
        }
    }finally{
        HighLevelClient.close();
    }
}

Source File: SumAggregationMain.java From elasticsearch-pool with Apache License 2.0

5 votes

public static void main(String[] args) throws IOException {
        RestHighLevelClient client = HighLevelClient.getInstance();
        try{
//            QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("cmd", "weather_hourforcast");
            SumAggregationBuilder aggregationBuilder = AggregationBuilders.sum("utm").field("utm");

            SearchRequest searchRequest = new SearchRequest("serverlog_20180715");//限定index
            searchRequest.types("log");//限定type

            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//            searchSourceBuilder.query(matchQueryBuilder);
            searchSourceBuilder.aggregation(aggregationBuilder);
            searchSourceBuilder.size(0);
            searchRequest.source(searchSourceBuilder);

            SearchResponse searchResponse = client.search(searchRequest);
            System.out.println(searchResponse);

            //统计结果
            Aggregations aggregations = searchResponse.getAggregations();
            Map<String, Aggregation> aggregationMap = aggregations.asMap();
            for(Map.Entry<String,Aggregation> each: aggregationMap.entrySet()){
                System.out.println(((ParsedSum)(each.getValue())).getValue());
            }

        }finally{
            HighLevelClient.close();
        }
    }

Source File: AggregationHelper.java From fast-elasticsearch-query-builder with Apache License 2.0

5 votes

private static void setCardinalityAggregation(SearchSourceBuilder searchSource, CardinalityAggregation aggregation,
        Boolean value) {
    if (!value) {
        return;
    }
    CardinalityAggregationBuilder cardinalityAggregation = AggregationBuilders.cardinality(aggregation.name()).field(aggregation.field());
    long precisionThreshold = aggregation.precisionThreshold();
    if (precisionThreshold > 0) {
        cardinalityAggregation.precisionThreshold(precisionThreshold);
    }
    searchSource.aggregation(cardinalityAggregation);
}

Source File: ElasticSearchRestHighImpl.java From sunbird-lms-service with MIT License

5 votes

private static SearchSourceBuilder addAggregations(
    SearchSourceBuilder searchSourceBuilder, List<Map<String, String>> facets) {
  long startTime = System.currentTimeMillis();
  ProjectLogger.log(
      "ElasticSearchRestHighImpl:addAggregations: method started at ==" + startTime,
      LoggerEnum.PERF_LOG.name());
  Map<String, String> map = facets.get(0);
  for (Map.Entry<String, String> entry : map.entrySet()) {

    String key = entry.getKey();
    String value = entry.getValue();
    if (JsonKey.DATE_HISTOGRAM.equalsIgnoreCase(value)) {
      searchSourceBuilder.aggregation(
          AggregationBuilders.dateHistogram(key)
              .field(key + ElasticSearchHelper.RAW_APPEND)
              .dateHistogramInterval(DateHistogramInterval.days(1)));

    } else if (null == value) {
      searchSourceBuilder.aggregation(
          AggregationBuilders.terms(key).field(key + ElasticSearchHelper.RAW_APPEND));
    }
  }
  ProjectLogger.log(
      "ElasticSearchRestHighImpl:addAggregations: method end =="
          + " ,Total time elapsed = "
          + calculateEndTime(startTime),
      LoggerEnum.PERF_LOG.name());
  return searchSourceBuilder;
}

Source File: DateHistogramAggregationMain.java From elasticsearch-pool with Apache License 2.0

5 votes

public static void main(String[] args) throws IOException {
        RestHighLevelClient client = HighLevelClient.getInstance();
        try{
            DateHistogramAggregationBuilder dateHistogramAggregationBuilder = AggregationBuilders.dateHistogram("ctm_date_histogram");
            dateHistogramAggregationBuilder.field("ctm");//设置直方图针对的字段
            dateHistogramAggregationBuilder.dateHistogramInterval(DateHistogramInterval.hours(6));//直方图每个分组对应的范围
            dateHistogramAggregationBuilder.timeZone(DateTimeZone.forOffsetHours(8));//时区偏移
            dateHistogramAggregationBuilder.keyed(true);//是否需要key名
            dateHistogramAggregationBuilder.format("yyyy-MM-dd HH:mm");//key名格式
//            dateHistogramAggregationBuilder.order(BucketOrder.aggregation("_key",true));//分组key的排序
//            dateHistogramAggregationBuilder.minDocCount(0);//对于每个分组最少具有多少条数据，少于这个设置，则该分组不显示
//            dateHistogramAggregationBuilder.extendedBounds(0,8000);//设置分组区间的下线和上线，只有当min_doc_count为0时有效

            TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("cmd","weather_info");

            SearchRequest searchRequest = new SearchRequest("serverlog_20180710");//限定index
            searchRequest.types("log");//限定type

            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(termQueryBuilder);
            searchSourceBuilder.aggregation(dateHistogramAggregationBuilder);
            searchRequest.source(searchSourceBuilder);

            SearchResponse searchResponse = client.search(searchRequest);
            System.out.println(searchResponse);

        }finally{
            HighLevelClient.close();
        }
    }

Source File: MetricsQueryEs7DAO.java From skywalking with Apache License 2.0

5 votes

@Override
public int readMetricsValue(final MetricsCondition condition,
                            final String valueColumnName,
                            final Duration duration) throws IOException {
    SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
    buildQuery(sourceBuilder, condition, duration);

    TermsAggregationBuilder entityIdAggregation = AggregationBuilders.terms(Metrics.ENTITY_ID)
                                                                     .field(Metrics.ENTITY_ID)
                                                                     .size(1);
    final Function function = ValueColumnMetadata.INSTANCE.getValueFunction(condition.getName());
    functionAggregation(function, entityIdAggregation, valueColumnName);

    sourceBuilder.aggregation(entityIdAggregation);

    SearchResponse response = getClient().search(condition.getName(), sourceBuilder);

    Terms idTerms = response.getAggregations().get(Metrics.ENTITY_ID);
    for (Terms.Bucket idBucket : idTerms.getBuckets()) {
        switch (function) {
            case Sum:
                Sum sum = idBucket.getAggregations().get(valueColumnName);
                return (int) sum.getValue();
            case Avg:
                Avg avg = idBucket.getAggregations().get(valueColumnName);
                return (int) avg.getValue();
            default:
                avg = idBucket.getAggregations().get(valueColumnName);
                return (int) avg.getValue();
        }
    }
    return ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
}

Source File: ElasticsearchDataQuery.java From frostmourne with MIT License

5 votes

private void attachAggregation(MetricContract metricContract, SearchSourceBuilder searchSourceBuilder) {
    String aggType = metricContract.getAggregation_type();
    String aggField = metricContract.getAggregation_field();
    if (aggType.equalsIgnoreCase("max")) {
        searchSourceBuilder.aggregation(AggregationBuilders.max("maxNumber").field(aggField));
    } else if (aggType.equalsIgnoreCase("min")) {
        searchSourceBuilder.aggregation(AggregationBuilders.min("minNumber").field(aggField));
    } else if (aggType.equalsIgnoreCase("avg")) {
        searchSourceBuilder.aggregation(AggregationBuilders.avg("avgNumber").field(aggField));
    } else if (aggType.equalsIgnoreCase("sum")) {
        searchSourceBuilder.aggregation(AggregationBuilders.sum("sumNumber").field(aggField));
    }
}

Source File: CountAggregationMain.java From elasticsearch-pool with Apache License 2.0

5 votes

public static void main(String[] args) throws IOException {
    RestHighLevelClient client = HighLevelClient.getInstance();
    try{
        QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("cmd", "get_fee_transfer_content_data");
        ValueCountAggregationBuilder aggregationBuilder = AggregationBuilders.count("cmd_count").field("cmd");

        SearchRequest searchRequest = new SearchRequest("serverlog_20180715");//限定index
        searchRequest.types("log");//限定type

        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(matchQueryBuilder);
        searchSourceBuilder.aggregation(aggregationBuilder);
        searchSourceBuilder.size(0);//不返回具体业务数据，只需要聚合结果
        searchRequest.source(searchSourceBuilder);

        SearchResponse searchResponse = client.search(searchRequest);
        System.out.println(searchResponse);

        //统计结果
        Aggregations aggregations = searchResponse.getAggregations();
        Map<String, Aggregation> aggregationMap = aggregations.asMap();
        for(Map.Entry<String,Aggregation> each: aggregationMap.entrySet()){
            System.out.println(((ParsedValueCount)(each.getValue())).getValue());
        }

    }finally{
        HighLevelClient.close();
    }
}

Source File: MaxMinAggregationMain.java From elasticsearch-pool with Apache License 2.0

5 votes

public static void main(String[] args) throws IOException {
        RestHighLevelClient client = HighLevelClient.getInstance();
        try{
//            MaxAggregationBuilder aggregationBuilder = AggregationBuilders.max("utm").field("utm");
            MinAggregationBuilder aggregationBuilder = AggregationBuilders.min("utm").field("utm");

            SearchRequest searchRequest = new SearchRequest("serverlog_20180715");//限定index
            searchRequest.types("log");//限定type

            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.aggregation(aggregationBuilder);
            searchSourceBuilder.size(0);//设置不需要文档数据，只需要返回聚合结果
            searchRequest.source(searchSourceBuilder);

            SearchResponse searchResponse = client.search(searchRequest);
            System.out.println(searchResponse);

            //统计结果
            Aggregations aggregations = searchResponse.getAggregations();
            Map<String, Aggregation> aggregationMap = aggregations.asMap();
            for(Map.Entry<String,Aggregation> each: aggregationMap.entrySet()){
//                System.out.println(((ParsedMax)(each.getValue())).getValue());
                System.out.println(((ParsedMin)(each.getValue())).getValue());
            }

        }finally{
            HighLevelClient.close();
        }
    }

Source File: MetricsQueryEsDAO.java From skywalking with Apache License 2.0

5 votes

@Override
public int readMetricsValue(final MetricsCondition condition,
                            final String valueColumnName,
                            final Duration duration) throws IOException {
    SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
    buildQuery(sourceBuilder, condition, duration);

    TermsAggregationBuilder entityIdAggregation = AggregationBuilders.terms(Metrics.ENTITY_ID)
                                                                     .field(Metrics.ENTITY_ID)
                                                                     .size(1);
    final Function function = ValueColumnMetadata.INSTANCE.getValueFunction(condition.getName());
    functionAggregation(function, entityIdAggregation, valueColumnName);

    sourceBuilder.aggregation(entityIdAggregation);

    SearchResponse response = getClient().search(condition.getName(), sourceBuilder);

    Terms idTerms = response.getAggregations().get(Metrics.ENTITY_ID);
    for (Terms.Bucket idBucket : idTerms.getBuckets()) {
        switch (function) {
            case Sum:
                Sum sum = idBucket.getAggregations().get(valueColumnName);
                return (int) sum.getValue();
            case Avg:
                Avg avg = idBucket.getAggregations().get(valueColumnName);
                return (int) avg.getValue();
            default:
                avg = idBucket.getAggregations().get(valueColumnName);
                return (int) avg.getValue();
        }
    }
    return ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
}

Source File: ParseUtils.java From anomaly-detection with Apache License 2.0

5 votes

public static SearchSourceBuilder generateInternalFeatureQuery(
    AnomalyDetector detector,
    long startTime,
    long endTime,
    NamedXContentRegistry xContentRegistry
) throws IOException {
    RangeQueryBuilder rangeQuery = new RangeQueryBuilder(detector.getTimeField())
        .from(startTime)
        .to(endTime)
        .format("epoch_millis")
        .includeLower(true)
        .includeUpper(false);

    BoolQueryBuilder internalFilterQuery = QueryBuilders.boolQuery().must(rangeQuery).must(detector.getFilterQuery());

    SearchSourceBuilder internalSearchSourceBuilder = new SearchSourceBuilder().query(internalFilterQuery);
    if (detector.getFeatureAttributes() != null) {
        for (Feature feature : detector.getFeatureAttributes()) {
            AggregatorFactories.Builder internalAgg = parseAggregators(
                feature.getAggregation().toString(),
                xContentRegistry,
                feature.getId()
            );
            internalSearchSourceBuilder.aggregation(internalAgg.getAggregatorFactories().iterator().next());
        }
    }

    return internalSearchSourceBuilder;
}

Java Code Examples for org.elasticsearch.search.builder.SearchSourceBuilder#aggregation()