org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval Java Exaples

Source File: CommonWebpageDAO.java From Gather-Platform with GNU General Public License v3.0

6 votes

/**
 * 统计指定网站每天抓取数量
 *
 * @param domain 网站域名
 * @return
 */
public Map<Date, Long> countDomainByGatherTime(String domain) {
    AggregationBuilder aggregation =
            AggregationBuilders
                    .dateHistogram("agg")
                    .field("gatherTime")
                    .dateHistogramInterval(DateHistogramInterval.DAY).order(Histogram.Order.KEY_DESC);
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(QueryBuilders.matchQuery("domain", domain))
            .addAggregation(aggregation);
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    Histogram agg = response.getAggregations().get("agg");
    Map<Date, Long> result = Maps.newHashMap();
    for (Histogram.Bucket entry : agg.getBuckets()) {
        DateTime key = (DateTime) entry.getKey();    // Key
        long docCount = entry.getDocCount();         // Doc count
        result.put(key.toDate(), docCount);
    }
    return result;
}

Source File: CommonWebpageDAO.java From spider with GNU General Public License v3.0

6 votes

/**
 * 统计指定网站每天抓取数量
 *
 * @param domain 网站域名
 * @return
 */
public Map<Date, Long> countDomainByGatherTime(String domain) {
    AggregationBuilder aggregation =
            AggregationBuilders
                    .dateHistogram("agg")
                    .field("gatherTime")
                    .dateHistogramInterval(DateHistogramInterval.DAY).order(Histogram.Order.KEY_DESC);
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(INDEX_NAME)
            .setTypes(TYPE_NAME)
            .setQuery(QueryBuilders.matchQuery("domain", domain))
            .addAggregation(aggregation);
    SearchResponse response = searchRequestBuilder.execute().actionGet();
    Histogram agg = response.getAggregations().get("agg");
    Map<Date, Long> result = Maps.newHashMap();
    for (Histogram.Bucket entry : agg.getBuckets()) {
        DateTime key = (DateTime) entry.getKey();    // Key
        long docCount = entry.getDocCount();         // Doc count
        result.put(key.toDate(), docCount);
    }
    return result;
}

Source File: EsEventPersistence.java From logsniffer with GNU Lesser General Public License v3.0

6 votes

protected DateHistogramInterval getInterval(final long timeRange, final int maxSlotsCount,
		final EventsCountHistogram histogram) {
	// year, quarter, month, week, day, hour, minute, second
	long dif = timeRange / maxSlotsCount / 1000;
	if (dif <= 0) {
		histogram.setInterval(HistogramInterval.SECOND);
		return DateHistogramInterval.SECOND;
	} else if (dif < 60) {
		histogram.setInterval(HistogramInterval.MINUTE);
		return DateHistogramInterval.MINUTE;
	} else if ((dif = dif / 60) < 60) {
		histogram.setInterval(HistogramInterval.HOUR);
		return DateHistogramInterval.HOUR;
	} else if ((dif = dif / 60) < 24) {
		histogram.setInterval(HistogramInterval.DAY);
		return DateHistogramInterval.DAY;
	} else if ((dif = dif / 24) < 7) {
		histogram.setInterval(HistogramInterval.WEEK);
		return DateHistogramInterval.WEEK;
	} else if ((dif = dif / 7) < 4) {
		histogram.setInterval(HistogramInterval.MONTH);
		return DateHistogramInterval.MONTH;
	}
	histogram.setInterval(HistogramInterval.YEAR);
	return DateHistogramInterval.YEAR;
}

Source File: Utils.java From foxtrot with Apache License 2.0

6 votes

public static DateHistogramInterval getHistogramInterval(Period period) {
    DateHistogramInterval interval;
    switch (period) {
        case seconds:
            interval = DateHistogramInterval.SECOND;
            break;
        case minutes:
            interval = DateHistogramInterval.MINUTE;
            break;
        case hours:
            interval = DateHistogramInterval.HOUR;
            break;
        case days:
            interval = DateHistogramInterval.DAY;
            break;
        default:
            interval = DateHistogramInterval.HOUR;
            break;
    }
    return interval;
}

Source File: ElasticsearchHelperService.java From xmfcn-spring-cloud with Apache License 2.0

5 votes

/**
 * 按天统计  各个日志级别的数量 查询条件
 *
 * @param EsModel es
 *                keywords
 *                highlights
 * @return
 */
public Search statisticsDayCondition(EsModel es) {
    Search search = null;
    if (es == null) {
        return search;
    }
    String indexName = es.getIndex();
    String type = es.getType();
    String startTime = es.getStartTime();
    String endTime = es.getEndTime();
    JSONObject keywords = es.getKeyWord();
    if (StringUtil.isBlank(indexName)) {
        return search;
    }
    if (StringUtil.isBlank(type)) {
        return search;
    }
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    if (StringUtil.isNotBlank(startTime) && StringUtil.isNotBlank(endTime)) {
        RangeQueryBuilder rangequerybuilder = QueryBuilders
                .rangeQuery("createTime")
                .from(startTime).to(endTime);
        searchSourceBuilder.query(rangequerybuilder);
    }
    AddKeyWords(keywords, searchSourceBuilder);
    ExtendedBounds extendedBounds = new ExtendedBounds(startTime, endTime);
    AggregationBuilder levelAgg = AggregationBuilders.terms("level_count").field("level").minDocCount(0);
    AggregationBuilder dateAgg = AggregationBuilders.dateHistogram("day_count")
            .field("createTime")
            .dateHistogramInterval(DateHistogramInterval.DAY)
            .format("yyyy-MM-dd")
            .extendedBounds(extendedBounds)
            .minDocCount(0L)//为空0补充
            .timeZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+8")));
    AggregationBuilder builder = dateAgg.subAggregation(levelAgg);
    searchSourceBuilder.aggregation(builder).size(0);
    search = new Search.Builder(searchSourceBuilder.toString())
            .addIndex(indexName).addType(type).build();
    return search;
}

Source File: DateHistogramAggregationMain.java From elasticsearch-pool with Apache License 2.0

5 votes

public static void main(String[] args) throws IOException {
        RestHighLevelClient client = HighLevelClient.getInstance();
        try{
            DateHistogramAggregationBuilder dateHistogramAggregationBuilder = AggregationBuilders.dateHistogram("ctm_date_histogram");
            dateHistogramAggregationBuilder.field("ctm");//设置直方图针对的字段
            dateHistogramAggregationBuilder.dateHistogramInterval(DateHistogramInterval.hours(6));//直方图每个分组对应的范围
            dateHistogramAggregationBuilder.timeZone(DateTimeZone.forOffsetHours(8));//时区偏移
            dateHistogramAggregationBuilder.keyed(true);//是否需要key名
            dateHistogramAggregationBuilder.format("yyyy-MM-dd HH:mm");//key名格式
//            dateHistogramAggregationBuilder.order(BucketOrder.aggregation("_key",true));//分组key的排序
//            dateHistogramAggregationBuilder.minDocCount(0);//对于每个分组最少具有多少条数据，少于这个设置，则该分组不显示
//            dateHistogramAggregationBuilder.extendedBounds(0,8000);//设置分组区间的下线和上线，只有当min_doc_count为0时有效

            TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("cmd","weather_info");

            SearchRequest searchRequest = new SearchRequest("serverlog_20180710");//限定index
            searchRequest.types("log");//限定type

            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(termQueryBuilder);
            searchSourceBuilder.aggregation(dateHistogramAggregationBuilder);
            searchRequest.source(searchSourceBuilder);

            SearchResponse searchResponse = client.search(searchRequest);
            System.out.println(searchResponse);

        }finally{
            HighLevelClient.close();
        }
    }

Source File: EsAggregationSearchTest.java From java-study with Apache License 2.0

5 votes

/**
 * @Author pancm
 * @Description 多个聚合条件测试
 * SQL: select age, name, count(*) as count1 from student group by age, name;
 * @Date  2019/7/3
 * @Param []
 * @return void
 **/
private static void groupbySearch() throws IOException{
    String buk="group";
    AggregationBuilder aggregation = AggregationBuilders.terms("age").field("age");
    AggregationBuilder aggregation2 = AggregationBuilders.terms("name").field("name");
    //根据创建时间按天分组
    AggregationBuilder aggregation3 = AggregationBuilders.dateHistogram("createtm")
            .field("createtm")
            .format("yyyy-MM-dd")
            .dateHistogramInterval(DateHistogramInterval.DAY);

    aggregation2.subAggregation(aggregation3);
    aggregation.subAggregation(aggregation2);
    agg(aggregation,buk);
}

Source File: BaseDemo.java From Elasticsearch-Tutorial-zh-CN with GNU General Public License v3.0

5 votes

/**
 * 聚合分析
 * 1. 先分组
 * 2. 子分组
 * 3. 最后算出子分组的平均值
 *
 * @param transportClient
 * @throws IOException
 */
private static void aggregate(TransportClient transportClient) throws IOException {

	SearchResponse searchResponse = transportClient.prepareSearch("product_index").setTypes("product")
			.addAggregation(AggregationBuilders.terms("product_group_by_price").field("price")
					.subAggregation(AggregationBuilders.dateHistogram("product_group_by_created_date_time").field("created_date_time")
							.dateHistogramInterval(DateHistogramInterval.YEAR)
							.subAggregation(AggregationBuilders.avg("product_avg_price").field("price")))
			).execute().actionGet();

	Map<String, Aggregation> aggregationMap = searchResponse.getAggregations().asMap();

	StringTerms productGroupByPrice = (StringTerms) aggregationMap.get("product_group_by_price");
	Iterator<Terms.Bucket> productGroupByPriceIterator = productGroupByPrice.getBuckets().iterator();
	while (productGroupByPriceIterator.hasNext()) {
		Terms.Bucket productGroupByPriceBucket = productGroupByPriceIterator.next();
		logger.info("--------------------------------：" + productGroupByPriceBucket.getKey() + ":" + productGroupByPriceBucket.getDocCount());

		Histogram productGroupByPrice1 = (Histogram) productGroupByPriceBucket.getAggregations().asMap().get("product_group_by_price");
		Iterator<org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket> groupByCreateDateTimeIterator = productGroupByPrice1.getBuckets().iterator();
		while (groupByCreateDateTimeIterator.hasNext()) {
			Histogram.Bucket groupByCreateDateTimeBucket = groupByCreateDateTimeIterator.next();
			logger.info("--------------------------------：" + groupByCreateDateTimeBucket.getKey() + ":" + groupByCreateDateTimeBucket.getDocCount());

			Avg avgPrice = (Avg) groupByCreateDateTimeBucket.getAggregations().asMap().get("product_avg_price");
			logger.info("--------------------------------：" + avgPrice.getValue());
		}
	}


}

Source File: ElasticSearchRestHighImpl.java From sunbird-lms-service with MIT License

5 votes

private static SearchSourceBuilder addAggregations(
    SearchSourceBuilder searchSourceBuilder, List<Map<String, String>> facets) {
  long startTime = System.currentTimeMillis();
  ProjectLogger.log(
      "ElasticSearchRestHighImpl:addAggregations: method started at ==" + startTime,
      LoggerEnum.PERF_LOG.name());
  Map<String, String> map = facets.get(0);
  for (Map.Entry<String, String> entry : map.entrySet()) {

    String key = entry.getKey();
    String value = entry.getValue();
    if (JsonKey.DATE_HISTOGRAM.equalsIgnoreCase(value)) {
      searchSourceBuilder.aggregation(
          AggregationBuilders.dateHistogram(key)
              .field(key + ElasticSearchHelper.RAW_APPEND)
              .dateHistogramInterval(DateHistogramInterval.days(1)));

    } else if (null == value) {
      searchSourceBuilder.aggregation(
          AggregationBuilders.terms(key).field(key + ElasticSearchHelper.RAW_APPEND));
    }
  }
  ProjectLogger.log(
      "ElasticSearchRestHighImpl:addAggregations: method end =="
          + " ,Total time elapsed = "
          + calculateEndTime(startTime),
      LoggerEnum.PERF_LOG.name());
  return searchSourceBuilder;
}

Source File: ElasticSearchHelper.java From sunbird-lms-service with MIT License

5 votes

/**
 * This method adds aggregations to the incoming SearchRequestBuilder object
 *
 * @param searchRequestBuilder which will be updated with facets if any present
 * @param facets Facets provide aggregated data based on a search query
 * @return SearchRequestBuilder
 */
public static SearchRequestBuilder addAggregations(
    SearchRequestBuilder searchRequestBuilder, List<Map<String, String>> facets) {
  long startTime = System.currentTimeMillis();
  ProjectLogger.log(
      "ElasticSearchHelper:addAggregations: method started at ==" + startTime,
      LoggerEnum.PERF_LOG.name());
  if (facets != null && !facets.isEmpty()) {
    Map<String, String> map = facets.get(0);
    if (!MapUtils.isEmpty(map)) {
      for (Map.Entry<String, String> entry : map.entrySet()) {

        String key = entry.getKey();
        String value = entry.getValue();
        if (JsonKey.DATE_HISTOGRAM.equalsIgnoreCase(value)) {
          searchRequestBuilder.addAggregation(
              AggregationBuilders.dateHistogram(key)
                  .field(key + RAW_APPEND)
                  .dateHistogramInterval(DateHistogramInterval.days(1)));

        } else if (null == value) {
          searchRequestBuilder.addAggregation(
              AggregationBuilders.terms(key).field(key + RAW_APPEND));
        }
      }
    }
    long elapsedTime = calculateEndTime(startTime);
    ProjectLogger.log(
        "ElasticSearchHelper:addAggregations method end =="
            + " ,Total time elapsed = "
            + elapsedTime,
        LoggerEnum.PERF_LOG.name());
  }

  return searchRequestBuilder;
}

Source File: StatsTrendAction.java From foxtrot with Apache License 2.0

5 votes

private AbstractAggregationBuilder buildAggregation(StatsTrendRequest request, String table) {
    final String field = request.getField();
    DateHistogramInterval interval = Utils.getHistogramInterval(request.getPeriod());
    AbstractAggregationBuilder dateHistogramBuilder = Utils.buildDateHistogramAggregation(request.getTimestamp(),
                                                                                          interval);
    boolean isNumericField = Utils.isNumericField(getTableMetadataManager(), table, field);
    if (isNumericField) {
        dateHistogramBuilder
                .subAggregation(Utils.buildStatsAggregation(field, getParameter().getStats()));
        if (!AnalyticsRequestFlags.hasFlag(request.getFlags(), AnalyticsRequestFlags.STATS_SKIP_PERCENTILES)) {
            dateHistogramBuilder.subAggregation(Utils.buildPercentileAggregation(
                    field, request.getPercentiles(), request.getCompression()));
        }
    }
    else {
        dateHistogramBuilder
                .subAggregation(Utils.buildStatsAggregation(field, Collections.singleton(Stat.COUNT)));
    }

    if (CollectionUtils.isNullOrEmpty(getParameter().getNesting())) {
        return dateHistogramBuilder;
    }
    return Utils.buildTermsAggregation(getParameter().getNesting()
                                               .stream()
                                               .map(x -> new ResultSort(x, ResultSort.Order.asc))
                                               .collect(Collectors.toList()), Sets.newHashSet(dateHistogramBuilder),
                                       elasticsearchTuningConfig.getAggregationSize());
}

Source File: HistogramAction.java From foxtrot with Apache License 2.0

5 votes

private AbstractAggregationBuilder buildAggregation(HistogramRequest parameter) {
    DateHistogramInterval interval = Utils.getHistogramInterval(getParameter().getPeriod());
    DateHistogramAggregationBuilder histogramBuilder = Utils.buildDateHistogramAggregation(getParameter().getField(),
                                                                                           interval);
    if (!CollectionUtils.isNullOrEmpty(getParameter().getUniqueCountOn())) {
        histogramBuilder.subAggregation(Utils.buildCardinalityAggregation(
                getParameter().getUniqueCountOn(), parameter.accept(new CountPrecisionThresholdVisitorAdapter(
                        elasticsearchTuningConfig.getPrecisionThreshold()))));
    }
    return histogramBuilder;
}

Source File: TrendAction.java From foxtrot with Apache License 2.0

5 votes

private AbstractAggregationBuilder buildAggregation(TrendRequest request) {
    DateHistogramInterval interval = Utils.getHistogramInterval(request.getPeriod());
    String field = request.getField();

    DateHistogramAggregationBuilder histogramBuilder = Utils.buildDateHistogramAggregation(request.getTimestamp(), interval);
    if(!CollectionUtils.isNullOrEmpty(getParameter().getUniqueCountOn())) {
        histogramBuilder.subAggregation(Utils.buildCardinalityAggregation(
                getParameter().getUniqueCountOn(), request.accept(new CountPrecisionThresholdVisitorAdapter(
                        elasticsearchTuningConfig.getPrecisionThreshold()))));
    }
    return Utils.buildTermsAggregation(Lists.newArrayList(new ResultSort(field, ResultSort.Order.asc)),
            Sets.newHashSet(histogramBuilder), elasticsearchTuningConfig.getAggregationSize());
}

Source File: Utils.java From foxtrot with Apache License 2.0

5 votes

public static DateHistogramAggregationBuilder buildDateHistogramAggregation(
        String field,
        DateHistogramInterval interval) {
    String metricKey = getDateHistogramKey(field);
    return AggregationBuilders.dateHistogram(metricKey)
            .minDocCount(0)
            .field(storedFieldName(field))
            .timeZone(DateTimeZone.getDefault())
            .dateHistogramInterval(interval);
}

Source File: TestTransportClient.java From jframe with Apache License 2.0

5 votes

@Test
public void testAggregation() {
    SearchResponse sr = client.prepareSearch().setQuery(QueryBuilders.matchAllQuery())
            .addAggregation(AggregationBuilders.terms("agg1").field("field"))
            .addAggregation(AggregationBuilders.dateHistogram("agg2").field("birth").interval(DateHistogramInterval.YEAR)).execute().actionGet();

    // Get your facet results
    Terms agg1 = sr.getAggregations().get("agg1");
    // DateHistogram agg2 = sr.getAggregations().get("agg2");

    sr = client.prepareSearch("index1").setTerminateAfter(1000).get();
    if (sr.isTerminatedEarly()) {
        // We finished early
    }

    // sr = client.prepareSearch()
    // .addAggregation(
    // AggregationBuilders.terms("by_country").field("country")
    // .subAggregation(AggregationBuilders.dateHistogram("by_year")
    // .field("dateOfBirth")
    // .interval((DateHistogramInterval.YEAR)
    // .subAggregation(AggregationBuilders.avg("avg_children").field("children"))
    // )
    // ).execute().actionGet();

    MetricsAggregationBuilder aggregation = AggregationBuilders.max("agg").field("height");

}

org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval Java Examples