org.elasticsearch.search.SearchHit Java Examples

/**
 * 
 * @param indices
 * @param field
 * @param queryString
 */
private static void matchQuery(String indices,String field,String queryString){
	Client client = createTransportClient();
	SearchResponse searchResponse = client.prepareSearch(indices)
	        .setQuery(QueryBuilders.matchQuery(field, queryString))
	        .execute()
	        .actionGet();
	SearchHits searchHits = searchResponse.getHits();
	System.out.println("---------------matchquery--在["+field+"]中搜索关键字["+queryString+"]---------------------");
	System.out.println("共匹配到:"+searchHits.getTotalHits()+"条记录!");
	SearchHit[] hits = searchHits.getHits();
	for (SearchHit searchHit : hits) {
		Map<String, Object> sourceAsMap = searchHit.sourceAsMap();
		Set<String> keySet = sourceAsMap.keySet();
		for (String string : keySet) {
			System.out.println(string+":"+sourceAsMap.get(string));
		}
		System.out.println();
	}
}
 

/**
 * @return  All modules registered in elastic search
 */
public List<Module> getAll() {

    SearchResponse response = elasticSearchClient
            .getClient()
            .prepareSearch( DAO_INDEX_KEY )
            .setTypes( DAO_TYPE_KEY )
            .setSize( MAX_RESULT_SIZE )
            .execute()
            .actionGet();

    LOG.debug( "response: {}", response );

    ArrayList<Module> modules = new ArrayList<Module>();

    for ( SearchHit hit : response.getHits().hits() ) {
        modules.add( toModule( hit ) );
    }

    return modules;
}
 

private void leftJoinWithAllFromSecondTable(boolean useNestedLoops) throws SQLFeatureNotSupportedException, IOException, SqlParseException {
    String query = String.format("select c.name.firstname , d.* from %s/gotCharacters c " +
            "LEFT JOIN %s/gotCharacters d on d.name = c.house " +
            "where d.sigil <> 'direwolf'"
            ,  TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_GAME_OF_THRONES);
    if(useNestedLoops) query = query.replace("select","select /*! USE_NL*/ ");
    SearchHit[] hits = joinAndGetHits(query);
    Assert.assertEquals(10, hits.length);
    for (SearchHit hit : hits) {
        if(hit.getId().endsWith("0")){
            Assert.assertEquals(1,hit.getSourceAsMap().size());
        }
        else {
            Assert.assertEquals(5,hit.getSourceAsMap().size());
        }
    }

}
 

@Override
public Event getEvent(final long snifferId, final String eventId) {
	return clientTpl.executeWithClient(new ClientCallback<Event>() {
		@Override
		public Event execute(final Client client) {
			try {
				final SearchResponse r = client.prepareSearch(indexNamingStrategy.getRetrievalNames(snifferId))
						.setIndicesOptions(IndicesOptions.lenientExpandOpen())
						.setQuery(QueryBuilders.idsQuery().ids(eventId)).execute().get();
				if (r != null && r.getHits().hits().length > 0) {
					final SearchHit hit = r.getHits().hits()[0];
					final Event event = jsonMapper.readValue(hit.getSourceAsString(), Event.class);
					event.setId(hit.getId());
					return event;
				} else {
					return null;
				}
			} catch (final Exception e) {
				throw new DataAccessException("Failed to load for sniffer=" + snifferId + " the event: " + eventId,
						e);
			}
		}
	});
}
 

private boolean hasData(int expect) {
    SearchHits searchHits = getData();

    if (searchHits == null) {
        LOG.debug("There are not search hit to return");

        return false;
    }

    SearchHit[] hits = searchHits.getHits();
    if (hits == null) {
        LOG.debug("Empty data set");

        return false;
    }

    int count = hits.length;

    if (count != expect) {
        LOG.debug("Not enough records: {} available, but {} expected", count, expect);

        return false;
    }

    return true;
}
 

/**
 * 
 * @param hit
 * @param seach_fileds
 * @return
 */
public static Map<String, Object> getHighlight(SearchHit hit, List<String> seach_fileds) {
    Map<String, Object> result = new HashMap<String, Object>();
    Map<String, HighlightField> highlights = hit.highlightFields();
    for (String filed : seach_fileds) {
        HighlightField highlight = highlights.get(filed);
        if (null == highlight) {
            continue;
        }
        StringBuffer sb = new StringBuffer();
        Text[] fragments = highlight.fragments();
        for (Text fragment : fragments) {
            sb.append(fragment);
        }
        result.put(filed + "_HIGH", sb.toString());
    }
    return result;

}
 

@Override
public List<NetworkAddressAlias> loadLastUpdate(long timeBucketInMinute) {
    List<NetworkAddressAlias> networkAddressAliases = new ArrayList<>();

    try {
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.rangeQuery(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET)
                                               .gte(timeBucketInMinute));
        searchSourceBuilder.size(resultWindowMaxSize);

        SearchResponse response = getClient().search(NetworkAddressAlias.INDEX_NAME, searchSourceBuilder);

        final NetworkAddressAlias.Builder builder = new NetworkAddressAlias.Builder();
        for (SearchHit searchHit : response.getHits().getHits()) {
            networkAddressAliases.add(builder.map2Data(searchHit.getSourceAsMap()));
        }
    } catch (Throwable t) {
        log.error(t.getMessage(), t);
    }

    return networkAddressAliases;
}
 

private static void parseAgentIds(SearchHit hit, Occurrence occ) {
  Function<Map<String, Object>, AgentIdentifier> mapFn = m -> {
    AgentIdentifier ai = new AgentIdentifier();
    extractValue(m, "type", AgentIdentifierType::valueOf).ifPresent(ai::setType);
    extractStringValue(m, "value").ifPresent(ai::setValue);
    return ai;
  };

  getObjectsListValue(hit, RECORDED_BY_ID)
    .map(i -> i.stream().map(mapFn).collect(Collectors.toList()))
    .ifPresent(occ::setRecordedByIds);

  getObjectsListValue(hit, IDENTIFIED_BY_ID)
    .map(i -> i.stream().map(mapFn).collect(Collectors.toList()))
    .ifPresent(occ::setIdentifiedByIds);
}
 

@Override
public void onResponse(SearchResponse response) {
    SearchHits hits = response.getHits();
    LOG.info("{} ES query returned {} hits in {} msec", logIdprefix,
            hits.getHits().length, response.getTook().getMillis());
    hasFinished = hits.getHits().length == 0;
    synchronized (this.queue) {
        // Unlike standard spouts, the scroll queries should never return
        // the same
        // document twice -> no need to look in the buffer or cache
        for (SearchHit hit : hits) {
            Map<String, Object> keyValues = hit.getSourceAsMap();
            String url = (String) keyValues.get("url");
            String status = (String) keyValues.get("status");
            String nextFetchDate = (String) keyValues.get("nextFetchDate");
            Metadata metadata = fromKeyValues(keyValues);
            metadata.setValue(
                    AbstractStatusUpdaterBolt.AS_IS_NEXTFETCHDATE_METADATA,
                    nextFetchDate);
            this.queue.add(new Values(url, metadata, Status.valueOf(status)));
        }
    }
    scrollId = response.getScrollId();
    // remove lock
    markQueryReceivedNow();
}
 

private List<DetectorDocument> getDetectorsFromElasticSearch(SearchRequest searchRequest) {
    SearchResponse response;
    try {
        response = legacyElasticSearchClient.search(searchRequest, RequestOptions.DEFAULT);
    } catch (IOException e) {
        log.error("Error ES lookup", e);
        throw new RuntimeException(e);
    }

    SearchHit[] hits = response.getHits().getHits();
    List<DetectorDocument> detectors = new ArrayList<>();
    for (val hit : hits) {

        val detector = (DetectorDocument) objectMapperUtil.convertToObject(hit.getSourceAsString(), new TypeReference<DetectorDocument>() {
        });
        val newElasticsearchDetector = getElasticSearchDetector(detector);
        detectors.add(newElasticsearchDetector);
    }
    return detectors;
}
 

private ProfileTask parseTask(SearchHit data) {
    return ProfileTask.builder()
                      .id(data.getId())
                      .serviceId((String) data.getSourceAsMap().get(ProfileTaskRecord.SERVICE_ID))
                      .endpointName((String) data.getSourceAsMap().get(ProfileTaskRecord.ENDPOINT_NAME))
                      .startTime(((Number) data.getSourceAsMap().get(ProfileTaskRecord.START_TIME)).longValue())
                      .createTime(((Number) data.getSourceAsMap()
                                                .get(ProfileTaskRecord.CREATE_TIME)).longValue())
                      .duration(((Number) data.getSourceAsMap().get(ProfileTaskRecord.DURATION)).intValue())
                      .minDurationThreshold(((Number) data.getSourceAsMap()
                                                          .get(
                                                              ProfileTaskRecord.MIN_DURATION_THRESHOLD)).intValue())
                      .dumpPeriod(((Number) data.getSourceAsMap()
                                                .get(ProfileTaskRecord.DUMP_PERIOD)).intValue())
                      .maxSamplingCount(((Number) data.getSourceAsMap()
                                                      .get(ProfileTaskRecord.MAX_SAMPLING_COUNT)).intValue())
                      .build();
}
 

void assertSearch(int expectedHits, SearchResponse response, Event... events) {
  SearchHits hitResponse = response.getHits();
  assertEquals(expectedHits, hitResponse.getTotalHits());

  SearchHit[] hits = hitResponse.getHits();
  Arrays.sort(hits, new Comparator<SearchHit>() {
    @Override
    public int compare(SearchHit o1, SearchHit o2) {
      return o1.getSourceAsString().compareTo(o2.getSourceAsString());
    }
  });

  for (int i = 0; i < events.length; i++) {
    Event event = events[i];
    SearchHit hit = hits[i];
    Map<String, Object> source = hit.getSource();
    assertEquals(new String(event.getBody()), source.get("@message"));
  }
}
 

protected Double getDoubleFieldValueOrNull(final SearchHit searchHit,
		final String fieldName) {
	final SearchHitField searchHitField = searchHit.field(fieldName);
	if (searchHitField != null && searchHitField.value() != null) {
		return Double.valueOf(searchHitField.value().toString());
	}
	return null;
}
 

/**
 * Gets all commits belonging to given module.
 * <p>
 * Commits, if any, in the returned list are ordered by createTime, older first.
 *
 * @param moduleId  Module id to filter returns
 * @return          List of date ordered commits, belonging to given module
 */
public List<Commit> getByModule( String moduleId ) {
    LOG.debug( "moduleId: {}", moduleId );

    SearchResponse response = getRequest( DAO_INDEX_KEY, DAO_TYPE_KEY )
            .setQuery( termQuery( "moduleId", moduleId ) )
            .setSize( MAX_RESULT_SIZE )
            .execute().actionGet();

    TreeMap<Date, Commit> commitMap = new TreeMap<Date, Commit>();

    for ( SearchHit hit : response.getHits().hits() ) {
        Map<String, Object> json = hit.getSource();

        BasicCommit commit = new BasicCommit(
                hit.getId(),
                Util.getString( json, "moduleId" ),
                Util.getString( json, "md5" ),
                Util.toDate( Util.getString(json, "createTime" ) ),
                Util.getString( json, "runnerPath" )
        );

        commitMap.put( commit.getCreateTime(), commit );
    }

    ArrayList<Commit> commitList = new ArrayList<Commit>( commitMap.values() );
    LOG.debug( "commits: {}", commitList.size() );

    return commitList;
}
 

private static Runner toRunner( SearchHit hit ) {

        Map<String, Object> json = hit.getSource();

        return new BasicRunner(
                Util.getString( json, "ipv4Address" ),
                Util.getString( json, "hostname" ),
                Util.getInt( json, "serverPort" ),
                Util.getString( json, "url" ),
                Util.getString( json, "tempDir" )
        );
    }
 

/**
 * 根据traceId和spanId获取annotationModel列表.
 */
public List<AnnotationModel> getSpanAnnotations(final String traceId, final String spanId) {
  // 声明SearchRequestBuilder实例
  final String indice = getDefaultIndice();
  final SearchRequestBuilder builder = client.prepareSearch(indice);
  builder.setTypes(props.getEsTypeName());

  // 设置查询条件
  final BoolQueryBuilder query = new BoolQueryBuilder();
  query.must(QueryBuilders.termQuery("traceId", traceId)) //
      .must(QueryBuilders.termQuery("spanId", spanId));

  // 执行查询
  final SearchResponse response = builder.setQuery(query).execute().actionGet();

  // 处理返回结果
  final List<AnnotationModel> annos = new LinkedList<AnnotationModel>();
  for (final SearchHit hit : response.getHits().hits()) {
    final String docStr = hit.getSourceAsString();
    try {
      final AnnotationModel model = JSON.parseObject(docStr, AnnotationModel.class);
      annos.add(model);
    } catch (JSONException ex) {
      log.error("failed load data {} to AnnotationModel, error {}", docStr, ex);
      continue;
    }
  }

  return annos;
}
 

private void doSenderWork( SearchHit hit )
{	
	String kafkaBrokerHostName = configProps.getProperty("kafkaBrokerHostName", "localhost" );
	String kafkaBrokerHostPort = configProps.getProperty("kafkaBrokerHostPort", "9092" );
	String kafkaTopicName = configProps.getProperty("kafkaTopicName", "test" );
	
	logger.debug( "kafkaBrokerHostName: " + kafkaBrokerHostName );
	logger.debug( "kafkaBrokerHostPort: " + kafkaBrokerHostPort );
	logger.debug( "kafkaTopicName: " + kafkaTopicName );
	
	String sourceData = hit.getSourceAsString();
	
	logger.debug( "Source Data: " + sourceData );
	Properties props = new Properties();
	 
	props.put("metadata.broker.list", kafkaBrokerHostName + ":" + kafkaBrokerHostPort );
	props.put("serializer.class", "kafka.serializer.StringEncoder");
	// props.put("partitioner.class", "example.producer.SimplePartitioner");
	props.put("request.required.acks", "1");
	 
	ProducerConfig config = new ProducerConfig(props);
	
	Producer<String, String> producer = new Producer<String, String>(config);
	
	KeyedMessage<String, String> data = new KeyedMessage<String, String>(kafkaTopicName, "", sourceData );
	 
	producer.send(data);		
}
 

/**
 * 查询 prefix
 *
 * @param transportClient
 * @throws IOException
 */
private static void queryByPrefix(TransportClient transportClient) throws IOException {
	SearchResponse searchResponse = transportClient.prepareSearch("product_index").setTypes("product")
			.setQuery(QueryBuilders.prefixQuery("product_name", "飞利"))
			.get();

	for (SearchHit searchHit : searchResponse.getHits().getHits()) {
		logger.info("--------------------------------：" + searchHit.getSourceAsString());
	}
}
 

@Test
public void givenSearchRequest_whenMatchAll_thenReturnAllResults() throws Exception {
    SearchRequest searchRequest = new SearchRequest();
    SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
    SearchHit[] searchHits = response.getHits()
        .getHits();
    List<Person> results = Arrays.stream(searchHits)
        .map(hit -> JSON.parseObject(hit.getSourceAsString(), Person.class))
        .collect(Collectors.toList());

    results.forEach(System.out::println);
}
 

@Override
public Observable<Optional<PersistentContainerKey>> call(final PersistentContainer persistentContainer) {

    String containerName = persistentContainer.getId();

    TermQueryBuilder query = termQuery("container_id", containerName);

    final Elasticsearch elasticSearch = vertxContext.verticle().elasticsearch();

    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(format("Search Request {%s,%s} = %s", elasticSearch.defaultType(), elasticSearch.containerKeyIndex(), Jsonify.toString(query)));
    }

    SearchRequestBuilder request =
            elasticSearch.get()
                    .prepareSearch(elasticSearch.containerKeyIndex())
                    .setTypes(elasticSearch.defaultType())
                    .addSort("create_ts", DESC)
                    .setQuery(query)
                    .setVersion(true)
                    .setSize(1)
                    .setTimeout(timeValueMillis(elasticSearch.getDefaultSearchTimeout() - 10));

    return elasticSearch.execute(vertxContext, request, elasticSearch.getDefaultSearchTimeout())
            .map(oSearchResponse -> {
                SearchResponse searchResponse = oSearchResponse.get();
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug(format("Search Response {%s,%s} = %s", elasticSearch.defaultType(), elasticSearch.containerKeyIndex(), Jsonify.toString(searchResponse)));
                }
                for (SearchHit searchHit : searchResponse.getHits()) {
                    if (!searchHit.isSourceEmpty()) {
                        return of(fromSearchHit(persistentContainer, searchHit));
                    }
                }
                return Optional.<PersistentContainerKey>absent();
            });

}
 

private void lookupAnnotationFields(Map<String, Document> idMap) {
	QueryBuilder qb = QueryBuilders.idsQuery(getDocumentType()).addIds(idMap.keySet());
	SearchRequestBuilder srb = getClient().prepareSearch(getIndexName())
			.addFields("*")
			.setQuery(qb)
			.setSize(idMap.size());
	LOGGER.debug("Annotation field lookup query: {}", srb.toString());

	SearchResponse response = srb.execute().actionGet();
	for (SearchHit hit : response.getHits().getHits()) {
		populateAnnotationFields(hit, idMap.get(hit.getId()));
	}
}
 

/**
 * loads the field from the SearchHit. Loads from field not from source since
 * we aren't storing the source.
 * @param field
 * @param hit
 * @return
 */
@Override
public String getFieldFromSearchHit(String field, SearchHit hit) {
    if (hit != null && hit.getFields() != null && hit.getFields().get(field) != null) {
        return hit.getFields().get(field).value();
    }
    return null;
}
 

public static PersistentContainer fromSearchHit(PersistentAccount persistentAccount, SearchHit searchHit) {
    JsonObject document = new JsonObject(searchHit.getSourceAsString());

    return
            new PersistentContainer(persistentAccount, searchHit.getId(), searchHit.getVersion())
                    .merge(document);

}
 

public static void loadNgramFeatureBinary(ESIndex index, DataSet dataSet, Ngram feature,
                                          IdTranslator idTranslator, String docFilter){
    int featureIndex = feature.getIndex();
    SearchResponse response = index.spanNear(feature, docFilter, idTranslator.numData());
    SearchHit[] hits = response.getHits().getHits();
    for (SearchHit hit: hits){
        String indexId = hit.getId();
        float score = hit.getScore();
        int algorithmId = idTranslator.toIntId(indexId);
        if (score>0){
            score=1;
        }
        dataSet.setFeatureValue(algorithmId,featureIndex,score);
    }
}
 

@Test
public void geoDistance() throws SQLFeatureNotSupportedException, SqlParseException, InterruptedException {
    SearchHits results = query(String.format("SELECT * FROM %s/location WHERE GEO_DISTANCE(center,'1km',100.5,0.500001)", TEST_INDEX_LOCATION));
    org.junit.Assert.assertEquals(1,results.getTotalHits().value);
    SearchHit result = results.getAt(0);
    Assert.assertEquals("square",result.getSourceAsMap().get("description"));
}
 

@Test
public void testHighlight() throws Exception {
    final String field = "url_highlight_test";
    Map<String, String> docContent = new HashMap<>();
    final String url = "http://www.foo.bar.com:8080/baz/bat?bob=blah";
    docContent.put(field, url);
    client().prepareIndex(INDEX, TYPE).setSource(docContent).get();
    refresh(INDEX);

    SearchResponse response = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery(field, "www.foo.bar.com:8080"))
            .highlighter(new HighlightBuilder().preTags("<b>").postTags("</b>").field("*").forceSource(true))
            .get();

    SearchHit[] hits = response.getHits().getHits();
    assertThat(hits.length, equalTo(1));

    SearchHit hit = hits[0];
    Map<String, Object> source = hit.getSource();
    assertThat(source.size(), equalTo(1));
    assertThat(source, hasKey(field));
    assertThat("URL was stored correctly", source.get(field), equalTo(url));
    assertThat(hit.highlightFields(), hasKey(field));
    HighlightField highlightField = hit.highlightFields().get(field);
    Text[] fragments = highlightField.getFragments();
    assertThat(fragments.length, equalTo(1));
    Text fragment = fragments[0];
    assertThat("URL was highlighted correctly", fragment.string(), equalTo("http://<b>www.foo.bar.com</b>:<b>8080</b>/baz/bat?bob=blah"));
}
 

@Test
public void innerQueryTest() throws SqlParseException, SQLFeatureNotSupportedException {
    String query = String.format("select * from %s/dog where holdersName IN (select firstname from %s/account where firstname = 'Hattie')",TEST_INDEX_DOG,TEST_INDEX_ACCOUNT);
    SearchHit[] hits = query(query).getHits();
    Assert.assertEquals(1,hits.length);
    Map<String, Object> hitAsMap = hits[0].getSourceAsMap();
    Assert.assertEquals("snoopy",hitAsMap.get("dog_name"));
    Assert.assertEquals("Hattie",hitAsMap.get("holdersName"));
    Assert.assertEquals(4,hitAsMap.get("age"));

}
 

@Test
public void extractAggregationValue() {
    int docId = 0, docId1 = 1, docId2 = 2, docId3 = 3, docId4 = 400, docId5 = 5000;
    SearchHit[] hit = new SearchHit[6];
    hit[0] = new SearchHit(docId);
    hit[1] = new SearchHit(docId1);
    hit[2] = new SearchHit(docId2);
    hit[3] = new SearchHit(docId3);
    hit[4] = new SearchHit(docId4);
    hit[5] = new SearchHit(docId5);
    SearchHits hits = new SearchHits(hit, 30000, 450);

    Aggregations aggregations = new Aggregations(new ArrayList());

    List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions = new ArrayList();
    suggestions.add(new Suggest.Suggestion("sug1", 1));
    suggestions.add(new Suggest.Suggestion("sug2", 2));
    suggestions.add(new Suggest.Suggestion("sug3", 3));
    suggestions.add(new Suggest.Suggestion("sug4", 4));
    suggestions.add(new Suggest.Suggestion("sug5", 50));

    Suggest suggest = new Suggest(suggestions);

    SearchProfileShardResults profileResults = new SearchProfileShardResults(Collections.emptyMap());

    SearchResponseSections internalResponse = new SearchResponseSections(hits, aggregations, suggest, false, false, profileResults, 0);

    ShardSearchFailure[] shardFailures = new ShardSearchFailure[0];

    SearchResponse.Clusters clusters = SearchResponse.Clusters.EMPTY;

    SearchResponse sRes = new SearchResponse(internalResponse, "id1", 1, 1, 1, 1, shardFailures, clusters);

    JSONObject res = QueryUtil.extractAggregationValue(sRes);

    assertNotNull(res.get("suggest"));
    assertNotNull(res.get("hits"));
}
 

private List<EventExecution> mapEventExecutionsResponse(SearchResponse response) throws IOException {
    SearchHit[] hits = response.getHits().getHits();
    List<EventExecution> executions = new ArrayList<>(hits.length);
    for (SearchHit hit : hits) {
        String source = hit.getSourceAsString();
        EventExecution tel = objectMapper.readValue(source, EventExecution.class);
        executions.add(tel);
    }
    return executions;
}
 

/**
 * 查询 bool
 *
 * @param transportClient
 * @throws IOException
 */
private static void queryByBool(TransportClient transportClient) throws IOException {
	QueryBuilder queryBuilder = QueryBuilders.boolQuery()
			.must(QueryBuilders.matchQuery("product_name", "飞利浦"))
			.should(QueryBuilders.rangeQuery("created_date_time").gte("2017-01-01").lte("2017-12-31"))
			.filter(QueryBuilders.rangeQuery("price").gte(150).lte(400));

	SearchResponse searchResponse = transportClient.prepareSearch("product_index").setTypes("product")
			.setQuery(queryBuilder)
			.get();

	for (SearchHit searchHit : searchResponse.getHits().getHits()) {
		logger.info("--------------------------------：" + searchHit.getSourceAsString());
	}
}