org.elasticsearch.search.SearchHit#getSource

Source File: AbstractElasticSearchSinkTest.java From mt-flume with Apache License 2.0

6 votes

void assertSearch(int expectedHits, SearchResponse response, Event... events) {
  SearchHits hitResponse = response.getHits();
  assertEquals(expectedHits, hitResponse.getTotalHits());

  SearchHit[] hits = hitResponse.getHits();
  Arrays.sort(hits, new Comparator<SearchHit>() {
    @Override
    public int compare(SearchHit o1, SearchHit o2) {
      return o1.getSourceAsString().compareTo(o2.getSourceAsString());
    }
  });

  for (int i = 0; i < events.length; i++) {
    Event event = events[i];
    SearchHit hit = hits[i];
    Map<String, Object> source = hit.getSource();
    assertEquals(new String(event.getBody()), source.get("@message"));
  }
}

Source File: ElasticSearchComponent.java From metron with Apache License 2.0

6 votes

public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType,
    String subMessage) throws IOException {
  getClient().admin().indices().refresh(new RefreshRequest());
  SearchResponse response = getClient().prepareSearch(index)
      .setTypes(sourceType)
      .setFrom(0)
      .setSize(1000)
      .execute().actionGet();
  List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>();
  for (SearchHit hit : response.getHits()) {
    Object o = null;
    if (subMessage == null) {
      o = hit.getSource();
    } else {
      o = hit.getSource().get(subMessage);
    }
    ret.add((Map<String, Object>) (o));
  }
  return ret;
}

Source File: ElasticsearchSearchDao.java From metron with Apache License 2.0

6 votes

private SearchResult getSearchResult(SearchHit searchHit, List<String> fields) {
  SearchResult searchResult = new SearchResult();
  searchResult.setId(searchHit.getId());
  Map<String, Object> source;
  if (fields != null) {
    Map<String, Object> resultSourceAsMap = searchHit.getSourceAsMap();
    source = new HashMap<>();
    fields.forEach(field -> {
      source.put(field, resultSourceAsMap.get(field));
    });
  } else {
    source = searchHit.getSource();
  }
  searchResult.setSource(source);
  searchResult.setScore(searchHit.getScore());
  searchResult.setIndex(searchHit.getIndex());
  return searchResult;
}

Source File: IndexingComponent.java From elasticsearch-reindex-tool with Apache License 2.0

6 votes

public Optional<BulkResult> indexData(ElasticDataPointer targetDataPointer, SearchHit[] hits) {
  BulkRequestBuilder bulkRequest = createBulkRequestBuilder();

  for (SearchHit hit : hits) {
    Map<String, Object> source = hit.getSource();

    IndexRequestBuilder requestBuilder = prepareIndex(targetDataPointer.getIndexName(), targetDataPointer
        .getTypeName(), hit.getId(), source, hit.getIndex());
    if (hit.getFields().get("_ttl") != null) {
      requestBuilder.setTTL(hit.getFields().get("_ttl").value());
    }
    if (hit.getFields().get("_routing") != null) {
      requestBuilder.setRouting(hit.getFields().get("_routing").value());
    }
    requestBuilder.setSource(source);
    bulkRequest.add(requestBuilder);
  }
  return executeBulk(hits.length, bulkRequest);
}

Source File: AbstractElasticSearchSinkTest.java From ingestion with Apache License 2.0

6 votes

void assertSearch(int expectedHits, SearchResponse response, Map<String, Object> expectedBody, Event... events) {
  SearchHits hitResponse = response.getHits();
  assertEquals(expectedHits, hitResponse.getTotalHits());

  SearchHit[] hits = hitResponse.getHits();
  Arrays.sort(hits, new Comparator<SearchHit>() {
    @Override
    public int compare(SearchHit o1, SearchHit o2) {
      return o1.getSourceAsString().compareTo(o2.getSourceAsString());
    }
  });

  for (int i = 0; i < events.length; i++) {
    Event event = events[i];
    SearchHit hit = hits[i];
    Map<String, Object> source = hit.getSource();
    if (expectedBody == null) {
      assertEquals(new String(event.getBody()), source.get("@message"));
    } else {
      assertEquals(expectedBody, source.get("@message"));
    }
  }
}

Source File: SearchResultsGeneratorImpl.java From nexus-public with Eclipse Public License 1.0

6 votes

@Override
public List<SearchResultComponent> getSearchResultList(final SearchResponse response) {
  List<SearchResultComponent> searchResultComponents = new ArrayList<>((int) response.getHits().getTotalHits());
  Set<String> componentIdSet = new HashSet<>();

  for (SearchHit hit : response.getHits().getHits()) {
    Map<String, Object> source = hit.getSource();
    String format = source.get(FORMAT).toString();
    SearchResultComponentGenerator generator = searchResultComponentGeneratorMap
        .getOrDefault(format, defaultSearchResultComponentGenerator);
    SearchResultComponent component = generator.from(hit, componentIdSet);
    if (null != component) {
      componentIdSet.add(component.getId());
      searchResultComponents.add(component);
    }
  }

  return searchResultComponents;
}

Source File: RunDao.java From usergrid with Apache License 2.0

5 votes

private static Run toRun( SearchHit hit ) {
    Map<String, Object> json = hit.getSource();

    BasicRun run = new BasicRun(
            Util.getString( json, "id" ),
            Util.getString( json, "commitId" ),
            Util.getString( json, "runner" ),
            Util.getInt( json, "runNumber" ),
            Util.getString(json, "testName" )
    );
    run.copyJson( hit.getSource() );

    return run;
}

Source File: ImageIntegrationTests.java From elasticsearch-image with Apache License 2.0

5 votes

private void assertImageScore(SearchHits hits, String name, float score) {
    for (SearchHit hit : hits) {
        if ((hit.getSource() != null && hit.getSource().get("name").equals(name))
                || (hit.getFields() != null && !hit.getFields().isEmpty() && hit.getFields().get("name").getValue().equals(name))){
            assertThat(hit.getScore(), equalTo(score));
            return;
        }
    }
    throw new AssertionError("Image " + name + " not found");
}

Source File: IndexTest.java From bigdata-tutorial with Apache License 2.0

5 votes

public void search() throws Exception {
	try {
		QueryBuilder qb = QueryBuilders.termQuery("title", "大师");
		SearchResponse scrollResp = client.prepareSearch("app").setSearchType(SearchType.SCAN).setScroll(
				new TimeValue(60000)).setQuery(qb).setSize(100).execute().actionGet();

		while (true) {
			scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
			for (SearchHit hit : scrollResp.getHits()) {
				Map<String, Object> source = hit.getSource();
				if (!source.isEmpty()) {
					for (Iterator<Map.Entry<String, Object>> it = source.entrySet().iterator(); it.hasNext();) {
						Map.Entry<String, Object> entry = it.next();
						System.out.println(entry.getKey() + "=======" + entry.getValue());

					}
				}

			}
			if (scrollResp.getHits().getTotalHits() == 0) {
				break;
			}

		}
	}
	catch (Exception e) {
		e.printStackTrace();
	}
	finally {
		client.close();
	}

}

Source File: URLTokenizerIntegrationTest.java From elasticsearch-analysis-url with Apache License 2.0

5 votes

@Test
public void testHighlight() throws Exception {
    final String field = "url_highlight_test";
    Map<String, String> docContent = new HashMap<>();
    final String url = "http://www.foo.bar.com:8080/baz/bat?bob=blah";
    docContent.put(field, url);
    client().prepareIndex(INDEX, TYPE).setSource(docContent).get();
    refresh(INDEX);

    SearchResponse response = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery(field, "www.foo.bar.com:8080"))
            .highlighter(new HighlightBuilder().preTags("<b>").postTags("</b>").field("*").forceSource(true))
            .get();

    SearchHit[] hits = response.getHits().getHits();
    assertThat(hits.length, equalTo(1));

    SearchHit hit = hits[0];
    Map<String, Object> source = hit.getSource();
    assertThat(source.size(), equalTo(1));
    assertThat(source, hasKey(field));
    assertThat("URL was stored correctly", source.get(field), equalTo(url));
    assertThat(hit.highlightFields(), hasKey(field));
    HighlightField highlightField = hit.highlightFields().get(field);
    Text[] fragments = highlightField.getFragments();
    assertThat(fragments.length, equalTo(1));
    Text fragment = fragments[0];
    assertThat("URL was highlighted correctly", fragment.string(), equalTo("http://<b>www.foo.bar.com</b>:<b>8080</b>/baz/bat?bob=blah"));
}

Source File: RunnerDao.java From usergrid with Apache License 2.0

5 votes

/**
 * Returns a runner group from the search
 */
private static RunnerGroup getGroup( SearchHit hit ) {
    Map<String, Object> json = hit.getSource();

    return new RunnerGroup(
            Util.getString( json, "user" ),
            Util.getString( json, "commitId" ),
            Util.getString( json, "moduleId" )
    );
}

Source File: RunResultDao.java From usergrid with Apache License 2.0

5 votes

private static RunResult toRunResult( SearchHit hit ) {

        Map<String, Object> json = hit.getSource();

        return new BasicRunResult(
                hit.getId(),
                Util.getString( json, "runId" ),
                Util.getInt( json, "runCount" ),
                Util.getInt( json, "runTime" ),
                Util.getInt( json, "ignoreCount" ),
                Util.getInt( json, "failureCount" ),
                Util.getString( json, "failures" )
        );
    }

Source File: Elasticsearch.java From jlogstash-input-plugin with Apache License 2.0

5 votes

/**
   * 处理响应结果
   *
   * @param searchResponse
   * @return 处理后的结果
   * @Title 处理查询响应结果
   * @category 处理查询响应结果
   * @Description: 循环遍历解析searchResponse，并处理高亮数据，将数据最终以List<Map
   * <String,Object>>形式装载并返回。 返回结果太多可能会导致内存消耗剧增
   */
  private int submitMessage(SearchResponse searchResponse) throws InterruptedException {

/* 判断是否有响应结果 */
      SearchHits hits = searchResponse.getHits();

/* 遍历结果集 */
      SearchHit[] hitsArr = hits.getHits();
      int len = hitsArr.length;
      for (int i = 0; i < len; i++) {
          SearchHit hit = hitsArr[i];
          Map<String, Object> row = hit.getSource(); // 一整行数据

          // 添加doc元数据信息
          if (docinfo) {
              Map<String, String> docInfoMap = new HashMap<String, String>();

              if (indexMetadata) {
                  docInfoMap.put("_index", hit.getIndex());
              }
              if (typeMetadata) {
                  docInfoMap.put("_type", hit.getType());
              }
              if (idMetadata) {
                  docInfoMap.put("_id", hit.getId());
              }

              row.put(docinfo_target, docInfoMap);
          }

          process(row);
      }
      return len;
  }

Source File: CommitDao.java From usergrid with Apache License 2.0

5 votes

/**
 * Gets all commits belonging to given module.
 * <p>
 * Commits, if any, in the returned list are ordered by createTime, older first.
 *
 * @param moduleId  Module id to filter returns
 * @return          List of date ordered commits, belonging to given module
 */
public List<Commit> getByModule( String moduleId ) {
    LOG.debug( "moduleId: {}", moduleId );

    SearchResponse response = getRequest( DAO_INDEX_KEY, DAO_TYPE_KEY )
            .setQuery( termQuery( "moduleId", moduleId ) )
            .setSize( MAX_RESULT_SIZE )
            .execute().actionGet();

    TreeMap<Date, Commit> commitMap = new TreeMap<Date, Commit>();

    for ( SearchHit hit : response.getHits().hits() ) {
        Map<String, Object> json = hit.getSource();

        BasicCommit commit = new BasicCommit(
                hit.getId(),
                Util.getString( json, "moduleId" ),
                Util.getString( json, "md5" ),
                Util.toDate( Util.getString(json, "createTime" ) ),
                Util.getString( json, "runnerPath" )
        );

        commitMap.put( commit.getCreateTime(), commit );
    }

    ArrayList<Commit> commitList = new ArrayList<Commit>( commitMap.values() );
    LOG.debug( "commits: {}", commitList.size() );

    return commitList;
}

Source File: HistoryEventsMappingTest.java From camunda-bpm-elasticsearch with Apache License 2.0

4 votes

@Test
//  @ElasticsearchIndex(
//      indexName = ES_DEFAULT_INDEX_NAME_CAMUNDA_BPM,
//      cleanAfter = true,
//      mappings = @ElasticsearchMapping(typeName = ES_DEFAULT_TYPE_NAME_CAMUNDA_BPM, properties = {
//          @Elasticsearch
//      }))
  public void testIndexingSingleInvoice() throws IOException {
    HashMap<String,ProcessDataContainer> processesById = TestDataGenerator.startInvoiceProcess(processEngineRule.getProcessEngine(), 1);

    String[] pids = processesById.keySet().toArray(new String[0]);

    // elasticsearch //////////////////////////////

    flushAndRefresh();

    showMappings(ES_DEFAULT_INDEX_NAME_CAMUNDA_BPM);

    FilteredQueryBuilder query = QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter("processInstanceId", pids[0]));

    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(ES_DEFAULT_INDEX_NAME_CAMUNDA_BPM)
        .setQuery(query);

    IoUtil.writeToFile(searchRequestBuilder.toString(), this.getClass().getSimpleName() + ".json", true);

    SearchResponse searchResponse = searchRequestBuilder.get();
    SearchHits hits = searchResponse.getHits();
    assertEquals(1, hits.totalHits());


    SearchHit hit = hits.getAt(0);
    assertEquals(pids[0], hit.getId());
    assertEquals(ES_DEFAULT_TYPE_NAME_CAMUNDA_BPM, hit.getType());

    Map<String,Object> source = hit.getSource();
    assertNotNull(source.get("startTime"));
    assertNotNull(source.get("endTime"));
    ArrayList variables = (ArrayList) source.get("variables");
    assertEquals(5, variables.size());
    ArrayList tasks = (ArrayList) source.get("tasks");
    assertEquals(9, tasks.size());
    ArrayList activities = (ArrayList) source.get("activities");
    assertEquals(19, activities.size());

    logger.info(searchResponse.toString());

//    for (SearchHit searchHit : searchResponse.getHits()) {
//      logger.info(searchHit.sourceAsString());
//    }

    showMappings(ES_DEFAULT_INDEX_NAME_CAMUNDA_BPM);

    assertMappings(ES_DEFAULT_INDEX_NAME_CAMUNDA_BPM);
    // TODO: write assertions for mapping
  }

Source File: FieldReader.java From elasticsearch-inout-plugin with Apache License 2.0

4 votes

@Override
public Map<String, Object> read(SearchHit hit) {
    return hit.getSource();
}

Source File: EsSearch.java From es-service-parent with Apache License 2.0

4 votes

/**
 * 搜索入口
 * 
 * @param request
 * @param timeout
 * @param timeUnit
 * @return
 */
public static EsResponse search(EsRequest request, int timeout, TimeUnit timeUnit) {
    // 索引
    String indexname = request.getIndexname();
    // 结束位置
    int size = request.getPs();
    // 开始位置
    int from = request.getPn() < 0 ? 0 : (request.getPn() - 1) * size;

    // 主查询体
    SearchRequestBuilder srb = ESClient.getClient().prepareSearch(indexname)
            .setTypes(request.getTypes().toArray(new String[] {})).setFrom(from).setSize(size)
            .setPreference(Constants.preference).setTimeout(new TimeValue(timeout, timeUnit));

    // 构造查询体
    EsQueryBuilder esQueryBuilder = new EsQueryBuilder(request);
    esQueryBuilder.makeFilterBuilder(srb);
    if (esQueryBuilder.makeQueryBuilder(srb) == null) {
        return new EsResponse();
    }

    // 处理返回字段
    if (!request.getSafeResultFileds().isEmpty()) {
        srb.addFields(request.getSafeResultFileds().toArray(new String[] {}));
    }
    // 高亮
    EsHighLight.setHighLight(srb, request.getSafeHighlightFields());
    // 排序
    EsSort.sortAdapter(srb, request.getSafeSortFileds());
    // 调试模式
    if (Constants.isDebug) {
        srb.setExplain(true);
        logger.info("\r\n" + srb.toString());
    }

    // 返回结果
    SearchResponse response = srb.execute().actionGet();
    List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
    for (SearchHit hit : response.getHits()) {
        Map<String, Object> record = hit.getSource();
        if (record == null && (record = new HashMap<String, Object>()) != null) {
            for (Map.Entry<String, SearchHitField> e : hit.fields().entrySet()) {
                record.put(e.getKey(), e.getValue().getValue());
            }
        }
        record.put("ESSCORE", hit.getScore());
        // 高亮返回
        if (request.getSafeHighlightFields() != null) {
            record.putAll(EsHighLight.getHighlight(hit, request.getSafeHighlightFields()
                    .getFields()));
        }
        result.add(record);
    }

    // 调试模式
    if (Constants.isDebug) {
        logger.info("search result :{}", JsonUtil.toJson(result));
        logger.info("search TotalHits :{}", response.getHits().getTotalHits());
        logger.info("search TookInMillis :{}", response.getTookInMillis());
    }

    logger.info("search log for index:{},keys:{}", indexname, request.getAllValue());
    return new EsResponse((int) response.getHits().getTotalHits(), JsonUtil.toJson(result), 0);
}

Source File: FullTextSearchSqlDocumentFilterConverter.java From metasfresh-webui-api-legacy with GNU General Public License v3.0

4 votes

private int extractId(final SearchHit hit, final String esKeyColumnName)
{
	final Map<String, Object> source = hit.getSource();
	return NumberUtils.asInt(source.get(esKeyColumnName), -1);
}

Source File: ESClientTest.java From emotional_analysis with Apache License 2.0

4 votes

@Test
  public void test5(){
      BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();  
      //使用should实现或者查询  
      boolBuilder.should(QueryBuilders.matchQuery("content","爱我"));  
      //c查询  
      SearchRequestBuilder searchRequestBuilder = this.client.prepareSearch("hotcomments")  
              .setTypes("logs")  
              .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) //设置查询类型：1.SearchType.DFS_QUERY_THEN_FETCH 精确查询； 2.SearchType.SCAN 扫描查询,无序  
              .setQuery(boolBuilder)  
              .setSize(10);  
 
      //设置高亮显示  
      searchRequestBuilder.setHighlighterPostTags("</span>");
      searchRequestBuilder.setHighlighterPreTags("<span style=\"color:red\">");
      searchRequestBuilder.addHighlightedField("content"); 
      //执行结果  
      SearchResponse response = searchRequestBuilder.get();  
      //接受结果  
      List<Map<String,Object>> result = new ArrayList<>();  
      //遍历结果  
      for(SearchHit hit:response.getHits()){  
          Map<String, Object> source = hit.getSource();  
          //处理高亮片段  
          Map<String, HighlightField> highlightFields = hit.getHighlightFields();  
          HighlightField nameField = highlightFields.get("content");  
          if(nameField!=null){  
              Text[] fragments = nameField.fragments();  
              String nameTmp ="";  
              for(Text text:fragments){  
                  nameTmp+=text;  
              }  
              //将高亮片段组装到结果中去  
              source.put("content",nameTmp);
          }  
          result.add(source);  
      }  
      for (Map<String, Object> map : result) {
	Set<Entry<String, Object>> entrySet = map.entrySet();
	for (Entry<String, Object> entry : entrySet) {
		System.out.println(entry.getKey()+":::"+entry.getValue());
	}
}
  }

Source File: SearchServiceImpl.java From emotional_analysis with Apache License 2.0

4 votes

@Override
public PageBean<Map<String,Object>> getSearchResult(int page, int rows,String content) throws Exception {

	PageBean<Map<String, Object>> searchListPageBean = new PageBean<Map<String, Object>>();
	BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();
	boolBuilder.must(QueryBuilders.matchQuery("content", content));
	PageHelper.startPage(page, rows);
	SearchRequestBuilder searchRequestBuilder = ESClientUtils.getClient().prepareSearch("hotcomments")
			.setTypes("logs").setSearchType(SearchType.DFS_QUERY_THEN_FETCH) 
			.setQuery(boolBuilder).setFrom(page).setSize(3);
	long totalHits = ESClientUtils.getClient().prepareSearch("hotcomments").setTypes("logs")
			.setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(boolBuilder).get().getHits().getTotalHits();
	searchRequestBuilder.setHighlighterPostTags("</span>");
	searchRequestBuilder.setHighlighterPreTags("<span style=\"color:red\">");
	searchRequestBuilder.addHighlightedField("content");
	// 执行结果
	SearchResponse response = searchRequestBuilder.get();
	// 接受结果
	List<Map<String, Object>> result = new ArrayList<>();
	// 遍历结果
	for (SearchHit hit : response.getHits()) {
		Map<String, Object> source = hit.getSource();
		String songName = null;
		String songUrl = null;
		Set<Entry<String, Object>> entrySet = source.entrySet();
		for (Entry<String, Object> entry : entrySet) {
			if (entry.getKey().equals("songId")) {
				Integer songId = (Integer) entry.getValue();
				songName = SearchUtils.getSongNameById(songId);
				songUrl = SearchUtils.getSongUrlById(songId);
			}
		}
		source.put("songName", songName);
		source.put("songUrl", songUrl);
		// 处理高亮片段
		Map<String, HighlightField> highlightFields = hit.getHighlightFields();
		HighlightField nameField = highlightFields.get("content");
		if (nameField != null) {
			Text[] fragments = nameField.fragments();
			String nameTmp = "";
			for (Text text : fragments) {
				nameTmp += text;
			}
			// 将高亮片段组装到结果中去
			source.put("content", nameTmp);
		}
		result.add(source);
	}
	long totalPage = (totalHits + 3 - 1) / 3;
	if(page > totalPage){
		page = 1;
	}
	searchListPageBean.setPage(page);
	searchListPageBean.setTotalPage(totalPage);
	searchListPageBean.setList(result);
	return searchListPageBean;
}

Java Code Examples for org.elasticsearch.search.SearchHit#getSource()