Java Code Examples for org.elasticsearch.action.bulk.BulkResponse#hasFailures()

@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
    if (response.hasFailures()) {
        LOGGER.error("Bulk[{}] executed with failures", executionId);
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                LOGGER.error("Failed on {} due to {}", item.getId(), item.getFailureMessage());
                numberOfEventInError.inc();
            }
        }
    } else {
        LOGGER.info("Successfully completed Bulk[{}] in {} ms", executionId, response.getTook().getMillis());
        latencyIndexingEvents.observe(response.getTook().getMillis());
    }
    CommittableOffset<String, byte[]> lastOffset = (CommittableOffset<String, byte[]>) request.payloads().get(request.payloads().size() - 1);
    lastOffset
            .commitAsync()
            .whenComplete((topicPartitionOffset, exception) -> {
                if (exception != null) {
                    LOGGER.warn("Could not commit kafka offset {}|{}", lastOffset.getOffset(), lastOffset.getPartition());
                    numberOfOffsetCommitError.inc();
                } else {
                    LOGGER.info("Committed kafka offset {}|{}", topicPartitionOffset.getOffset(), topicPartitionOffset.getPartition());
                }
            });
}
 

@Test
public void testBulk() throws Exception {
    BulkRequestBuilder bulkRequest = client.prepareBulk();

    // either use client#prepare, or use Requests# to directly build
    // index/delete requests
    bulkRequest.add(client.prepareIndex("twitter", "tweet", "1").setSource(XContentFactory.jsonBuilder().startObject().field("user", "kimchy")
            .field("postDate", new Date()).field("message", "trying out Elasticsearch").endObject()));

    bulkRequest.add(client.prepareIndex("twitter", "tweet", "2").setSource(XContentFactory.jsonBuilder().startObject().field("user", "kimchy")
            .field("postDate", new Date()).field("message", "another post").endObject()));

    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        // process failures by iterating through each bulk response item
    }
}
 

@Override
public <T extends Entity> boolean bulkUpdate(String indexName, List<? extends Entity> entities) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();
    entities.stream().map(e -> createUpdateRequest(indexName, getType(e), e.getId(), getJson(e), getParent(e), getRoot(e))).
            forEach(bulkRequest::add);
    bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);

    BulkResponse bulkResponse = client.bulk(bulkRequest);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse resp : bulkResponse.getItems()) {
            if (resp.isFailed()) {
                LOGGER.error("bulk update failed : {}", resp.getFailureMessage());
            }
        }
        return false;
    }
    return true;
}
 

private static void prepareData(IndicesAdminClient indicesAdminClient) throws InterruptedException {
    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    //批量添加
    for (int i = 0; i < 1000; i++) {
        ItemInfo iteminfo=new ItemInfo(i,"商品"+i, new Random().nextFloat(),new Date());
        // 当opType是Index时,如果文档id存在,更新文档,否则创建文档 当opType是Create,如果文档id存在,抛出文档存在的错误 *
        IndexRequestBuilder indexRequestBuilder = client.prepareIndex(INDEX_NAME_v1, INDEX_TYPE).setId(i+"").setSource(JSONObject.toJSONString(iteminfo)).setOpType(IndexRequest.OpType.INDEX);
        bulkRequestBuilder.add(indexRequestBuilder);
        //数据日期不一样
    }
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
     log.error("bulk index error:{}",bulkResponse.buildFailureMessage());
    } else {
        log.info("index docs : {}" , bulkResponse);
    }
}
 

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.execute().actionGet();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}
 

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.get();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}
 

private void bulkDelete(String indexType, JsonNode data) {
    SearchHits hits = elasticSearchService
            .searchHitsByKeys(elasticSearchIndex, indexType, uniqueFields, uniqueFields, data);
    List<String> ids = new ArrayList<>();
    for (SearchHit hit : hits.getHits()) {
        if (hit != null) {
            ids.add(hit.getId());
        }
    }
    if (ids.size() == 0) {
        return;
    }
    BulkResponse resp = elasticSearchService.bulkDelete(elasticSearchIndex, indexType, ids);
    if (resp.hasFailures()) {
        throw new BadRequestException(resp.buildFailureMessage());
    }
}
 

private void commit(BulkRequestBuilder bulkRequestBuilder) {
    if (bulkRequestBuilder.numberOfActions() > 0) {
        BulkResponse response = bulkRequestBuilder.execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    System.out.println(itemResponse.getFailureMessage());
                } else {
                    System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}
 

void executeBuilder(BulkRequestBuilder bulkRequestBuilder)
{
	if(bulkRequestBuilder.numberOfActions() == 0)
		return;
	
	++stats.flushes;
	BulkResponse response = bulkRequestBuilder.execute().actionGet();
	if (response.hasFailures()) {
              logger.warn("failed to execute" + response.buildFailureMessage());
          }
	
	for(BulkItemResponse resp : response){
		if(resp.isFailed()){
			stats.failed++;
		}else{
			stats.succeeded++;
		}
	}
}
 

private void bulkIndex(String indexType, JsonNode data) {
    if (data.size() == 0) {
        return;
    }
    if (uniqueFields.size() > 0) {
        Set<String> keys = new HashSet<>();
        ArrayNode uniqued = Json.newArray();
        for (JsonNode point : data) {
            String key = FeatureExtractorUtilities.composeConcatenatedKey(point, dataFields);
            if (!keys.contains(key)) {
                keys.add(key);
                uniqued.add(point);
            }
        }
        data = uniqued;
    }
    BulkResponse resp = elasticSearchService.bulkIndex(elasticSearchIndex, indexType, data);
    if (resp.hasFailures()) {
        throw new BadRequestException(resp.buildFailureMessage());
    }
}
 

@Override
public void batchInsertById(String index, String type, Map<String, Map<String, Object>> idDataMap) {
    BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

    idDataMap.forEach((id, dataMap) -> bulkRequestBuilder.add(transportClient.prepareIndex(index, type, id).setSource(dataMap)));
    try {
        BulkResponse bulkResponse = bulkRequestBuilder.execute().get();
        if (bulkResponse.hasFailures()) {
            logger.error("elasticsearch批量插入错误, index=" + index + ", type=" + type + ", data=" + JsonUtil.toJson(idDataMap) + ", cause:" + bulkResponse.buildFailureMessage());
        }
    } catch (Exception e) {
        logger.error("elasticsearch批量插入错误, index=" + index + ", type=" + type + ", data=" + JsonUtil.toJson(idDataMap), e);
    }
}
 

@Override
public void executeRequest(List<HistoryEvent> historyEvents) {
  BulkRequestBuilder bulkRequestBuilder = esClient.prepareBulk();
  try {
    for (HistoryEvent historyEvent : historyEvents) {
      UpdateRequestBuilder updateRequestBuilder = prepareUpdateRequest(historyEvent);
      bulkRequestBuilder.add(updateRequestBuilder);
    }

    BulkResponse bulkResponse;
    if (WAIT_FOR_RESPONSE > 0) {
      bulkResponse = bulkRequestBuilder.get(TimeValue.timeValueSeconds(WAIT_FOR_RESPONSE));
    } else {
      bulkResponse = bulkRequestBuilder.get();
    }

    if (bulkResponse.hasFailures()) {
      LOGGER.severe("Error while executing bulk request: " + bulkResponse.buildFailureMessage());
    }

    if (LOGGER.isLoggable(Level.FINEST)) {
      for (BulkItemResponse bulkItemResponse : bulkResponse) {
        LOGGER.finest("[" + bulkItemResponse.getIndex() +
            "][" + bulkItemResponse.getType() +
            "][" + bulkItemResponse.getOpType() +
            "] process instance with id '" + bulkItemResponse.getId() + "'");
      }
    }
  } catch (IOException e) {
    LOGGER.log(Level.SEVERE, e.getMessage(), e);
  }
}
 

/**
 * @return boolean
 * @Author pancm
 * @Description 批量新增/更新数据
 * @Date 2019/3/21
 * @Param [mapList:存储参数, index:索引库名, type:索引库类型,key:存储的主键，为空表示使用ES主键]
 **/
public static boolean saveBulk(List<Map<String, Object>> mapList, String index, String type, String key) throws IOException {

    if (mapList == null || mapList.size() == 0) {
        return true;
    }
    if (index == null || index.trim().length() == 0 || type == null || type.trim().length() == 0) {
        return false;
    }
    try {
        BulkRequest request = new BulkRequest();
        mapList.forEach(map -> {
            if (key != null) {
                String id = map.get(key) + "";
                if (id == null || id.trim().length() == 0) {
                    request.add(new IndexRequest(index, type).source(map, XContentType.JSON));
                } else {
                    request.add(new IndexRequest(index, type, id).source(map, XContentType.JSON));
                }
            } else {
                request.add(new IndexRequest(index, type).source(map, XContentType.JSON));
            }
        });

        BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
        //说明至少有一个失败了，这里就直接返回false
        if (bulkResponse.hasFailures()) {
            return false;
        }

        return true;
    } finally {
        if (isAutoClose) {
            close();
        }
    }
}
 

public static void batchImport() throws SQLException, IOException {
    try {
        int frequency = 0;
        Long total = total();//导入数据的总数
        Long basicTotal = 10000L;//每次导入的量
        Long start = 0L, end = basicTotal;
        while (total > 0L) {
            List<Sku> skuList = fetchData(start, end);
            BulkResponse response = bulkImport(skuList);
            System.out.println("http：" + response.status().getStatus()
                    + " 耗时：" + response.getTook());

            if (response.hasFailures()) System.out.println("失败原因：" + response.buildFailureMessage());

            if (total <= basicTotal) {
                basicTotal = total;
            }
            total -= basicTotal;
            start += basicTotal;
            end += basicTotal;
            frequency++;
        }
        System.out.println("分批次数：" + frequency);
    } catch (SQLException e) {
        e.printStackTrace();
    } finally {
        conn.close();
        restHighLevelClient.close();
    }
}
 

@Override
public void execute() throws Exception {
    try {
        BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
        if (bulkResponse.hasFailures()) {
            throw new EventDeliveryException(bulkResponse.buildFailureMessage());
        }
    } finally {
        bulkRequestBuilder = client.prepareBulk();
    }
}
 

@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}
 

/**
 * 批量更新网页
 *
 * @param webpageList 网页列表
 * @return
 */
public boolean update(List<Webpage> webpageList) throws ExecutionException, InterruptedException {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (Webpage webpage : webpageList) {
        UpdateRequest updateRequest = new UpdateRequest();
        updateRequest.index(INDEX_NAME);
        updateRequest.type(TYPE_NAME);
        updateRequest.id(webpage.getId());
        updateRequest.doc(gson.toJson(webpage));
        bulkRequest.add(updateRequest);
    }
    BulkResponse bulkResponse = bulkRequest.get();
    return bulkResponse.hasFailures();
}
 

public boolean bulkUpdate(final Consumer<BulkRequestBuilder> consumer) {
    final BulkRequestBuilder builder = ComponentUtil.getFessEsClient().prepareBulk();
    consumer.accept(builder);
    try {
        final BulkResponse response = builder.execute().get();
        if (response.hasFailures()) {
            throw new FessEsClientException(response.buildFailureMessage());
        } else {
            return true;
        }
    } catch (InterruptedException | ExecutionException e) {
        throw new FessEsClientException("Failed to update bulk data.", e);
    }
}
 

@Override
public void handle(CollectDataFrame frame) {

    if (this.log.isDebugEnable()) {
        this.log.debug(this, frame.toJSONString());
    }

    BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();
    String appUUID = frame.getTarget();
    // 从uuid中获取appid
    String appid = appUUID.split("---")[1];
    for (Line line : frame.getLines()) {

        String content = line.getContent();

        try {
            // 提取epinfo
            StringBuilder builder = new StringBuilder();
            int headPoint = 0;
            for (int i = 0; i < content.length(); i++) {
                char item = content.charAt(i);
                if (item == ';') {
                    headPoint++;
                    if (headPoint == 3) {
                        break;
                    }
                }
                else {
                    if (headPoint > 1) {
                        builder.append(item);
                    }
                }
            }
            String epinfo = builder.toString();
            // 若epinfo为数字则说明为方法级
            if (DataConvertHelper.toInt(epinfo, -1) != -1) {
                epinfo = "method";
            }
            IActionEngine engine = this.getActionEngineMgr().getActionEngine("SlowOperActionEngine");
            ActionContext ac = new ActionContext();
            ac.putParam("content", content);
            ac.putParam("appid", appid);
            engine.execute(epinfo, ac);
            SlowOperSpan span = (SlowOperSpan) ac.getParam("span");

            pushSpanToBulkRequest(appUUID, frame.getAppgroup(), span, bulkRequest,
                    (String) ac.getParam("protocolType"));
        }
        catch (Exception e) {
            // 防止有不合法的协议报文出现
            this.log.err(this, "unsupported protocol,content is" + frame.toJSONString(), e);
        }
    }
    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        log.err(this, "INSERT InvokeChain Data to ES FAIL: " + bulkResponse.buildFailureMessage());
    }

}
 

public int doIndex() {

		element_count++;
		
		if (element_count != _bulk_size)
			return 0;

		if (element_count == _bulk_size) {
			_LOG.debug("Starting bulk load of size: " + _bulk_size);
			BulkResponse resp = bulkRequest.execute().actionGet();
			element_count = 0;
			_LOG.debug("Received bulk response: " + resp.toString());

			if (resp.hasFailures()) {
				_LOG.error("Bulk update failed");
				return 2;
			}

			if (!running_index_postfix.equals(index_postfix)) {

				_LOG.debug("Attempting to apply a new alias");

				try {

					String alias = "{\"actions\" : [{ \"add\" : { \"index\" : \""
							+ _index_name
							+ "-"
							+ index_postfix
							+ "\", \"alias\" : \"" + _index_name + "\" } } ]}";

					post.setEntity(new StringEntity(alias));

					HttpResponse response = httpclient.execute(post);
					String res = EntityUtils.toString(response.getEntity());

					_LOG.debug("Alias request received the following response: "
							+ res);

					running_index_postfix = index_postfix;
				}

				catch (Exception e) {
					e.printStackTrace();
					_LOG.error("Alias request failed...");
					return 2;
				}
			}

			index_postfix = dateFormat.format(new Date());
		}

		_LOG.debug("Adding to bulk load: element " + element_count
				+ " of bulk size " + _bulk_size);

		return 1;
	}