Java Code Examples for org.elasticsearch.action.bulk.BulkRequest#add()

public static BulkResponse bulkImport(List<Sku> skuList) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();
    for (Sku sku : skuList) {
        Map<String, Object> skuMap = new HashMap<String, Object>();
        IndexRequest indexRequest = new IndexRequest("sku", "doc", sku.getSkuId());
        skuMap.put("name", sku.getName());
        skuMap.put("price", sku.getPrice());
        skuMap.put("image", sku.getImage());
        skuMap.put("createTime", DateUtils.dateFormat(sku.getCreateTime()));
        skuMap.put("categoryName", sku.getCategoryName());
        skuMap.put("brandName", sku.getBrandName());
        // "{'颜色': '红色', '版本': '8GB+128GB'}"
        Map spec = JSON.parseObject(sku.getSpec());
        skuMap.put("spec", spec);
        skuMap.put("commentNum", sku.getCommentNum());
        skuMap.put("saleNum", sku.getSaleNum());
        skuMap.put("spuId", sku.getSpuId());
        indexRequest.source(skuMap);
        bulkRequest.add(indexRequest);
    }
    return restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}
 

public void bulkDocument(String type, Map<String, Map<String, Object>> sources) {
    try {
        if (this.instance() == null) return;
        BulkRequest requests = new BulkRequest();
        Iterator<String> it = sources.keySet().iterator();
        int count = 0;
        while (it.hasNext()) {
            count++;
            String next = it.next();
            IndexRequest request = new IndexRequest(name, type, next);
            request.source(sources.get(next));
            requests.add(request);
            if (count % 1000 == 0) {
                client.bulk(requests, RequestOptions.DEFAULT);
                requests.requests().clear();
                count = 0;
            }
        }
        if (requests.numberOfActions() > 0) client.bulk(requests, RequestOptions.DEFAULT);
    } catch (IOException e) {
        log.error(e.getMessage());
    }
}
 

protected Pair<BulkRequest, FutureCallbackHolder> prepareBatch(Batch<Object> batch, WriteCallback callback) {
  BulkRequest bulkRequest = new BulkRequest();
  final StringBuilder stringBuilder = new StringBuilder();
  for (Object record : batch.getRecords()) {
    try {
      byte[] serializedBytes = this.serializer.serializeToJson(record);
      log.debug("serialized record: {}", serializedBytes);
      IndexRequest indexRequest = new IndexRequest(this.indexName, this.indexType)
          .source(serializedBytes, 0, serializedBytes.length, XContentType.JSON);
      if (this.idMappingEnabled) {
        String id = this.typeMapper.getValue(this.idFieldName, record);
        indexRequest.id(id);
        stringBuilder.append(";").append(id);
      }
      bulkRequest.add(indexRequest);
    }
    catch (Exception e) {
      log.error("Encountered exception {}", e);
    }
  }
  FutureCallbackHolder futureCallbackHolder = new FutureCallbackHolder(callback,
      exception -> log.error("Batch: {} failed on ids; {} with exception {}", batch.getId(),
          stringBuilder.toString(), exception),
      this.malformedDocPolicy);
  return new Pair(bulkRequest, futureCallbackHolder);
}
 

public static void main(String[] args) throws IOException {
    try{
        RestHighLevelClient client = HighLevelClient.getInstance();
        BulkRequest bulkRequest = new BulkRequest();
        for(int i=1;i<4;i++) {
            bulkRequest.add(new IndexRequest("jingma2_20180716", "testlog", String.valueOf(i)).source(buildIndexData()));
        }
        bulkRequest.add(new DeleteRequest("jingma2_20180716", "testlog", "1"));
        bulkRequest.add(new UpdateRequest("jingma2_20180716", "testlog", "2").doc(XContentType.JSON,"name","马靖2"));

        BulkResponse bulkResponse = client.bulk(bulkRequest);
        System.out.println(bulkResponse);

    }finally{
        HighLevelClient.close();
    }
}
 

public String bulkDelete(String index,String type,String... ids){
	try {
		if(xclient==null){
			init();
		}
		BulkRequest request = new BulkRequest();
		for (String id : ids) {
			request.add(new DeleteRequest(index, type, id));
		}
		BulkResponse result = xclient.bulk(request);
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 

private Mono<BulkResponse> deleteRetrievedIds(SearchResponse searchResponse, RoutingKey routingKey) {
    BulkRequest request = new BulkRequest();

    for (SearchHit hit : searchResponse.getHits()) {
        request.add(
            new DeleteRequest(aliasName.getValue())
                .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
                .id(hit.getId())
                .routing(routingKey.asString()));
    }

    return client.bulk(request, RequestOptions.DEFAULT);
}
 

public void bulkExecute(List<Event> events) throws Exception {
    //批量插入数据
    BulkRequest request = new BulkRequest();
    String indexName = null;
    for (Event event : events) {
        //如果没有切换天，那么索引可以服用，无需重复创建
        if (StringUtils.isEmpty(indexName) || !indexName.endsWith(indexNameBuilder.getIndexSuffix(event))) {
            indexName = indexNameBuilder.getIndexName(event);
        }
        request.add(new IndexRequest(indexName, indexType).source(eventSerializer.serializer(event), XContentType.JSON));
    }
    BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
    TimeValue took = bulkResponse.getTook();
    logger.debug("[批量新增花费的毫秒]:" + took + "," + took.getMillis() + "," + took.getSeconds() + ",events[" + events.size() + "]");
}
 

@Override
public boolean bulkAdd(final String indexName, Pipeline.Type nerType, List<NamedEntity> namedEntities, Document parent) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();

    String routing = ofNullable(parent.getRootDocument()).orElse(parent.getId());
    bulkRequest.add(new UpdateRequest(indexName, esCfg.indexType, parent.getId()).doc(
            jsonBuilder().startObject()
                    .field("status", Document.Status.DONE)
                    .endObject()).routing(routing));
    bulkRequest.add(new UpdateRequest(indexName, esCfg.indexType, parent.getId())
            .script(new Script(ScriptType.INLINE, "painless",
                    "if (!ctx._source.nerTags.contains(params.nerTag)) ctx._source.nerTags.add(params.nerTag);",
                    new HashMap<String, Object>() {{put("nerTag", nerType.toString());}})).routing(routing));

    for (Entity child : namedEntities) {
        bulkRequest.add(createIndexRequest(indexName, JsonObjectMapper.getType(child), child.getId(),
                getJson(child), parent.getId(), routing));
    }
    bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);

    BulkResponse bulkResponse = client.bulk(bulkRequest);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse resp : bulkResponse.getItems()) {
            if (resp.isFailed()) {
                LOGGER.error("bulk add failed : {}", resp.getFailureMessage());
            }
        }
        return false;
    }
    return true;
}
 

public BulkResponse add(Map<String, Object> jsonDoc) throws IOException {
    final BulkRequest bulkIndexRequest = new BulkRequest();
    bulkIndexRequest.add(ElasticRequestUtils.getIndexRequest(defaultIndex,jsonDoc));
    bulkIndexRequest.timeout(TimeValue.timeValueMillis(connectionTimeOut));
    bulkIndexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    return client.bulk(bulkIndexRequest, RequestOptions.DEFAULT);
}
 

@Override
public void synchronous(List<PrepareRequest> prepareRequests) {
    if (CollectionUtils.isNotEmpty(prepareRequests)) {
        BulkRequest request = new BulkRequest();

        for (PrepareRequest prepareRequest : prepareRequests) {
            if (prepareRequest instanceof InsertRequest) {
                request.add((IndexRequest) prepareRequest);
            } else {
                request.add((UpdateRequest) prepareRequest);
            }
        }
        getClient().synchronousBulk(request);
    }
}
 

@Override
public BulkDocumentWriterResults<D> write() {
    BulkDocumentWriterResults<D> results = new BulkDocumentWriterResults<>();
    try {
        // create an index request for each document
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.setRefreshPolicy(refreshPolicy);
        for(Indexable doc: documents) {
            DocWriteRequest request = createRequest(doc.document, doc.index);
            bulkRequest.add(request);
        }

        // submit the request and handle the response
        BulkResponse bulkResponse = client.getHighLevelClient().bulk(bulkRequest);
        handleBulkResponse(bulkResponse, documents, results);
        if (LOG.isDebugEnabled()) {
            String shards = Arrays.stream(bulkResponse.getItems())
                    .map(bulkItemResponse -> bulkItemResponse.getResponse().getShardId().toString())
                    .collect(Collectors.joining(","));
            LOG.debug("{} results written to shards {} in {} ms; batchSize={}, success={}, failed={}",
                    bulkResponse.getItems().length, shards, bulkResponse.getTookInMillis(),
                    documents.size(), results.getSuccesses().size(), results.getFailures().size());
        }
    } catch(IOException e) {
        // assume all documents have failed
        for(Indexable indexable: documents) {
            D failed = indexable.document;
            results.addFailure(failed, e, ExceptionUtils.getRootCauseMessage(e));
        }
        LOG.error("Failed to submit bulk request; all documents failed", e);

    } finally {
        // flush all documents no matter which ones succeeded or failed
        documents.clear();
    }
    return results;
}
 

private BulkRequest buildBulkRequest(List<AnomalyModel> anomalyModels){
    BulkRequest bulkRequest = new BulkRequest();
    for (AnomalyModel anomalyModel : anomalyModels) {
        String json = VisualizerUtility.convertToJson(anomalyModel);
        bulkRequest.add(buildIndexRequest(json));
    }
    return bulkRequest;
}
 

private void callAddES(ConfigurationLogstash cl) throws IOException {
    BulkRequest bulk = new BulkRequest();
    ObjectMapper objectMapper = new ObjectMapper();
    String fluxJson = objectMapper.writeValueAsString(ConfEsSkalogs.builder().configurationLogstash(cl).pipeline(StringEscapeUtils.escapeJava(utilsConfig.generateConfig(cl))).build());
    bulk.add(new IndexRequest(INDEX_STORAGE)
            .type("doc")
            .id(cl.getIdConfiguration())
            .source(fluxJson, XContentType.JSON));
    BulkResponse bulkResponse = restHighLevelClient.bulk(bulk);
    if (bulkResponse.getItems().length == 1) {
        cl.setIdEs(bulkResponse.getItems()[0].getId());
    } else {
        log.error("Problem with return ES {}", bulkResponse);
    }
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    br.waitForActiveShards(ActiveShardCount.ONE);
    return br;
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    br.waitForActiveShards(ActiveShardCount.ONE);
    return br;
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    return br;
}
 

@Override
public boolean log(Collection<AuditEventBase> events) {
    boolean ret = false;
    try {
        logStatusIfRequired();
        addTotalCount(events.size());

        RestHighLevelClient client = getClient();
        if (null == client) {
            // ElasticSearch is still not initialized. So need return error
            addDeferredCount(events.size());
            return ret;
        }

        ArrayList<AuditEventBase> eventList = new ArrayList<>(events);
        BulkRequest bulkRequest = new BulkRequest();
        try {
            for (AuditEventBase event : eventList) {
                AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
                String id = authzEvent.getEventId();
                Map<String, Object> doc = toDoc(authzEvent);
                bulkRequest.add(new IndexRequest(index).id(id).source(doc).type(""));
            }
        } catch (Exception ex) {
            addFailedCount(eventList.size());
            logFailedEvent(eventList, ex);
        }
        BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT);
        if (response.status().getStatus() >= 400) {
            addFailedCount(eventList.size());
            logFailedEvent(eventList, "HTTP " + response.status().getStatus());
        } else {
            BulkItemResponse[] items = response.getItems();
            for (int i = 0; i < items.length; i++) {
                AuditEventBase itemRequest = eventList.get(i);
                BulkItemResponse itemResponse = items[i];
                if (itemResponse.isFailed()) {
                    addFailedCount(1);
                    logFailedEvent(Arrays.asList(itemRequest), itemResponse.getFailureMessage());
                } else {
                    if(LOG.isDebugEnabled()) {
                        LOG.debug(String.format("Indexed %s", itemRequest.getEventKey()));
                    }
                    addSuccessCount(1);
                    ret = true;
                }
            }
        }
    } catch (Throwable t) {
        addDeferredCount(events.size());
        logError("Error sending message to ElasticSearch", t);
    }
    return ret;
}
 

@Override
public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
    BulkRequest bulkRequest = Requests.bulkRequest();
    String defaultIndex = request.param("index");
    String defaultType = request.param("type");
    String defaultRouting = request.param("routing");
    String fieldsParam = request.param("fields");
    String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null;

    String consistencyLevel = request.param("consistency");
    if (consistencyLevel != null) {
        bulkRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
    }
    bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
    bulkRequest.refresh(request.paramAsBoolean("refresh", bulkRequest.refresh()));
    bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, allowExplicitIndex);

    client.bulk(bulkRequest, new RestBuilderListener<BulkResponse>(channel) {
        @Override
        public RestResponse buildResponse(BulkResponse response, XContentBuilder builder) throws Exception {
            builder.startObject();
            builder.field(Fields.TOOK, response.getTookInMillis());
            builder.field(Fields.ERRORS, response.hasFailures());
            builder.startArray(Fields.ITEMS);
            for (BulkItemResponse itemResponse : response) {
                builder.startObject();
                builder.startObject(itemResponse.getOpType());
                builder.field(Fields._INDEX, itemResponse.getIndex());
                builder.field(Fields._TYPE, itemResponse.getType());
                builder.field(Fields._ID, itemResponse.getId());
                long version = itemResponse.getVersion();
                if (version != -1) {
                    builder.field(Fields._VERSION, itemResponse.getVersion());
                }
                if (itemResponse.isFailed()) {
                    builder.field(Fields.STATUS, itemResponse.getFailure().getStatus().getStatus());
                    builder.startObject(Fields.ERROR);
                    ElasticsearchException.toXContent(builder, request, itemResponse.getFailure().getCause());
                    builder.endObject();
                } else {
                    ActionWriteResponse.ShardInfo shardInfo = itemResponse.getResponse().getShardInfo();
                    shardInfo.toXContent(builder, request);
                    if (itemResponse.getResponse() instanceof DeleteResponse) {
                        DeleteResponse deleteResponse = itemResponse.getResponse();
                        if (deleteResponse.isFound()) {
                            builder.field(Fields.STATUS, shardInfo.status().getStatus());
                        } else {
                            builder.field(Fields.STATUS, RestStatus.NOT_FOUND.getStatus());
                        }
                        builder.field(Fields.FOUND, deleteResponse.isFound());
                    } else if (itemResponse.getResponse() instanceof IndexResponse) {
                        IndexResponse indexResponse = itemResponse.getResponse();
                        if (indexResponse.isCreated()) {
                            builder.field(Fields.STATUS, RestStatus.CREATED.getStatus());
                        } else {
                            builder.field(Fields.STATUS, shardInfo.status().getStatus());
                        }
                    } else if (itemResponse.getResponse() instanceof UpdateResponse) {
                        UpdateResponse updateResponse = itemResponse.getResponse();
                        if (updateResponse.isCreated()) {
                            builder.field(Fields.STATUS, RestStatus.CREATED.getStatus());
                        } else {
                            builder.field(Fields.STATUS, shardInfo.status().getStatus());
                        }
                        if (updateResponse.getGetResult() != null) {
                            builder.startObject(Fields.GET);
                            updateResponse.getGetResult().toXContentEmbedded(builder, request);
                            builder.endObject();
                        }
                    }
                }
                builder.endObject();
                builder.endObject();
            }
            builder.endArray();

            builder.endObject();
            return new BytesRestResponse(OK, builder);
        }
    });
}
 

/**
 * This method will do the bulk data insertion.
 *
 * @param index String index name
 * @param type String type name
 * @param dataList List<Map<String, Object>>
 * @return boolean
 */
@Override
public Future<Boolean> bulkInsert(String index, List<Map<String, Object>> dataList) {
  long startTime = System.currentTimeMillis();
  ProjectLogger.log(
      "ElasticSearchRestHighImpl:bulkInsert: method started at =="
          + startTime
          + " for Index "
          + index,
      LoggerEnum.PERF_LOG.name());
  BulkRequest request = new BulkRequest();
  Promise<Boolean> promise = Futures.promise();
  for (Map<String, Object> data : dataList) {
    request.add(new IndexRequest(index, _DOC, (String) data.get(JsonKey.ID)).source(data));
  }
  ActionListener<BulkResponse> listener =
      new ActionListener<BulkResponse>() {
        @Override
        public void onResponse(BulkResponse bulkResponse) {
          Iterator<BulkItemResponse> responseItr = bulkResponse.iterator();
          if (responseItr != null) {
            promise.success(true);
            while (responseItr.hasNext()) {

              BulkItemResponse bResponse = responseItr.next();

              if (bResponse.isFailed()) {
                ProjectLogger.log(
                    "ElasticSearchRestHighImpl:bulkinsert: api response==="
                        + bResponse.getId()
                        + " "
                        + bResponse.getFailureMessage(),
                    LoggerEnum.INFO.name());
              }
            }
          }
        }

        @Override
        public void onFailure(Exception e) {
          ProjectLogger.log("ElasticSearchRestHighImpl:bulkinsert: Bulk upload error block", e);
          promise.success(false);
        }
      };
  ConnectionManager.getRestClient().bulkAsync(request, listener);

  ProjectLogger.log(
      "ElasticSearchRestHighImpl:bulkInsert: method end =="
          + " for Index "
          + index
          + " ,Total time elapsed = "
          + calculateEndTime(startTime),
      LoggerEnum.PERF_LOG.name());
  return promise.future();
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    return br;
}