org.elasticsearch.action.bulk.BulkRequest Java Examples

@Override
protected void populateData(TransportClient tc) {

    try {
        tc.admin().indices().create(new CreateIndexRequest("logs").mapping("_doc", FileHelper.loadFile("dlsfls/masked_field_mapping.json"), XContentType.JSON)).actionGet();


        byte[] data = FileHelper.loadFile("dlsfls/logs_bulk_data.json").getBytes(StandardCharsets.UTF_8);
        BulkRequest br = new BulkRequest().add(data, 0, data.length, XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE);
        if(tc.bulk(br).actionGet().hasFailures()) {
            Assert.fail("bulk import failed");
        }
        Thread.sleep(1000);

    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.toString());
    }

}
 

private void indexObject(final String index, final String docType, final String docId, final Object doc) {

        byte[] docBytes;
        try {
            docBytes = objectMapper.writeValueAsBytes(doc);
        } catch (JsonProcessingException e) {
            logger.error("Failed to convert {} '{}' to byte string", docType, docId);
            return;
        }

        IndexRequest request = new IndexRequest(index, docType, docId);
        request.source(docBytes, XContentType.JSON);

        if(bulkRequests.get(docType) == null) {
            bulkRequests.put(docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest()));
        }

        bulkRequests.get(docType).getBulkRequest().add(request);
        if (bulkRequests.get(docType).getBulkRequest().numberOfActions() >= this.indexBatchSize) {
            indexBulkRequest(docType);
        }
    }
 

public static BulkResponse bulkImport(List<Sku> skuList) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();
    for (Sku sku : skuList) {
        Map<String, Object> skuMap = new HashMap<String, Object>();
        IndexRequest indexRequest = new IndexRequest("sku", "doc", sku.getSkuId());
        skuMap.put("name", sku.getName());
        skuMap.put("price", sku.getPrice());
        skuMap.put("image", sku.getImage());
        skuMap.put("createTime", DateUtils.dateFormat(sku.getCreateTime()));
        skuMap.put("categoryName", sku.getCategoryName());
        skuMap.put("brandName", sku.getBrandName());
        // "{'颜色': '红色', '版本': '8GB+128GB'}"
        Map spec = JSON.parseObject(sku.getSpec());
        skuMap.put("spec", spec);
        skuMap.put("commentNum", sku.getCommentNum());
        skuMap.put("saleNum", sku.getSaleNum());
        skuMap.put("spuId", sku.getSpuId());
        indexRequest.source(skuMap);
        bulkRequest.add(indexRequest);
    }
    return restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
}
 

@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
	LOG.error("Failed Elasticsearch bulk request: {}", failure.getMessage(), failure);

	try {
		for (ActionRequest action : request.requests()) {
			failureHandler.onFailure(action, failure, -1, failureRequestIndexer);
		}
	} catch (Throwable t) {
		// fail the sink and skip the rest of the items
		// if the failure handler decides to throw an exception
		failureThrowable.compareAndSet(null, t);
	}

	if (flushOnCheckpoint) {
		numPendingRequests.getAndAdd(-request.numberOfActions());
	}
}
 

HttpBulkProcessor(Client client, Listener listener, @Nullable String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, @Nullable TimeValue flushInterval) {
    this.client = client;
    this.listener = listener;
    this.concurrentRequests = concurrentRequests;
    this.bulkActions = bulkActions;
    this.bulkSize = bulkSize.bytes();

    this.semaphore = new Semaphore(concurrentRequests);
    this.bulkRequest = new BulkRequest();

    if (flushInterval != null) {
        this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, EsExecutors.daemonThreadFactory(client.settings(), (name != null ? "[" + name + "]" : "") + "bulk_processor"));
        this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
        this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
        this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(new Flush(), flushInterval.millis(), flushInterval.millis(), TimeUnit.MILLISECONDS);
    } else {
        this.scheduler = null;
        this.scheduledFuture = null;
    }
}
 

protected Pair<BulkRequest, FutureCallbackHolder> prepareBatch(Batch<Object> batch, WriteCallback callback) {
  BulkRequest bulkRequest = new BulkRequest();
  final StringBuilder stringBuilder = new StringBuilder();
  for (Object record : batch.getRecords()) {
    try {
      byte[] serializedBytes = this.serializer.serializeToJson(record);
      log.debug("serialized record: {}", serializedBytes);
      IndexRequest indexRequest = new IndexRequest(this.indexName, this.indexType)
          .source(serializedBytes, 0, serializedBytes.length, XContentType.JSON);
      if (this.idMappingEnabled) {
        String id = this.typeMapper.getValue(this.idFieldName, record);
        indexRequest.id(id);
        stringBuilder.append(";").append(id);
      }
      bulkRequest.add(indexRequest);
    }
    catch (Exception e) {
      log.error("Encountered exception {}", e);
    }
  }
  FutureCallbackHolder futureCallbackHolder = new FutureCallbackHolder(callback,
      exception -> log.error("Batch: {} failed on ids; {} with exception {}", batch.getId(),
          stringBuilder.toString(), exception),
      this.malformedDocPolicy);
  return new Pair(bulkRequest, futureCallbackHolder);
}
 

@Test
public void configReturnsACopyOfServerUrisList() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    builder.withServerUris("http://localhost:9200;http://localhost:9201;http://localhost:9202");
    ClientObjectFactory<TransportClient, BulkRequest> config = builder.build();

    // when
    Collection<String> serverUrisList = config.getServerList();
    serverUrisList.add("test");

    // then
    assertNotEquals(serverUrisList.size(), config.getServerList().size());

}
 

public void bulkDocument(String type, Map<String, Map<String, Object>> sources) {
    try {
        if (this.instance() == null) return;
        BulkRequest requests = new BulkRequest();
        Iterator<String> it = sources.keySet().iterator();
        int count = 0;
        while (it.hasNext()) {
            count++;
            String next = it.next();
            IndexRequest request = new IndexRequest(name, type, next);
            request.source(sources.get(next));
            requests.add(request);
            if (count % 1000 == 0) {
                client.bulk(requests, RequestOptions.DEFAULT);
                requests.requests().clear();
                count = 0;
            }
        }
        if (requests.numberOfActions() > 0) client.bulk(requests, RequestOptions.DEFAULT);
    } catch (IOException e) {
        log.error(e.getMessage());
    }
}
 

public Mono<BulkResponse> update(List<UpdatedRepresentation> updatedDocumentParts, RoutingKey routingKey) {
    Preconditions.checkNotNull(updatedDocumentParts);
    Preconditions.checkNotNull(routingKey);
    BulkRequest request = new BulkRequest();
    updatedDocumentParts.forEach(updatedDocumentPart -> request.add(
        new UpdateRequest(aliasName.getValue(),
            NodeMappingFactory.DEFAULT_MAPPING_NAME,
            updatedDocumentPart.getId().asString())
            .doc(updatedDocumentPart.getUpdatedDocumentPart(), XContentType.JSON)
            .routing(routingKey.asString())));

    return client.bulk(request, RequestOptions.DEFAULT)
        .onErrorResume(ValidationException.class, exception -> {
            LOGGER.warn("Error while updating index", exception);
            return Mono.empty();
        });
}
 

private void indexObject(final String index, final String docType, final String docId, final Object doc) {

        byte[] docBytes;
        try {
            docBytes = objectMapper.writeValueAsBytes(doc);
        } catch (JsonProcessingException e) {
            logger.error("Failed to convert {} '{}' to byte string", docType, docId);
            return;
        }

        IndexRequest request = new IndexRequest(index, docType, docId);
        request.source(docBytes, XContentType.JSON);

        if (bulkRequests.get(docType) == null) {
            bulkRequests.put(docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest()));
        }

        bulkRequests.get(docType).getBulkRequest().add(request);
        if (bulkRequests.get(docType).getBulkRequest().numberOfActions() >= this.indexBatchSize) {
            indexBulkRequest(docType);
        }
    }
 

@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    logger.debug("afterBulk {} failures:{}", executionId,
            response.hasFailures());
    if (response.hasFailures()) {
        long succeeded = 0;
        for (Iterator<BulkItemResponse> i = response.iterator(); i
                .hasNext(); ) {
            if (!i.next().isFailed()) {
                succeeded++;
            }
        }
        if (succeeded > 0) {
            succeededDocs.addAndGet(succeeded);
        }
    } else {
        succeededDocs.addAndGet(request.numberOfActions());
    }
    bulkProcessed();
}
 

@Test
public void testScenarioAsBulkRequest() throws IOException {
    client.bulk(new BulkRequest()
        .add(new IndexRequest(INDEX, DOC_TYPE, "2").source(
            jsonBuilder()
                .startObject()
                .field(FOO, BAR)
                .endObject()
        ))
        .add(new IndexRequest(INDEX, DOC_TYPE, "3").source(
            jsonBuilder()
                .startObject()
                .field(FOO, BAR)
                .endObject()
        ))
    , RequestOptions.DEFAULT);
}
 

@Test
public void testRun() throws IOException {
    AnomalyModel anomalyModel = AnomalyModel.newBuilder()
            .key("key")
            .value(100)
            .level("NORMAL")
            .uuid("test")
            .timestamp("date")
            .anomalyThresholds(null)
            .tags(null)
            .build();
    anomalyModel = AnomalyModel.newBuilder(anomalyModel).build();
    List<AnomalyModel> anomalyModels = new ArrayList<>();
    anomalyModels.add(anomalyModel);
    BulkResponse bulkResponse = buildBulkResponseHappy();

    when(client.bulk(any(BulkRequest.class), any(RequestOptions.class))).thenReturn(bulkResponse);
    when(client.close()).thenReturn(true);
    ElasticSearchBulkService elasticSearchBulkService = new ElasticSearchBulkService(anomalyModels);
    elasticSearchBulkService.setElasticSearchClient(client);
    elasticSearchBulkService.run();
    verify(elasticSearchBulkService.getElasticSearchClient(), times(1))
            .bulk(any(BulkRequest.class), any(RequestOptions.class));

}
 

private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
    if(list.isEmpty()){
        return;
    }
    BulkRequest request = new BulkRequest();
    for(Map<String, Object> data : list) {
        String id = data.get("id").toString();
        request.add(
                new IndexRequest(index, type, id)
                        .source(data));

    }
    BulkResponse bulkResponse = CLIENT.bulk(request);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                LOGGER.error("ES索引失败: {}", failure.getMessage());
            }
        }
    }
}
 

@Test
public void failureHandlerExecutesFailoverForEachBatchItemSeparately() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<TransportClient, BulkRequest> config = builder.build();

    FailoverPolicy failoverPolicy = spy(new NoopFailoverPolicy());

    String payload1 = "test1";
    String payload2 = "test2";
    BulkRequest bulk = new BulkRequest()
            .add(spy(new IndexRequest().source(payload1, XContentType.CBOR)))
            .add(spy(new IndexRequest().source(payload2, XContentType.CBOR)));

    // when
    config.createFailureHandler(failoverPolicy).apply(bulk);

    // then
    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
    verify(failoverPolicy, times(2)).deliver(captor.capture());

    assertTrue(captor.getAllValues().contains(payload1));
    assertTrue(captor.getAllValues().contains(payload2));
}
 

@Override
public void delete(final long snifferId, final String[] eventIds) {
	clientTpl.executeWithClient(new ClientCallback<Object>() {
		@Override
		public Object execute(final Client client) {
			final BulkRequest deletes = new BulkRequest().refresh(true);
			for (final String id : eventIds) {
				for (final String index : indexNamingStrategy.getRetrievalNames(snifferId)) {
					deletes.add(new DeleteRequest(index, getType(snifferId), id));
				}
			}
			client.bulk(deletes).actionGet();
			logger.info("Deleted events: {}", (Object[]) eventIds);
			return null;
		}
	});
}
 

private BulkProcessor build(final RestHighLevelClient client) {
    logger.trace("Bulk processor name: [{}]  bulkActions: [{}], bulkSize: [{}], flush interval time: [{}]," +
                    " concurrent Request: [{}], backoffPolicyTimeInterval: [{}], backoffPolicyRetries: [{}] ",
            new Object[]{bulkProcessorName, bulkActions, bulkSize, flushIntervalTime,
                    concurrentRequest, backoffPolicyTimeInterval, backoffPolicyRetries});
    BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer =
            (request, bulkListener) -> client
                    .bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
    return BulkProcessor.builder(bulkConsumer, getListener())
            .setBulkActions(bulkActions)
            .setBulkSize(bulkSize)
            .setFlushInterval(flushIntervalTime)
            .setConcurrentRequests(concurrentRequest)
            .setBackoffPolicy(BackoffPolicy.exponentialBackoff(
                    Util.getTimeValue(backoffPolicyTimeInterval,
                            DEFAULT_ES_BACKOFF_POLICY_START_DELAY),
                    backoffPolicyRetries))
            .build();
}
 

@Override
public <T extends Entity> boolean bulkUpdate(String indexName, List<? extends Entity> entities) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();
    entities.stream().map(e -> createUpdateRequest(indexName, getType(e), e.getId(), getJson(e), getParent(e), getRoot(e))).
            forEach(bulkRequest::add);
    bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);

    BulkResponse bulkResponse = client.bulk(bulkRequest);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse resp : bulkResponse.getItems()) {
            if (resp.isFailed()) {
                LOGGER.error("bulk update failed : {}", resp.getFailureMessage());
            }
        }
        return false;
    }
    return true;
}
 

@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
    if (response.hasFailures()) {
        LOGGER.error("Bulk[{}] executed with failures", executionId);
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                LOGGER.error("Failed on {} due to {}", item.getId(), item.getFailureMessage());
                numberOfEventInError.inc();
            }
        }
    } else {
        LOGGER.info("Successfully completed Bulk[{}] in {} ms", executionId, response.getTook().getMillis());
        latencyIndexingEvents.observe(response.getTook().getMillis());
    }
    CommittableOffset<String, byte[]> lastOffset = (CommittableOffset<String, byte[]>) request.payloads().get(request.payloads().size() - 1);
    lastOffset
            .commitAsync()
            .whenComplete((topicPartitionOffset, exception) -> {
                if (exception != null) {
                    LOGGER.warn("Could not commit kafka offset {}|{}", lastOffset.getOffset(), lastOffset.getPartition());
                    numberOfOffsetCommitError.inc();
                } else {
                    LOGGER.info("Committed kafka offset {}|{}", topicPartitionOffset.getOffset(), topicPartitionOffset.getPartition());
                }
            });
}
 

@SuppressWarnings("rawtypes")
private BulkResponse writeAcl(ACLDocumentOperation operation, Collection<SearchGuardACLDocument> docs) throws Exception {
    BulkRequestBuilder builder = client.getClient().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    BulkRequest request = operation.buildRequest(this.client.getClient(), builder, docs);
    client.addCommonHeaders();
    return this.client.getClient().bulk(request).actionGet();
}
 

private synchronized void indexBulkRequest(String docType) {
    if (bulkRequests.get(docType).getBulkRequest() != null && bulkRequests.get(docType).getBulkRequest().numberOfActions() > 0) {
        synchronized (bulkRequests.get(docType).getBulkRequest()) {
            indexWithRetry(bulkRequests.get(docType).getBulkRequest().get(), "Bulk Indexing " + docType, docType);
            bulkRequests.put(docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest()));
        }
    }
}
 

@Override
public Function<BulkRequest, Boolean> createFailureHandler(FailoverPolicy failover) {
    return new Function<BulkRequest, Boolean>() {

        private final BulkRequestIntrospector introspector = new BulkRequestIntrospector();

        @Override
        public Boolean apply(BulkRequest bulk) {
            introspector.items(bulk).forEach(failedItem -> failover.deliver(failedItem));
            return true;
        }

    };
}
 

@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
    if (response.hasFailures()) {
        failureHandler.apply(request);
    }

}
 

public void bulkExecute(List<Event> events) throws Exception {
    //批量插入数据
    BulkRequest request = new BulkRequest();
    String indexName = null;
    for (Event event : events) {
        //如果没有切换天，那么索引可以服用，无需重复创建
        if (StringUtils.isEmpty(indexName) || !indexName.endsWith(indexNameBuilder.getIndexSuffix(event))) {
            indexName = indexNameBuilder.getIndexName(event);
        }
        request.add(new IndexRequest(indexName, indexType).source(eventSerializer.serializer(event), XContentType.JSON));
    }
    BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
    TimeValue took = bulkResponse.getTook();
    logger.debug("[批量新增花费的毫秒]:" + took + "," + took.getMillis() + "," + took.getSeconds() + ",events[" + events.size() + "]");
}
 

public Mono<BulkResponse> delete(List<DocumentId> ids, RoutingKey routingKey) {
    BulkRequest request = new BulkRequest();
    ids.forEach(id -> request.add(
        new DeleteRequest(aliasName.getValue())
            .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
            .id(id.asString())
            .routing(routingKey.asString())));

    return client.bulk(request, RequestOptions.DEFAULT)
        .onErrorResume(ValidationException.class, exception -> {
            LOGGER.warn("Error while deleting index", exception);
            return Mono.empty();
        });
}
 

@Override
public void afterBulk(long executionId, BulkRequest request,
        Throwable failure) {
    logger.debug("afterBulk failed {} {}", executionId, failure);

    bulkProcessed();
}
 

@Test
public void failoverIsExecutedAfterNonSuccessfulRequest() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<TransportClient, BulkRequest> config = spy(builder.build());

    FailoverPolicy failoverPolicy = spy(new NoopFailoverPolicy());
    Function handler = spy(config.createFailureHandler(failoverPolicy));
    when(config.createFailureHandler(any())).thenReturn(handler);

    Settings settings = Settings.builder().build();
    TransportClient client = spy(new PreBuiltTransportClient(settings));
    client.addTransportAddress(new TransportAddress(new InetSocketAddress(9999)));
    when(config.createClient()).thenReturn(client);

    BulkProcessorFactory bulkProcessorFactory = new BulkProcessorFactory();
    BatchEmitter batchEmitter = bulkProcessorFactory.createInstance(
            1,
            100,
            config,
            failoverPolicy);

    String payload1 = "test1";
    ActionRequest testRequest = createTestRequest(payload1);

    // when
    batchEmitter.add(testRequest);

    // then
    ArgumentCaptor<BulkRequest> captor = ArgumentCaptor.forClass(BulkRequest.class);
    verify(handler, times(1)).apply(captor.capture());

    assertEquals(payload1, new BulkRequestIntrospector().items(captor.getValue()).iterator().next());
}
 

@Test
public void clientIsCalledWhenBatchItemIsAdded() {

    // given
    Builder builder = createTestObjectFactoryBuilder();
    ClientObjectFactory<TransportClient, BulkRequest> config = spy(builder.build());

    Settings settings = Settings.builder()
            .put("node.local", true)
            .build();

    TransportClient client = spy(TransportClient.builder().settings(settings).build());
    client.addTransportAddress(new LocalTransportAddress("1"));
    when(config.createClient()).thenReturn(client);

    FailoverPolicy failoverPolicy = spy(new NoopFailoverPolicy());

    BulkProcessorFactory bulkProcessorFactory = new BulkProcessorFactory();
    BatchEmitter batchEmitter = bulkProcessorFactory.createInstance(
                    1,
                    100,
                    config,
                    failoverPolicy);

    String payload1 = "test1";
    ActionRequest testRequest = createTestRequest(payload1);

    // when
    batchEmitter.add(testRequest);

    // then
    ArgumentCaptor<BulkRequest> captor = ArgumentCaptor.forClass(BulkRequest.class);
    verify(client, times(1)).bulk(captor.capture(), Mockito.any());

    assertEquals(payload1, new BulkRequestIntrospector().items(captor.getValue()).iterator().next());
}
 

@Override
public BulkRequest buildRequest(Client client, BulkRequestBuilder builder, Collection<SearchGuardACLDocument> docs) throws IOException{
    for (SearchGuardACLDocument doc : docs) {
        logContent("Expired doc {} to be: {}", doc.getType(), doc);
        Map<String, Object> content = new HashMap<>();
        content.put(doc.getType(), new BytesArray(XContentHelper.toString(doc)));
        IndexRequestBuilder indexBuilder = client
                .prepareIndex(searchGuardIndex, doc.getType(), SEARCHGUARD_CONFIG_ID)
                .setOpType(OpType.INDEX)
                .setVersion(doc.getVersion())
                .setSource(content);
        builder.add(indexBuilder.request());
    }
    return builder.request();
}
 

private void execute() {
    final BulkRequest bulkRequest = this.bulkRequest;
    final long executionId = executionIdGen.incrementAndGet();

    this.bulkRequest = new BulkRequest();
    this.bulkRequestHandler.execute(bulkRequest, executionId);
}