org.elasticsearch.action.bulk.BulkResponse#buildFailureMessage

Source File: Elasticsearch2Client.java From presto-connectors with Apache License 2.0

6 votes

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.get();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}

Source File: Elasticsearch6Client.java From presto-connectors with Apache License 2.0

6 votes

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.execute().actionGet();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}

Source File: Elasticsearch5Client.java From presto-connectors with Apache License 2.0

6 votes

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.execute().actionGet();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}

Source File: TransportClientUtil.java From bboss-elasticsearch with Apache License 2.0

6 votes

public Object execute(String options) throws ElasticSearchException {
  try {
  	if(options != null) {
  		if(options.indexOf("refresh=true") > 0) {
		bulkRequestBuilder.setRefreshPolicy("true");
	}
	else if(options.indexOf("refresh=wait_for") > 0){
		bulkRequestBuilder.setRefreshPolicy("wait_for");
	}
	else if(options.indexOf("refresh=false") > 0){
		bulkRequestBuilder.setRefreshPolicy("false");
	}
	else if(options.indexOf("refresh") > 0){
		bulkRequestBuilder.setRefreshPolicy("true");
	}
}
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
      throw new EventDeliveryException(bulkResponse.buildFailureMessage());
    }
    return bulkResponse;
  } finally {
    
  }
}

Source File: ESBasedIndexer.java From samantha with MIT License

6 votes

private void bulkIndex(String indexType, JsonNode data) {
    if (data.size() == 0) {
        return;
    }
    if (uniqueFields.size() > 0) {
        Set<String> keys = new HashSet<>();
        ArrayNode uniqued = Json.newArray();
        for (JsonNode point : data) {
            String key = FeatureExtractorUtilities.composeConcatenatedKey(point, dataFields);
            if (!keys.contains(key)) {
                keys.add(key);
                uniqued.add(point);
            }
        }
        data = uniqued;
    }
    BulkResponse resp = elasticSearchService.bulkIndex(elasticSearchIndex, indexType, data);
    if (resp.hasFailures()) {
        throw new BadRequestException(resp.buildFailureMessage());
    }
}

Source File: ESBasedIndexer.java From samantha with MIT License

6 votes

private void bulkDelete(String indexType, JsonNode data) {
    SearchHits hits = elasticSearchService
            .searchHitsByKeys(elasticSearchIndex, indexType, uniqueFields, uniqueFields, data);
    List<String> ids = new ArrayList<>();
    for (SearchHit hit : hits.getHits()) {
        if (hit != null) {
            ids.add(hit.getId());
        }
    }
    if (ids.size() == 0) {
        return;
    }
    BulkResponse resp = elasticSearchService.bulkDelete(elasticSearchIndex, indexType, ids);
    if (resp.hasFailures()) {
        throw new BadRequestException(resp.buildFailureMessage());
    }
}

Source File: SearchHelper.java From fess with Apache License 2.0

5 votes

public boolean bulkUpdate(final Consumer<BulkRequestBuilder> consumer) {
    final BulkRequestBuilder builder = ComponentUtil.getFessEsClient().prepareBulk();
    consumer.accept(builder);
    try {
        final BulkResponse response = builder.execute().get();
        if (response.hasFailures()) {
            throw new FessEsClientException(response.buildFailureMessage());
        } else {
            return true;
        }
    } catch (InterruptedException | ExecutionException e) {
        throw new FessEsClientException("Failed to update bulk data.", e);
    }
}

Source File: BulkSave.java From code with Apache License 2.0

5 votes

public static void main(String[] args) throws IOException {
    //1.连接rest接口
    HttpHost http = new HttpHost("127.0.0.1", 9200, "http");
    RestClientBuilder builder = RestClient.builder(http);//rest构建器
    RestHighLevelClient restHighLevelClient = new
            RestHighLevelClient(builder);//高级客户端对象
    //2.封装请求对象
    BulkRequest bulkRequest = new BulkRequest();
    IndexRequest indexRequest = new IndexRequest("sku", "doc", "4");
    Map skuMap = new HashMap();
    skuMap.put("name", "华为p30pro 火爆上市");
    skuMap.put("brandName", "华为");
    skuMap.put("categoryName", "手机");
    skuMap.put("price", 1010221);
    skuMap.put("createTime", "2019‐05‐01");
    skuMap.put("saleNum", 101021);
    skuMap.put("commentNum", 10102321);
    Map spec = new HashMap();
    spec.put("网络制式", "移动4G");
    spec.put("屏幕尺寸", "5");
    skuMap.put("spec", spec);
    indexRequest.source(skuMap);
    bulkRequest.add(indexRequest);//可以多次添加
    //3.获取响应结果
    BulkResponse response =
            restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
    int status = response.status().getStatus();
    System.out.println(status);
    String message = response.buildFailureMessage();
    System.out.println(message);
    restHighLevelClient.close();
}

Source File: ElasticSearchTransportClient.java From ingestion with Apache License 2.0

5 votes

@Override
public void execute() throws Exception {
  try {
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
      throw new EventDeliveryException(bulkResponse.buildFailureMessage());
    }
  } finally {
    bulkRequestBuilder = client.prepareBulk();
  }
}

Source File: FessEsClient.java From fess with Apache License 2.0

5 votes

public void addAll(final String index, final List<Map<String, Object>> docList,
        final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (final Map<String, Object> doc : docList) {
        final Object id = doc.remove(fessConfig.getIndexFieldId());
        final IndexRequestBuilder builder = client.prepareIndex().setIndex(index).setId(id.toString()).setSource(new DocMap(doc));
        options.accept(doc, builder);
        bulkRequestBuilder.add(builder);
    }
    final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
    if (response.hasFailures()) {
        if (logger.isDebugEnabled()) {
            final List<DocWriteRequest<?>> requests = bulkRequestBuilder.request().requests();
            final BulkItemResponse[] items = response.getItems();
            if (requests.size() == items.length) {
                for (int i = 0; i < requests.size(); i++) {
                    final BulkItemResponse resp = items[i];
                    if (resp.isFailed() && resp.getFailure() != null) {
                        final DocWriteRequest<?> req = requests.get(i);
                        final Failure failure = resp.getFailure();
                        logger.debug("Failed Request: {}\n=>{}", req, failure.getMessage());
                    }
                }
            }
        }
        throw new FessEsClientException(response.buildFailureMessage());
    }
}

Source File: EsAbstractBehavior.java From fess with Apache License 2.0

5 votes

@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}

Source File: ElasticSearchTypeImpl.java From core-ng-project with Apache License 2.0

5 votes

@Override
public void bulkIndex(BulkIndexRequest<T> request) {
    var watch = new StopWatch();
    if (request.sources == null || request.sources.isEmpty()) throw new Error("request.sources must not be empty");
    String index = request.index == null ? this.index : request.index;
    var bulkRequest = new BulkRequest();
    for (Map.Entry<String, T> entry : request.sources.entrySet()) {
        String id = entry.getKey();
        T source = entry.getValue();
        validator.validate(source, false);
        var indexRequest = new org.elasticsearch.action.index.IndexRequest(index).id(id).source(mapper.toJSON(source), XContentType.JSON);
        bulkRequest.add(indexRequest);
    }
    long esTook = 0;
    try {
        BulkResponse response = elasticSearch.client().bulk(bulkRequest, RequestOptions.DEFAULT);
        esTook = response.getTook().nanos();
        if (response.hasFailures()) throw new SearchException(response.buildFailureMessage());
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        long elapsed = watch.elapsed();
        ActionLogContext.track("elasticsearch", elapsed, 0, request.sources.size());
        logger.debug("bulkIndex, index={}, size={}, esTook={}, elapsed={}", index, request.sources.size(), esTook, elapsed);
        checkSlowOperation(elapsed);
    }
}

Source File: EsAbstractBehavior.java From fess with Apache License 2.0

5 votes

@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}

Source File: TraceServiceElasticsearch.java From hawkular-apm with Apache License 2.0

5 votes

@Override
public void storeFragments(String tenantId, List<Trace> traces)
        throws StoreException {
    client.initTenant(tenantId);

    BulkRequestBuilder bulkRequestBuilder = client.getClient().prepareBulk();

    try {
        for (int i = 0; i < traces.size(); i++) {
            Trace trace = traces.get(i);
            String json = mapper.writeValueAsString(trace);

            if (msgLog.isTraceEnabled()) {
                msgLog.tracef("Storing trace: %s", json);
            }

            bulkRequestBuilder.add(client.getClient().prepareIndex(client.getIndex(tenantId),
                    TRACE_TYPE, trace.getFragmentId()).setSource(json));
        }
    } catch (JsonProcessingException e) {
        throw new StoreException(e);
    }

    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();

    if (bulkItemResponses.hasFailures()) {

        if (msgLog.isTraceEnabled()) {
            msgLog.trace("Failed to store traces to elasticsearch: "
                    + bulkItemResponses.buildFailureMessage());
        }

        throw new StoreException(bulkItemResponses.buildFailureMessage());

    } else {
        if (msgLog.isTraceEnabled()) {
            msgLog.trace("Success storing traces to elasticsearch");
        }
    }
}

Source File: AnalyticsServiceElasticsearch.java From hawkular-apm with Apache License 2.0

5 votes

private void bulkStoreApmEvents(String tenantId, List<? extends ApmEvent> events, String type) throws StoreException {
    client.initTenant(tenantId);

    BulkRequestBuilder bulkRequestBuilder = client.getClient().prepareBulk();

    for (ApmEvent event : events) {
        String json = toJson(event);
        if (null == json) {
            continue;
        }

        if (msgLog.isTraceEnabled()) {
            msgLog.tracef("Storing event: %s", json);
        }

        bulkRequestBuilder.add(toIndexRequestBuilder(client, tenantId, type, event.getId(), json));
    }

    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();

    if (bulkItemResponses.hasFailures()) {
        if (msgLog.isTraceEnabled()) {
            msgLog.trace("Failed to store event to elasticsearch: " + bulkItemResponses.buildFailureMessage());
        }
        throw new StoreException(bulkItemResponses.buildFailureMessage());
    } else {
        if (msgLog.isTraceEnabled()) {
            msgLog.trace("Success storing event to elasticsearch");
        }
    }
}

Source File: SpanServiceElasticsearch.java From hawkular-apm with Apache License 2.0

5 votes

@Override
public void storeSpan(String tenantId, List<Span> spans, Function<Span, String> spanIdSupplier)
        throws StoreException {

    client.initTenant(tenantId);

    BulkRequestBuilder bulkRequestBuilder = client.getClient().prepareBulk();

    for (Span span : spans) {
        String json;
        try {
            json = serialize(span);
        } catch(IOException ex){
            log.errorf("Failed to serialize span %s", span);
            throw new StoreException(ex);
        }

        log.tracef("Storing span: %s", json);
        // modified id is used in index
        final String modifiedId = spanIdSupplier.apply(span);

        bulkRequestBuilder.add(client.getClient()
                .prepareIndex(client.getIndex(tenantId), SPAN_TYPE, modifiedId)
                .setSource(json));
    }

    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();

    if (bulkItemResponses.hasFailures()) {
        log.tracef("Failed to store spans to elasticsearch: %s", bulkItemResponses.buildFailureMessage());
        throw new StoreException(bulkItemResponses.buildFailureMessage());
    }

    log.trace("Success storing spans to elasticsearch");
}

Source File: ElasticsearchBulkUpdater.java From rdf4j with BSD 3-Clause "New" or "Revised" License

5 votes

@Override
public void end() throws IOException {
	if (bulkRequest.numberOfActions() > 0) {
		BulkResponse response = bulkRequest.execute().actionGet();
		if (response.hasFailures()) {
			throw new IOException(response.buildFailureMessage());
		}
	}
}

Source File: FessEsClient.java From fess with Apache License 2.0

4 votes

public long updateByQuery(final String index, final Function<SearchRequestBuilder, SearchRequestBuilder> option,
        final BiFunction<UpdateRequestBuilder, SearchHit, UpdateRequestBuilder> builder) {

    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    SearchResponse response =
            option.apply(
                    client.prepareSearch(index).setScroll(scrollForUpdate).setSize(sizeForUpdate)
                            .setPreference(Constants.SEARCH_PREFERENCE_LOCAL)).execute()
                    .actionGet(fessConfig.getIndexScrollSearchTimeout());

    int count = 0;
    String scrollId = response.getScrollId();
    try {
        while (scrollId != null) {
            final SearchHits searchHits = response.getHits();
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                final UpdateRequestBuilder requestBuilder =
                        builder.apply(client.prepareUpdate().setIndex(index).setId(hit.getId()), hit);
                if (requestBuilder != null) {
                    bulkRequest.add(requestBuilder);
                }
                count++;
            }
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(fessConfig.getIndexBulkTimeout());
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response =
                    client.prepareSearchScroll(scrollId).setScroll(scrollForUpdate).execute()
                            .actionGet(fessConfig.getIndexBulkTimeout());
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
            scrollId = response.getScrollId();
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}

Source File: ElasticSearchSink.java From mt-flume with Apache License 2.0

4 votes

@Override
public Status process() throws EventDeliveryException {
  logger.debug("processing...");
  Status status = Status.READY;
  Channel channel = getChannel();
  Transaction txn = channel.getTransaction();
  try {
    txn.begin();
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (int i = 0; i < batchSize; i++) {
      Event event = channel.take();

      if (event == null) {
        break;
      }

      IndexRequestBuilder indexRequest =
          indexRequestFactory.createIndexRequest(
              client, indexName, indexType, event);

      if (ttlMs > 0) {
        indexRequest.setTTL(ttlMs);
      }

      bulkRequest.add(indexRequest);
    }

    int size = bulkRequest.numberOfActions();
    if (size <= 0) {
      sinkCounter.incrementBatchEmptyCount();
      counterGroup.incrementAndGet("channel.underflow");
      status = Status.BACKOFF;
    } else {
      if (size < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
        status = Status.BACKOFF;
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }

      sinkCounter.addToEventDrainAttemptCount(size);

      BulkResponse bulkResponse = bulkRequest.execute().actionGet();
      if (bulkResponse.hasFailures()) {
        throw new EventDeliveryException(bulkResponse.buildFailureMessage());
      }
    }
    txn.commit();
    sinkCounter.addToEventDrainSuccessCount(size);
    counterGroup.incrementAndGet("transaction.success");
  } catch (Throwable ex) {
    try {
      txn.rollback();
      counterGroup.incrementAndGet("transaction.rollback");
    } catch (Exception ex2) {
      logger.error(
          "Exception in rollback. Rollback might not have been successful.",
          ex2);
    }

    if (ex instanceof Error || ex instanceof RuntimeException) {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      Throwables.propagate(ex);
    } else {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      throw new EventDeliveryException(
          "Failed to commit transaction. Transaction rolled back.", ex);
    }
  } finally {
    txn.close();
  }
  return status;
}

Source File: ElasticsearchDataStructure.java From rdf4j with BSD 3-Clause "New" or "Revised" License

4 votes

synchronized private void flushRemoveStatementBuffer() {

		if (deleteStatementBuffer.isEmpty()) {
			return;
		}

		BulkRequestBuilder bulkRequest = clientProvider.getClient().prepareBulk();

		int failures = 0;

		do {

			deleteStatementBuffer.forEach(statement -> {

				bulkRequest.add(clientProvider.getClient()
						.prepareDelete(index, ELASTICSEARCH_TYPE, statement.getElasticsearchId()));

			});

			BulkResponse bulkResponse = bulkRequest.get();
			if (bulkResponse.hasFailures()) {
				failures++;
				if (failures < 10) {
					logger.warn("Elasticsearch has failures when adding data, retrying. Message: {}",
							bulkResponse.buildFailureMessage());
				} else {
					throw new RuntimeException("Elasticsearch has failed " + failures
							+ " times when adding data, retrying. Message: " + bulkResponse.buildFailureMessage());
				}

			} else {
				failures = 0;
			}

		} while (failures > 0);

		logger.debug("Removed {} statements", deleteStatementBuffer.size());

		deleteStatementBuffer = Collections.synchronizedSet(new HashSet<>(BUFFER_THRESHOLD));

	}

Java Code Examples for org.elasticsearch.action.bulk.BulkResponse#buildFailureMessage()