org.elasticsearch.action.DocWriteRequest Java Examples

@Override
public void afterBulk(long executionId, BulkRequest request,
        Throwable failure) {
    eventCounter.scope("bulks_received").incrBy(1);
    LOG.error("Exception with bulk {} - failing the whole lot ",
            executionId, failure);
    synchronized (waitAck) {
        // WHOLE BULK FAILED
        // mark all the docs as fail
        Iterator<DocWriteRequest<?>> itreq = request.requests().iterator();
        while (itreq.hasNext()) {
            String id = itreq.next().id();
            Tuple t = waitAck.getIfPresent(id);
            if (t != null) {
                LOG.debug("Failed tuple for ID {}", id);
                // fail it
                _collector.fail(t);
                waitAck.invalidate(id);
            } else {
                LOG.warn("Could not find unacked tuple for {}", id);
            }
        }
    }
}
 

public boolean bulk(List<DocWriteRequest> doc, int retry) {
    if (CollectionUtils.isEmpty(doc)) {
        return true;
    }
    while (retry > 0) {
        try {
            this.restClient.bulk(createBulkRequest(doc), requestOptions);
            return true;
        } catch (Exception e) {
            //
        } finally {
            retry--;
        }
    }
    return false;
}
 

@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
	try {
		for (DocWriteRequest writeRequest : request.requests()) {
			if (writeRequest instanceof ActionRequest) {
				failureHandler.onFailure((ActionRequest) writeRequest, failure, -1, failureRequestIndexer);
			} else {
				throw new UnsupportedOperationException("The sink currently only supports ActionRequests");
			}
		}
	} catch (Throwable t) {
		// fail the sink and skip the rest of the items
		// if the failure handler decides to throw an exception
		failureThrowable.compareAndSet(null, t);
	}

	if (flushOnCheckpoint) {
		numPendingRequests.getAndAdd(-request.numberOfActions());
	}
}
 

private void retry(List<DocWriteRequest> retryList) {
    if (retryList.size() == 0) {
        return;
    }

    log.info("retry bulk {} docs", retryList.size());

    BulkRequestBuilder bulkRequest = esclient.prepareBulk();
    retryList.forEach(f -> {
                bulkRequest.add((IndexRequest) f);
            }
    );
    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        log.info("retry bulk request has failures");
    } else {
        log.info("retry bulk request done");
    }
}
 

private void populateIndexRequestBuilder(IndexRequestBuilder builder, IndexOptions options) {
    if (options != null) {
        if (options.getId() != null) builder.setId(options.getId());
        if (options.getRouting() != null) builder.setRouting(options.getRouting());
        if (options.getParent() != null) builder.setParent(options.getParent());
        if (options.getOpType() != null)
            builder.setOpType(DocWriteRequest.OpType.valueOf(options.getOpType().name()));
        if (options.getWaitForActiveShard() != null)
            builder.setWaitForActiveShards(options.getWaitForActiveShard());
        if (options.getRefreshPolicy() != null)
            builder.setRefreshPolicy(WriteRequest.RefreshPolicy.valueOf(options.getRefreshPolicy().name()));
        if (options.getVersion() != null) builder.setVersion(options.getVersion());
        if (options.getVersionType() != null) builder.setVersionType(options.getVersionType());
        if (options.getTimeout() != null) builder.setTimeout(options.getTimeout());
    }
}
 

public boolean bulk(List<DocWriteRequest> doc, int retry) {
    if (CollectionUtils.isEmpty(doc)) {
        return true;
    }
    while (retry > 0) {
        try {
            this.restClient.bulk(createBulkRequest(doc), requestOptions);
            return true;
        } catch (Exception e) {
            //
        } finally {
            retry--;
        }
    }
    return false;
}
 

private DocWriteRequest eventToRequest(EsConfig esConfig, EventType eventType, Map<String, String> values) {

        DocWriteRequest req = null;

        // column_name,column_name
        String id = esConfig.getEsIdFieldNames()
                .stream()
                .map(esId -> String.valueOf(values.get(esId)))
                .reduce((s1, s2) -> s1 + esConfig.getSeparator() + s2)
                .orElse("");

        if (StringUtils.isBlank(id)) {
            return null;
        }
        String type = esConfig.getType();
        String index = esConfig.getIndex();


        switch (eventType) {
            case INSERT:
            case UPDATE:
                UpdateRequest ur = new UpdateRequest(index, type, id);
                ur.doc(values);
                ur.docAsUpsert(true);
                req = ur;
                break;
            case DELETE:
                DeleteRequest dr = new DeleteRequest(index, type, id);
                dr.id(id);
                req = dr;
                break;
            default:
                break;
        }
        return req;
    }
 

@Override
public void afterBulk(long executionId, BulkRequest request,
        Throwable throwable) {
    eventCounter.scope("bulks_received").incrBy(1);
    LOG.error("Exception with bulk {} - failing the whole lot ",
            executionId, throwable);
    synchronized (waitAck) {
        // WHOLE BULK FAILED
        // mark all the docs as fail
        Iterator<DocWriteRequest<?>> itreq = request.requests().iterator();
        while (itreq.hasNext()) {
            DocWriteRequest bir = itreq.next();
            String id = bir.id();
            List<Tuple> xx = waitAck.getIfPresent(id);
            if (xx != null) {
                LOG.debug("Failed {} tuple(s) for ID {}", xx.size(), id);
                for (Tuple x : xx) {
                    // fail it
                    _collector.fail(x);
                }
                waitAck.invalidate(id);
            } else {
                LOG.warn("Could not find unacked tuple for {}", id);
            }
        }
    }
}
 

public void addAll(final String index, final List<Map<String, Object>> docList,
        final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (final Map<String, Object> doc : docList) {
        final Object id = doc.remove(fessConfig.getIndexFieldId());
        final IndexRequestBuilder builder = client.prepareIndex().setIndex(index).setId(id.toString()).setSource(new DocMap(doc));
        options.accept(doc, builder);
        bulkRequestBuilder.add(builder);
    }
    final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
    if (response.hasFailures()) {
        if (logger.isDebugEnabled()) {
            final List<DocWriteRequest<?>> requests = bulkRequestBuilder.request().requests();
            final BulkItemResponse[] items = response.getItems();
            if (requests.size() == items.length) {
                for (int i = 0; i < requests.size(); i++) {
                    final BulkItemResponse resp = items[i];
                    if (resp.isFailed() && resp.getFailure() != null) {
                        final DocWriteRequest<?> req = requests.get(i);
                        final Failure failure = resp.getFailure();
                        logger.debug("Failed Request: {}\n=>{}", req, failure.getMessage());
                    }
                }
            }
        }
        throw new FessEsClientException(response.buildFailureMessage());
    }
}
 

private DocWriteRequest toDocWriteRequest(DocumentAction documentAction) {
  String indexName = documentAction.getIndex().getName();
  String documentId = documentAction.getDocument().getId();

  DocWriteRequest docWriteRequest;
  switch (documentAction.getOperation()) {
    case INDEX:
      XContentBuilder source = documentAction.getDocument().getContent();
      if (source == null) {
        throw new IndexException(
            format("Document action is missing document source '%s'", documentAction));
      }
      docWriteRequest =
          Requests.indexRequest(indexName)
              .type(indexName)
              .id(documentId)
              .source(source)
              .opType(INDEX);
      break;
    case DELETE:
      docWriteRequest = Requests.deleteRequest(indexName).type(indexName).id(documentId);
      break;
    default:
      throw new UnexpectedEnumException(documentAction.getOperation());
  }
  return docWriteRequest;
}
 

public void processDocumentActions(Stream<DocumentAction> documentActions) {
  LOG.trace("Processing document actions ...");
  BulkProcessor bulkProcessor = bulkProcessorFactory.create(client);
  try {
    documentActions.forEachOrdered(
        documentAction -> {
          DocWriteRequest docWriteRequest = toDocWriteRequest(documentAction);
          bulkProcessor.add(docWriteRequest);
        });
  } finally {
    waitForCompletion(bulkProcessor);
    LOG.debug("Processed document actions.");
  }
}
 

@Override
public BulkDocumentWriterResults<D> write() {
    BulkDocumentWriterResults<D> results = new BulkDocumentWriterResults<>();
    try {
        // create an index request for each document
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.setRefreshPolicy(refreshPolicy);
        for(Indexable doc: documents) {
            DocWriteRequest request = createRequest(doc.document, doc.index);
            bulkRequest.add(request);
        }

        // submit the request and handle the response
        BulkResponse bulkResponse = client.getHighLevelClient().bulk(bulkRequest);
        handleBulkResponse(bulkResponse, documents, results);
        if (LOG.isDebugEnabled()) {
            String shards = Arrays.stream(bulkResponse.getItems())
                    .map(bulkItemResponse -> bulkItemResponse.getResponse().getShardId().toString())
                    .collect(Collectors.joining(","));
            LOG.debug("{} results written to shards {} in {} ms; batchSize={}, success={}, failed={}",
                    bulkResponse.getItems().length, shards, bulkResponse.getTookInMillis(),
                    documents.size(), results.getSuccesses().size(), results.getFailures().size());
        }
    } catch(IOException e) {
        // assume all documents have failed
        for(Indexable indexable: documents) {
            D failed = indexable.document;
            results.addFailure(failed, e, ExceptionUtils.getRootCauseMessage(e));
        }
        LOG.error("Failed to submit bulk request; all documents failed", e);

    } finally {
        // flush all documents no matter which ones succeeded or failed
        documents.clear();
    }
    return results;
}
 

protected void parseResultErrors(BulkRequest request, BulkResponse bulkItemResponses) {
    for (BulkItemResponse bir : bulkItemResponses) {
        DocWriteRequest docWriteRequest = request.requests().get(bir.getItemId());
        if (bir.isFailed()) {
            if (isRetryable(bir)) {
                routeToNextTopic(bir, toRawMessage(docWriteRequest), false);
            } else {
                routeToNextTopic(bir, toRawMessage(docWriteRequest), true);
            }
        }
    }
}
 

private String toRawMessage(DocWriteRequest docWriteRequest) {
    if (docWriteRequest.opType() == DocWriteRequest.OpType.INDEX) {
        IndexRequest indexRequest = (IndexRequest) docWriteRequest;
        if (!indexRequest.isRetry()) {
            return new String(indexRequest.source().toBytesRef().utf8ToString());
        }
    }
    return null;
}
 

private void parseErrorsTechnical(BulkRequest bulkRequest, Throwable failure) {
    bulkRequest.requests().stream()
            .filter(request -> request.opType() == DocWriteRequest.OpType.INDEX)
            .map(this::toRawMessage)
            .filter(message -> message != null)
            .forEach(rawMessage -> routeErrorTechnical(rawMessage, failure));

}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    br.waitForActiveShards(ActiveShardCount.ONE);
    return br;
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    br.waitForActiveShards(ActiveShardCount.ONE);
    return br;
}
 

private DocWriteRequest eventToRequest(EsConfig esConfig, EventType eventType, Map<String, String> values) {

        DocWriteRequest req = null;

        // column_name,column_name
        String id = esConfig.getEsIdFieldNames()
                .stream()
                .map(esId -> String.valueOf(values.get(esId)))
                .reduce((s1, s2) -> s1 + esConfig.getSeparator() + s2)
                .orElse("");

        if (StringUtils.isBlank(id)) {
            return null;
        }
        String type = esConfig.getType();
        String index = esConfig.getIndex();


        switch (eventType) {
            case INSERT:
            case UPDATE:
                UpdateRequest ur = new UpdateRequest(index, type, id);
                ur.doc(values);
                ur.docAsUpsert(true);
                req = ur;
                break;
            case DELETE:
                DeleteRequest dr = new DeleteRequest(index, type, id);
                dr.id(id);
                req = dr;
                break;
            default:
                break;
        }
        return req;
    }
 

private List<DocWriteRequest> toRequests(List<Invocation> invocations) {
    return invocations.stream()
            .map(this::toRequest)
            .filter(Objects::nonNull)
            .collect(toList());
}
 

public static List<AuditMessage> resolve(
        final Category category,
        final Origin origin,
        final String action,
        final String privilege,
        final String effectiveUser,
        final Boolean securityadmin,
        final String initiatingUser,
        final TransportAddress remoteAddress,
        final TransportRequest request,
        final Map<String, String> headers,
        final Task task,
        final IndexNameExpressionResolver resolver,
        final ClusterService cs,
        final Settings settings,
        final boolean logRequestBody,
        final boolean resolveIndices,
        final boolean resolveBulk,
        final String opendistrosecurityIndex,
        final boolean excludeSensitiveHeaders,
        final Throwable exception)  {

    if(resolveBulk && request instanceof BulkShardRequest) {
        final BulkItemRequest[] innerRequests = ((BulkShardRequest) request).items();
        final List<AuditMessage> messages = new ArrayList<AuditMessage>(innerRequests.length);

        for(BulkItemRequest ar: innerRequests) {
            final DocWriteRequest<?> innerRequest = ar.request();
            final AuditMessage msg = resolveInner(
                    category,
                    effectiveUser,
                    securityadmin,
                    initiatingUser,
                    remoteAddress,
                    action,
                    privilege,
                    origin,
                    innerRequest,
                    headers,
                    task,
                    resolver,
                    cs,
                    settings,
                    logRequestBody,
                    resolveIndices,
                    opendistrosecurityIndex,
                    excludeSensitiveHeaders,
                    exception);
             msg.addShardId(((BulkShardRequest) request).shardId());

            messages.add(msg);
        }

        return messages;
    }

    if(request instanceof BulkShardRequest) {

        if(category != Category.FAILED_LOGIN
                && category != Category.MISSING_PRIVILEGES
                && category != Category.OPENDISTRO_SECURITY_INDEX_ATTEMPT) {

            return Collections.emptyList();
        }
    }

    return Collections.singletonList(resolveInner(
            category,
            effectiveUser,
            securityadmin,
            initiatingUser,
            remoteAddress,
            action,
            privilege,
            origin,
            request,
            headers,
            task,
            resolver,
            cs,
            settings,
            logRequestBody,
            resolveIndices,
            opendistrosecurityIndex,
            excludeSensitiveHeaders,
            exception));
}
 

@Override
public void execute(Tuple tuple) {

    String url = tuple.getStringByField("url");

    // Distinguish the value used for indexing
    // from the one used for the status
    String normalisedurl = valueForURL(tuple);

    LOG.info("Indexing {} as {}", url, normalisedurl);

    Metadata metadata = (Metadata) tuple.getValueByField("metadata");
    String text = tuple.getStringByField("text");

    boolean keep = filterDocument(metadata);
    if (!keep) {
        LOG.info("Filtered {}", url);
        eventCounter.scope("Filtered").incrBy(1);
        // treat it as successfully processed even if
        // we do not index it
        _collector.emit(StatusStreamName, tuple, new Values(url, metadata,
                Status.FETCHED));
        _collector.ack(tuple);
        return;
    }

    String docID = org.apache.commons.codec.digest.DigestUtils
            .sha256Hex(normalisedurl);

    try {
        XContentBuilder builder = jsonBuilder().startObject();

        // display text of the document?
        if (fieldNameForText() != null) {
            builder.field(fieldNameForText(), trimText(text));
        }

        // send URL as field?
        if (fieldNameForURL() != null) {
            builder.field(fieldNameForURL(), normalisedurl);
        }

        // which metadata to display?
        Map<String, String[]> keyVals = filterMetadata(metadata);

        Iterator<String> iterator = keyVals.keySet().iterator();
        while (iterator.hasNext()) {
            String fieldName = iterator.next();
            String[] values = keyVals.get(fieldName);
            if (values.length == 1) {
                builder.field(fieldName, values[0]);
            } else if (values.length > 1) {
                builder.array(fieldName, values);
            }
        }

        builder.endObject();

        String sha256hex = org.apache.commons.codec.digest.DigestUtils
                .sha256Hex(normalisedurl);

        IndexRequest indexRequest = new IndexRequest(getIndexName(metadata))
                .source(builder).id(sha256hex);

        DocWriteRequest.OpType optype = DocWriteRequest.OpType.INDEX;

        if (create) {
            optype = DocWriteRequest.OpType.CREATE;
        }

        indexRequest.opType(optype);

        if (pipeline != null) {
            indexRequest.setPipeline(pipeline);
        }

        connection.getProcessor().add(indexRequest);

        eventCounter.scope("Indexed").incrBy(1);
        perSecMetrics.scope("Indexed").update(1);

        synchronized (waitAck) {
            waitAck.put(docID, tuple);
        }
    } catch (IOException e) {
        LOG.error("Error building document for ES", e);
        // do not send to status stream so that it gets replayed
        _collector.fail(tuple);
        if (docID != null) {
            synchronized (waitAck) {
                waitAck.invalidate(docID);
            }
        }
    }
}
 

private List<DocWriteRequest> toRequests(List<Helper> helpers) {
    EsConfig esConfig = helpers.get(0).esConfig;

    String sql = esConfig.getSql();
    List<String> parameters = esConfig.getParameters();
    boolean sqlExists = StringUtils.isNotBlank(sql) && parameters != null && !parameters.isEmpty();
    if (sqlExists) {
        // select xx where id in (?)
        // select xx where (id,name) in (?)
        List<String> param = new ArrayList<>();
        for (Helper helper : helpers) {
            Map<String, String> values = helper.context.getEvent().getDataList()
                    .stream()
                    .collect(Collectors.toMap(ColumnData::getName, ColumnData::getValue));
            List<String> args = new ArrayList<>();
            for (String k : parameters) {
                args.add(getValue(values.get(k)));
            }

            String arg = "(" + StringUtils.join(args, ",") + ")";
            param.add(arg);
        }

        sql = sql.replace("?", StringUtils.join(param, ","));

        List<Map<String, Object>> data = execute(sql, helpers.get(0).context);

        return data.stream()
                .map(line -> {
                    Map<String, String> lines = new LinkedHashMap<>();
                    for (Map.Entry<String, Object> e : line.entrySet()) {
                        lines.put(e.getKey(), String.valueOf(e.getValue()));
                    }
                    return lines;
                })
                .map(line -> eventToRequest(esConfig, EventType.INSERT, line))
                .filter(Objects::nonNull)
                .collect(Collectors.toList());
    } else {
        return helpers.stream()
                .map(helper ->
                        eventToRequest(
                                helper.esConfig,
                                helper.context.getEvent().getEventType(),
                                helper.context.getEvent().getDataList()
                                        .stream()
                                        .collect(Collectors.toMap(ColumnData::getName, ColumnData::getValue))
                        )
                )
                .filter(Objects::nonNull)
                .collect(Collectors.toList());
    }

}
 

public boolean bulk(List<DocWriteRequest> doc) {
    return bulk(doc, 10);
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    return br;
}
 

private List<DocWriteRequest> mapListToRequests(List<Map<String, Object>> data) {
    return null;
}
 

@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
	if (response.hasFailures()) {
		BulkItemResponse itemResponse;
		Throwable failure;
		RestStatus restStatus;
		DocWriteRequest actionRequest;

		try {
			for (int i = 0; i < response.getItems().length; i++) {
				itemResponse = response.getItems()[i];
				failure = callBridge.extractFailureCauseFromBulkItemResponse(itemResponse);
				if (failure != null) {
					restStatus = itemResponse.getFailure().getStatus();
					actionRequest = request.requests().get(i);
					if (restStatus == null) {
						if (actionRequest instanceof ActionRequest) {
							failureHandler.onFailure((ActionRequest) actionRequest, failure, -1, failureRequestIndexer);
						} else {
							throw new UnsupportedOperationException("The sink currently only supports ActionRequests");
						}
					} else {
						if (actionRequest instanceof ActionRequest) {
							failureHandler.onFailure((ActionRequest) actionRequest, failure, restStatus.getStatus(), failureRequestIndexer);
						} else {
							throw new UnsupportedOperationException("The sink currently only supports ActionRequests");
						}
					}
				}
			}
		} catch (Throwable t) {
			// fail the sink and skip the rest of the items
			// if the failure handler decides to throw an exception
			failureThrowable.compareAndSet(null, t);
		}
	}

	if (flushOnCheckpoint) {
		numPendingRequests.getAndAdd(-request.numberOfActions());
	}
}
 

public static <Request extends ActionRequest> List<String> getIndexFromRequest(Request request) {
	List<String> indexs = new ArrayList<>();

	if (request instanceof SingleShardRequest) {
		indexs.add(((SingleShardRequest<?>) request).index());
		return indexs;
	}

	if (request instanceof ReplicationRequest) {
		indexs.add(((ReplicationRequest<?>) request).index());
		return indexs;
	}

	if (request instanceof InstanceShardOperationRequest) {
		indexs.add(((InstanceShardOperationRequest<?>) request).index());
		return indexs;
	}

	if (request instanceof CreateIndexRequest) {
		indexs.add(((CreateIndexRequest) request).index());
		return indexs;
	}

	if (request instanceof PutMappingRequest) {
		indexs.add(((PutMappingRequest) request).getConcreteIndex().getName());
		return indexs;
	}

	if (request instanceof SearchRequest) {
		return Arrays.asList(((SearchRequest) request).indices());
	}

	if (request instanceof IndicesStatsRequest) {
		return Arrays.asList(((IndicesStatsRequest) request).indices());
	}

	if (request instanceof OpenIndexRequest) {
		return Arrays.asList(((OpenIndexRequest) request).indices());
	}

	if (request instanceof DeleteIndexRequest) {
		return Arrays.asList(((DeleteIndexRequest) request).indices());
	}

	if (request instanceof BulkRequest) {
		@SuppressWarnings("rawtypes") List<DocWriteRequest<?>> requests = ((BulkRequest) request).requests();

		if (CollectionUtils.isNotEmpty(requests)) {
			for (DocWriteRequest<?> docWriteRequest : requests) {
				indexs.add(docWriteRequest.index());
			}
			return indexs;
		}
	}

	if (request instanceof MultiGetRequest) {
		List<Item> items = ((MultiGetRequest) request).getItems();
		if (CollectionUtils.isNotEmpty(items)) {
			for (Item item : items) {
				indexs.add(item.index());
			}
			return indexs;
		}
	}

	// No matched request type to find specific index , set default value *
	indexs.add("*");
	return indexs;
}
 

private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
    BulkRequest br = new BulkRequest();
    br.add(doc);
    br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    return br;
}
 

private DocWriteRequest toRequest(Invocation invocation) {
    DocWriteRequest req = null;

    EsConfig esConfig = invocation.getParameter(Constants.CONFIG_NAME);

    Map<String, String> values = invocation.getEvent().getDataList()
            .stream()
            .collect(Collectors.toMap(CellData::getName, CellData::getValue));

    // column_name,column_name
    String id = esConfig.getEsId()
            .stream()
            .map(esId -> String.valueOf(values.get(esId)))
            .reduce((s1, s2) -> s1 + esConfig.getSeparator() + s2)
            .orElse("");

    if (StringUtils.isBlank(id)) {
        return null;
    }

    String type = esConfig.getType();
    String index = esConfig.getIndex();

    switch (invocation.getEvent().getWalType()) {
        case INSERT:
        case UPDATE:
            UpdateRequest ur = new UpdateRequest(index, type, id);
            ur.doc(values);
            ur.docAsUpsert(true);
            req = ur;
            break;
        case DELETE:
            DeleteRequest dr = new DeleteRequest(index, type, id);
            dr.id(id);
            req = dr;
            break;
        default:
            break;
    }
    return req;
}
 

private List<DocWriteRequest> toRequests(List<Helper> helpers) {
    EsConfig esConfig = helpers.get(0).esConfig;

    String sql = esConfig.getSql();
    List<String> parameters = esConfig.getParameters();
    boolean sqlExists = StringUtils.isNotBlank(sql) && parameters != null && !parameters.isEmpty();
    if (sqlExists) {
        // select xx where id in (?)
        // select xx where (id,name) in (?)
        List<String> param = new ArrayList<>();
        for (Helper helper : helpers) {
            Map<String, String> values = helper.context.getEvent().getDataList()
                    .stream()
                    .collect(Collectors.toMap(ColumnData::getName, ColumnData::getValue));
            List<String> args = new ArrayList<>();
            for (String k : parameters) {
                args.add(getValue(values.get(k)));
            }

            String arg = "(" + StringUtils.join(args, ",") + ")";
            param.add(arg);
        }

        sql = sql.replace("?", StringUtils.join(param, ","));

        List<Map<String, Object>> data = execute(sql, helpers.get(0).context);

        return data.stream()
                .map(line -> {
                    Map<String, String> lines = new LinkedHashMap<>();
                    for (Map.Entry<String, Object> e : line.entrySet()) {
                        lines.put(e.getKey(), String.valueOf(e.getValue()));
                    }
                    return lines;
                })
                .map(line -> eventToRequest(esConfig, EventType.INSERT, line))
                .filter(Objects::nonNull)
                .collect(Collectors.toList());
    } else {
        return helpers.stream()
                .map(helper ->
                        eventToRequest(
                                helper.esConfig,
                                helper.context.getEvent().getEventType(),
                                helper.context.getEvent().getDataList()
                                        .stream()
                                        .collect(Collectors.toMap(ColumnData::getName, ColumnData::getValue))
                        )
                )
                .filter(Objects::nonNull)
                .collect(Collectors.toList());
    }

}