org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Elasticsearch6SinkExample.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable {
	if (action instanceof IndexRequest) {
		Map<String, Object> json = new HashMap<>();
		json.put("data", ((IndexRequest) action).source());

		indexer.add(
			Requests.indexRequest()
				.index(index)
				.type(type)
				.id(((IndexRequest) action).id())
				.source(json));
	} else {
		throw new IllegalStateException("unexpected");
	}
}
Example #2
Source File: Elasticsearch6SinkFunction.java    From alchemy with Apache License 2.0 6 votes vote down vote up 
private void processUpsert(Row row, RequestIndexer indexer) {
    final byte[] document = serializationSchema.serialize(row);
    if (keyFieldIndices.length == 0) {
        final IndexRequest indexRequest = requestFactory.createIndexRequest(
                getIndex(row),
                docType,
                contentType,
                document);
        indexer.add(indexRequest);
    } else {
        final String key = createKey(row);
        final UpdateRequest updateRequest = requestFactory.createUpdateRequest(
                getIndex(row),
                docType,
                key,
                contentType,
                document);
        indexer.add(updateRequest);
    }
}
Example #3
Source File: LogSink2ES.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void sink2es(SingleOutputStreamOperator<LogEvent> logDataStream, ParameterTool parameterTool) {
    List<HttpHost> esAddresses;
    try {
         esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    } catch (MalformedURLException e) {
        log.error("get es address has an error", e);
        return;
    }
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 5);

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, logDataStream,
            (LogEvent logEvent, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index("zhisheng_log")
                        .type(ZHISHENG)
                        .source(GsonUtil.toJSONBytes(logEvent), XContentType.JSON));
            },
            parameterTool);
}
Example #4
Source File: Elasticsearch6SinkExample.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable {
	if (action instanceof IndexRequest) {
		Map<String, Object> json = new HashMap<>();
		json.put("data", ((IndexRequest) action).source());

		indexer.add(
			Requests.indexRequest()
				.index(index)
				.type(type)
				.id(((IndexRequest) action).id())
				.source(json));
	} else {
		throw new IllegalStateException("unexpected");
	}
}
Example #5
Source File: RetryRequestFailureHandler.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest actionRequest, Throwable throwable, int i, RequestIndexer requestIndexer) throws Throwable {
    if (ExceptionUtils.findThrowable(throwable, EsRejectedExecutionException.class).isPresent()) {
        requestIndexer.add(new ActionRequest[]{actionRequest});
    } else {
        if (ExceptionUtils.findThrowable(throwable, SocketTimeoutException.class).isPresent()) {
            return;
        } else {
            Optional<IOException> exp = ExceptionUtils.findThrowable(throwable, IOException.class);
            if (exp.isPresent()) {
                IOException ioExp = exp.get();
                if (ioExp != null && ioExp.getMessage() != null && ioExp.getMessage().contains("max retry timeout")) {
                    log.error(ioExp.getMessage());
                    return;
                }
            }
        }
        throw throwable;
    }
}
Example #6
Source File: Sink2ES6Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);
    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);

    List<HttpHost> esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 5);

    log.info("-----esAddresses = {}, parameterTool = {}, ", esAddresses, parameterTool);

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, data,
            (MetricEvent metric, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index(ZHISHENG + "_" + metric.getName())
                        .type(ZHISHENG)
                        .source(GsonUtil.toJSONBytes(metric), XContentType.JSON));
            },
            parameterTool);
    env.execute("flink learning connectors es6");
}
Example #7
Source File: RetryRequestFailureHandler.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest actionRequest, Throwable throwable, int i, RequestIndexer requestIndexer) throws Throwable {
    if (ExceptionUtils.findThrowable(throwable, EsRejectedExecutionException.class).isPresent()) {
        requestIndexer.add(new ActionRequest[]{actionRequest});
    } else {
        if (ExceptionUtils.findThrowable(throwable, SocketTimeoutException.class).isPresent()) {
            return;
        } else {
            Optional<IOException> exp = ExceptionUtils.findThrowable(throwable, IOException.class);
            if (exp.isPresent()) {
                IOException ioExp = exp.get();
                if (ioExp != null && ioExp.getMessage() != null && ioExp.getMessage().contains("max retry timeout")) {
                    log.error(ioExp.getMessage());
                    return;
                }
            }
        }
        throw throwable;
    }
}
Example #8
Source File: RetryRequestFailureHandler.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest actionRequest, Throwable throwable, int i, RequestIndexer requestIndexer) throws Throwable {
    if (ExceptionUtils.findThrowable(throwable, EsRejectedExecutionException.class).isPresent()) {
        requestIndexer.add(new ActionRequest[]{actionRequest});
    } else {
        if (ExceptionUtils.findThrowable(throwable, SocketTimeoutException.class).isPresent()) {
            return;
        } else {
            Optional<IOException> exp = ExceptionUtils.findThrowable(throwable, IOException.class);
            if (exp.isPresent()) {
                IOException ioExp = exp.get();
                if (ioExp != null && ioExp.getMessage() != null && ioExp.getMessage().contains("max retry timeout")) {
                    log.error(ioExp.getMessage());
                    return;
                }
            }
        }
        throw throwable;
    }
}
Example #9
Source File: ESSink.java    From Mastering-Distributed-Tracing with MIT License 6 votes vote down vote up 
public static ElasticsearchSink<TraceSummary> build() {
    List<HttpHost> httpHosts = new ArrayList<>();
    httpHosts.add(new HttpHost("127.0.0.1", 9200, "http"));

    ElasticsearchSink.Builder<TraceSummary> esSinkBuilder = new ElasticsearchSink.Builder<>(httpHosts,
            new ElasticsearchSinkFunction<TraceSummary>() {

                @Override
                public void process(TraceSummary summary, RuntimeContext ctx, RequestIndexer indexer) {
                    indexer.add(Requests.indexRequest()//
                        .index("trace-summaries") //
                        .type("trace-summaries") //
                        .id(summary.traceId) //
                        .source(asJson(summary)));
                }
            });

    // configuration for the bulk requests; this instructs the sink to emit after
    // every element, otherwise they would be buffered
    esSinkBuilder.setBulkFlushMaxActions(1);

    return esSinkBuilder.build();
}
Example #10
Source File: RetryRequestFailureHandler.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest actionRequest, Throwable throwable, int i, RequestIndexer requestIndexer) throws Throwable {
    if (ExceptionUtils.findThrowable(throwable, EsRejectedExecutionException.class).isPresent()) {
        requestIndexer.add(new ActionRequest[]{actionRequest});
    } else {
        if (ExceptionUtils.findThrowable(throwable, SocketTimeoutException.class).isPresent()) {
            return;
        } else {
            Optional<IOException> exp = ExceptionUtils.findThrowable(throwable, IOException.class);
            if (exp.isPresent()) {
                IOException ioExp = exp.get();
                if (ioExp != null && ioExp.getMessage() != null && ioExp.getMessage().contains("max retry timeout")) {
                    log.error(ioExp.getMessage());
                    return;
                }
            }
        }
        throw throwable;
    }
}
Example #11
Source File: RowElasticsearchSinkFunction.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void process(
		RowData element,
		RuntimeContext ctx,
		RequestIndexer indexer) {
	switch (element.getRowKind()) {
		case INSERT:
		case UPDATE_AFTER:
			processUpsert(element, indexer);
			break;
		case UPDATE_BEFORE:
		case DELETE:
			processDelete(element, indexer);
			break;
		default:
			throw new TableException("Unsupported message kind: " + element.getRowKind());
	}
}
Example #12
Source File: RetryRequestFailureHandler.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest actionRequest, Throwable throwable, int i, RequestIndexer requestIndexer) throws Throwable {
    if (ExceptionUtils.findThrowable(throwable, EsRejectedExecutionException.class).isPresent()) {
        requestIndexer.add(new ActionRequest[]{actionRequest});
    } else {
        if (ExceptionUtils.findThrowable(throwable, SocketTimeoutException.class).isPresent()) {
            return;
        } else {
            Optional<IOException> exp = ExceptionUtils.findThrowable(throwable, IOException.class);
            if (exp.isPresent()) {
                IOException ioExp = exp.get();
                if (ioExp != null && ioExp.getMessage() != null && ioExp.getMessage().contains("max retry timeout")) {
                    log.error(ioExp.getMessage());
                    return;
                }
            }
        }
        throw throwable;
    }
}
Example #13
Source File: Sink2ES6Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);
    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);

    List<HttpHost> esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 5);

    log.info("-----esAddresses = {}, parameterTool = {}, ", esAddresses, parameterTool);

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, data,
            (MetricEvent metric, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index(ZHISHENG + "_" + metric.getName())
                        .type(ZHISHENG)
                        .source(GsonUtil.toJSONBytes(metric), XContentType.JSON));
            },
            parameterTool);
    env.execute("flink learning connectors es6");
}
Example #14
Source File: RetryRequestFailureHandler.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest actionRequest, Throwable throwable, int i, RequestIndexer requestIndexer) throws Throwable {
    if (ExceptionUtils.findThrowable(throwable, EsRejectedExecutionException.class).isPresent()) {
        requestIndexer.add(new ActionRequest[]{actionRequest});
    } else {
        if (ExceptionUtils.findThrowable(throwable, SocketTimeoutException.class).isPresent()) {
            return;
        } else {
            Optional<IOException> exp = ExceptionUtils.findThrowable(throwable, IOException.class);
            if (exp.isPresent()) {
                IOException ioExp = exp.get();
                if (ioExp != null && ioExp.getMessage() != null && ioExp.getMessage().contains("max retry timeout")) {
                    log.error(ioExp.getMessage());
                    return;
                }
            }
        }
        throw throwable;
    }
}
Example #15
Source File: LogSink2ES.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void sink2es(SingleOutputStreamOperator<LogEvent> logDataStream, ParameterTool parameterTool) {
    List<HttpHost> esAddresses;
    try {
         esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    } catch (MalformedURLException e) {
        log.error("get es address has an error", e);
        return;
    }
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 5);

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, logDataStream,
            (LogEvent logEvent, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index("zhisheng_log")
                        .type(ZHISHENG)
                        .source(GsonUtil.toJSONBytes(logEvent), XContentType.JSON));
            },
            parameterTool);
}
Example #16
Source File: Elasticsearch6SinkExample.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable {
	if (action instanceof IndexRequest) {
		Map<String, Object> json = new HashMap<>();
		json.put("data", ((IndexRequest) action).source());

		indexer.add(
			Requests.indexRequest()
				.index(index)
				.type(type)
				.id(((IndexRequest) action).id())
				.source(json));
	} else {
		throw new IllegalStateException("unexpected");
	}
}
Example #17
Source File: RowElasticsearchSinkFunction.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void processUpsert(RowData row, RequestIndexer indexer) {
	final byte[] document = serializationSchema.serialize(row);
	final String key = createKey.apply(row);
	if (key != null) {
		final UpdateRequest updateRequest = requestFactory.createUpdateRequest(
			indexGenerator.generate(row),
			docType,
			key,
			contentType,
			document);
		indexer.add(updateRequest);
	} else {
		final IndexRequest indexRequest = requestFactory.createIndexRequest(
			indexGenerator.generate(row),
			docType,
			key,
			contentType,
			document);
		indexer.add(indexRequest);
	}
}
Example #18
Source File: Elasticsearch6ApiCallBridge.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public RequestIndexer createBulkProcessorIndexer(
		BulkProcessor bulkProcessor,
		boolean flushOnCheckpoint,
		AtomicLong numPendingRequestsRef) {
	return new Elasticsearch6BulkProcessorIndexer(
		bulkProcessor,
		flushOnCheckpoint,
		numPendingRequestsRef);
}
Example #19
Source File: Elasticsearch7ApiCallBridge.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public RequestIndexer createBulkProcessorIndexer(
		BulkProcessor bulkProcessor,
		boolean flushOnCheckpoint,
		AtomicLong numPendingRequestsRef) {
	return new Elasticsearch7BulkProcessorIndexer(
		bulkProcessor,
		flushOnCheckpoint,
		numPendingRequestsRef);
}
Example #20
Source File: RowElasticsearchSinkFunction.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void processDelete(RowData row, RequestIndexer indexer) {
	final String key = createKey.apply(row);
	final DeleteRequest deleteRequest = requestFactory.createDeleteRequest(
		indexGenerator.generate(row),
		docType,
		key);
	indexer.add(deleteRequest);
}
Example #21
Source File: RetryRejectedExecutionFailureHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable {
	LOG.error("Failed Elasticsearch item request: {}", failure.getMessage(), failure);
	if (ExceptionUtils.findThrowable(failure, EsRejectedExecutionException.class).isPresent()) {
		indexer.add(action);
	} else {
		// rethrow all other failures
		throw failure;
	}
}
Example #22
Source File: Elasticsearch7SinkExample.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable {
	if (action instanceof IndexRequest) {
		Map<String, Object> json = new HashMap<>();
		json.put("data", ((IndexRequest) action).source());

		indexer.add(
			Requests.indexRequest()
				.index(index)
				.id(((IndexRequest) action).id())
				.source(json));
	} else {
		throw new IllegalStateException("unexpected");
	}
}
Example #23
Source File: Elasticsearch5SinkExample.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
				env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
Example #24
Source File: Elasticsearch6SinkFunction.java    From alchemy with Apache License 2.0 5 votes vote down vote up 
private void processDelete(Row row, RequestIndexer indexer) {
    final String key = createKey(row);
    final DeleteRequest deleteRequest = requestFactory.createDeleteRequest(
            getIndex(row),
            docType,
            key);
    indexer.add(deleteRequest);
}
Example #25
Source File: Elasticsearch5TableFunction.java    From alchemy with Apache License 2.0 5 votes vote down vote up 
@Override
public void process(Row row, RuntimeContext runtimeContext,
                    RequestIndexer requestIndexer) {
    if (row == null) {
        return;
    }
    requestIndexer.add(createIndexRequest(row));
    numRecordsOut = createOrGet(numRecordsOut, runtimeContext);
    numRecordsOut.inc();
}
Example #26
Source File: OldNewElasticsearchSinkFunctionBridge.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void process(T element, RuntimeContext ctx, RequestIndexer indexer) {
	if (reusedRequestIndexerBridge == null) {
		reusedRequestIndexerBridge = new OldNewRequestIndexerBridge(indexer);
	}
	deprecated.process(element, ctx, reusedRequestIndexerBridge);
}
Example #27
Source File: Elasticsearch6ApiCallBridge.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public RequestIndexer createBulkProcessorIndexer(
		BulkProcessor bulkProcessor,
		boolean flushOnCheckpoint,
		AtomicLong numPendingRequestsRef) {
	return new Elasticsearch6BulkProcessorIndexer(
		bulkProcessor,
		flushOnCheckpoint,
		numPendingRequestsRef);
}
Example #28
Source File: RetryRejectedExecutionFailureHandler.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable {
	if (ExceptionUtils.findThrowable(failure, EsRejectedExecutionException.class).isPresent()) {
		indexer.add(action);
	} else {
		// rethrow all other failures
		throw failure;
	}
}
Example #29
Source File: Elasticsearch5SinkExample.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableSysoutLogging();
		env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
Example #30
Source File: Elasticsearch6ApiCallBridge.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public RequestIndexer createBulkProcessorIndexer(
		BulkProcessor bulkProcessor,
		boolean flushOnCheckpoint,
		AtomicLong numPendingRequestsRef) {
	return new Elasticsearch6BulkProcessorIndexer(
		bulkProcessor,
		flushOnCheckpoint,
		numPendingRequestsRef);
}