org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction Java Examples

public static ElasticsearchSink<TraceSummary> build() {
    List<HttpHost> httpHosts = new ArrayList<>();
    httpHosts.add(new HttpHost("127.0.0.1", 9200, "http"));

    ElasticsearchSink.Builder<TraceSummary> esSinkBuilder = new ElasticsearchSink.Builder<>(httpHosts,
            new ElasticsearchSinkFunction<TraceSummary>() {

                @Override
                public void process(TraceSummary summary, RuntimeContext ctx, RequestIndexer indexer) {
                    indexer.add(Requests.indexRequest()//
                        .index("trace-summaries") //
                        .type("trace-summaries") //
                        .id(summary.traceId) //
                        .source(asJson(summary)));
                }
            });

    // configuration for the bulk requests; this instructs the sink to emit after
    // every element, otherwise they would be buffered
    esSinkBuilder.setBulkFlushMaxActions(1);

    return esSinkBuilder.build();
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSinkForNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction,
		String ipAddress) throws Exception {

	ArrayList<HttpHost> httpHosts = new ArrayList<>();
	httpHosts.add(new HttpHost(ipAddress, 9200, "http"));

	ElasticsearchSink.Builder<Tuple2<Integer, String>> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction);
	builder.setBulkFlushMaxActions(bulkFlushMaxActions);

	return builder.build();
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction,
		String ipAddress) throws Exception {

	List<InetSocketAddress> transports = new ArrayList<>();
	transports.add(new InetSocketAddress(InetAddress.getByName(ipAddress), 9300));

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			transports,
			elasticsearchSinkFunction);
}
 

private ElasticsearchSink(
	Map<String, String> bulkRequestsConfig,
	List<HttpHost> httpHosts,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler,
	RestClientFactory restClientFactory) {

	super(new Elasticsearch6ApiCallBridge(httpHosts, restClientFactory),  bulkRequestsConfig, elasticsearchSinkFunction, failureHandler);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSink(
		int bulkFlushMaxActions,
		String clusterName,
		List<InetSocketAddress> transportAddresses,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) {

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			transportAddresses,
			elasticsearchSinkFunction);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSinkForNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction,
		String ipAddress) throws Exception {

	ArrayList<HttpHost> httpHosts = new ArrayList<>();
	httpHosts.add(new HttpHost(ipAddress, 9200, "http"));

	ElasticsearchSink.Builder<Tuple2<Integer, String>> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction);
	builder.setBulkFlushMaxActions(bulkFlushMaxActions);

	return builder.build();
}
 

public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableSysoutLogging();
		env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
 

private ElasticsearchSink(
	Map<String, String> bulkRequestsConfig,
	List<HttpHost> httpHosts,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler,
	RestClientFactory restClientFactory) {

	super(new Elasticsearch7ApiCallBridge(httpHosts, restClientFactory),  bulkRequestsConfig, elasticsearchSinkFunction, failureHandler);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSink(
		int bulkFlushMaxActions,
		String clusterName,
		List<HttpHost> httpHosts,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) {

	ElasticsearchSink.Builder<Tuple2<Integer, String>> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction);
	builder.setBulkFlushMaxActions(bulkFlushMaxActions);

	return builder.build();
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSink(
		int bulkFlushMaxActions,
		String clusterName,
		List<HttpHost> httpHosts,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) {

	ElasticsearchSink.Builder<Tuple2<Integer, String>> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction);
	builder.setBulkFlushMaxActions(bulkFlushMaxActions);

	return builder.build();
}
 

/**
 * Creates a new {@code ElasticsearchSink} that connects to the cluster using a {@link TransportClient}.
 *
 * @param userConfig The map of user settings that are used when constructing the {@link TransportClient} and {@link BulkProcessor}
 * @param transportAddresses The addresses of Elasticsearch nodes to which to connect using a {@link TransportClient}
 * @param elasticsearchSinkFunction This is used to generate multiple {@link ActionRequest} from the incoming element
 */
public ElasticsearchSink(
	Map<String, String> userConfig,
	List<InetSocketAddress> transportAddresses,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction) {

	this(userConfig, transportAddresses, elasticsearchSinkFunction, new NoOpFailureHandler());
}
 

/**
 * Creates a new {@code ElasticsearchSink} that connects to the cluster using a {@link TransportClient}.
 *
 * @param userConfig The map of user settings that are used when constructing the {@link TransportClient} and {@link BulkProcessor}
 * @param transportAddresses The addresses of Elasticsearch nodes to which to connect using a {@link TransportClient}
 * @param elasticsearchSinkFunction This is used to generate multiple {@link ActionRequest} from the incoming element
 * @param failureHandler This is used to handle failed {@link ActionRequest}
 */
public ElasticsearchSink(
	Map<String, String> userConfig,
	List<InetSocketAddress> transportAddresses,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler) {

	super(new Elasticsearch5ApiCallBridge(transportAddresses), userConfig, elasticsearchSinkFunction, failureHandler);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSink(
		int bulkFlushMaxActions,
		String clusterName,
		List<InetSocketAddress> addresses,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) {

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			addresses,
			elasticsearchSinkFunction);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction,
		String ipAddress) throws Exception {

	List<InetSocketAddress> transports = new ArrayList<>();
	transports.add(new InetSocketAddress(InetAddress.getByName(ipAddress), 9300));

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			transports,
			elasticsearchSinkFunction);
}
 

private ElasticsearchSink(
	Map<String, String> bulkRequestsConfig,
	List<HttpHost> httpHosts,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler,
	RestClientFactory restClientFactory) {

	super(new Elasticsearch6ApiCallBridge(httpHosts, restClientFactory),  bulkRequestsConfig, elasticsearchSinkFunction, failureHandler);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSink(
		int bulkFlushMaxActions,
		String clusterName,
		List<HttpHost> httpHosts,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) {

	ElasticsearchSink.Builder<Tuple2<Integer, String>> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction);
	builder.setBulkFlushMaxActions(bulkFlushMaxActions);

	return builder.build();
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSinkForNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction,
		String ipAddress) throws Exception {

	ArrayList<HttpHost> httpHosts = new ArrayList<>();
	httpHosts.add(new HttpHost(ipAddress, 9200, "http"));

	ElasticsearchSink.Builder<Tuple2<Integer, String>> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction);
	builder.setBulkFlushMaxActions(bulkFlushMaxActions);

	return builder.build();
}
 

public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
				env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, RestHighLevelClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction,
		String ipAddress) throws Exception {

	List<InetSocketAddress> transports = new ArrayList<>();
	transports.add(new InetSocketAddress(InetAddress.getByName(ipAddress), 9300));

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			transports,
			elasticsearchSinkFunction);
}
 

/**
 * Creates a new {@code ElasticsearchSink} that connects to the cluster using a {@link TransportClient}.
 *
 * @param userConfig The map of user settings that are used when constructing the {@link TransportClient} and {@link BulkProcessor}
 * @param transportAddresses The addresses of Elasticsearch nodes to which to connect using a {@link TransportClient}
 * @param elasticsearchSinkFunction This is used to generate multiple {@link ActionRequest} from the incoming element
 */
public ElasticsearchSink(
	Map<String, String> userConfig,
	List<InetSocketAddress> transportAddresses,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction) {

	this(userConfig, transportAddresses, elasticsearchSinkFunction, new NoOpFailureHandler());
}
 

/**
 * Creates a new {@code ElasticsearchSink} that connects to the cluster using a {@link TransportClient}.
 *
 * @param userConfig The map of user settings that are used when constructing the {@link TransportClient} and {@link BulkProcessor}
 * @param transportAddresses The addresses of Elasticsearch nodes to which to connect using a {@link TransportClient}
 * @param elasticsearchSinkFunction This is used to generate multiple {@link ActionRequest} from the incoming element
 * @param failureHandler This is used to handle failed {@link ActionRequest}
 */
public ElasticsearchSink(
	Map<String, String> userConfig,
	List<InetSocketAddress> transportAddresses,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler) {

	super(new Elasticsearch5ApiCallBridge(transportAddresses), userConfig, elasticsearchSinkFunction, failureHandler);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSink(
		int bulkFlushMaxActions,
		String clusterName,
		List<InetSocketAddress> addresses,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) {

	return new ElasticsearchSink<>(
			Collections.unmodifiableMap(createUserConfig(bulkFlushMaxActions, clusterName)),
			addresses,
			elasticsearchSinkFunction);
}
 

@Override
protected ElasticsearchSinkBase<Tuple2<Integer, String>, TransportClient> createElasticsearchSinkForEmbeddedNode(
		int bulkFlushMaxActions,
		String clusterName,
		ElasticsearchSinkFunction<Tuple2<Integer, String>> elasticsearchSinkFunction) throws Exception {

	return createElasticsearchSinkForNode(
			bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1");
}