org.apache.flink.streaming.connectors.elasticsearch5.ElasticsearchSink Java Examples

public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableSysoutLogging();
		env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
 

public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableSysoutLogging();
		env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
 

private ElasticsearchSink<Row> createEsSink() {
    Map<String, String> userConfig = createUserConfig();
    List<InetSocketAddress> transports = new ArrayList<>();
    addTransportAddress(transports, this.elasticsearch5Properties.getTransports());
    ActionRequestFailureHandler actionRequestFailureHandler = ActionRequestFailureHandlerUtil.createFailureHandler(this.elasticsearch5Properties.getFailureHandler());
    Integer fieldIndex = findIndex(this.elasticsearch5Properties.getIndexField(), this.fieldNames);
    return new ElasticsearchSink<>(userConfig, transports,
            new Elasticsearch5TableFunction(
                    this.elasticsearch5Properties.getIndex(),
                    fieldIndex,
                    this.elasticsearch5Properties.getIndexType(),
                    jsonRowSchema),
            actionRequestFailureHandler);
}
 

public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
				env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
 

@Override
public void emitDataStream(DataStream<Row> dataStream) {
    ElasticsearchSink<Row> elasticsearchSink = createEsSink();
    dataStream.addSink(elasticsearchSink);
}
 

@Override
public void run() throws Exception {

	// Configure the Pravega event reader
	FlinkPravegaReader<Event> flinkPravegaReader = FlinkPravegaReader.<Event>builder()
			.withPravegaConfig(getPravegaConfig())
			.forStream(getStreamId())
			.withDeserializationSchema(PravegaSerialization.deserializationFor(Event.class))
			.build();

	// Configure the Flink job environment
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.setParallelism(appConfiguration.getPipeline().getParallelism());
	if(!appConfiguration.getPipeline().isDisableCheckpoint()) {
		long checkpointInterval = appConfiguration.getPipeline().getCheckpointIntervalInMilliSec();
		env.enableCheckpointing(checkpointInterval, CheckpointingMode.EXACTLY_ONCE);
	}

	// Construct a dataflow program:

	// 1. read network events
	DataStream<Event> events = env.addSource(flinkPravegaReader).name("AnomalyEventReader");
	events.print();

	// 2. detect anomalies in event sequences
	DataStream<Event.Alert> anomalies = events
			.keyBy("sourceAddress")
			.flatMap(new EventStateMachineMapper())
			.name("AnomalyDetector");
	anomalies.print();

	// 3. aggregate the alerts by network over time
	long maxOutOfOrderness = appConfiguration.getPipeline().getWatermarkOffsetInSec();
	DataStream<Event.Alert> timestampedAnomalies = anomalies
			.assignTimestampsAndWatermarks(new EventTimeExtractor(Time.seconds(maxOutOfOrderness)))
			.name("TimeExtractor");

	long windowIntervalInSeconds = appConfiguration.getPipeline().getWindowIntervalInSeconds();
	DataStream<Result> aggregate = timestampedAnomalies
			.keyBy("networkId")
			.window(TumblingEventTimeWindows.of(Time.seconds(windowIntervalInSeconds)))
			.fold(new Result(), new FoldAlertsToResult())
			.name("Aggregate");
	aggregate.print();

	// 4. emit the alerts to Elasticsearch (optional)
	if(appConfiguration.getPipeline().getElasticSearch().isSinkResults()) {
		ElasticsearchSink<Result> elasticSink = sinkToElasticSearch(appConfiguration);
		aggregate.addSink(elasticSink).name("ElasticSearchSink");
	}

	// Execute the program in the Flink environment
	env.execute(appConfiguration.getName());
}
 

private ElasticsearchSink<Result> sinkToElasticSearch(AppConfiguration appConfiguration) throws Exception {

		String host = appConfiguration.getPipeline().getElasticSearch().getHost();
		int port = appConfiguration.getPipeline().getElasticSearch().getPort();
		String cluster = appConfiguration.getPipeline().getElasticSearch().getCluster();

		String index = appConfiguration.getPipeline().getElasticSearch().getIndex();
		String type = appConfiguration.getPipeline().getElasticSearch().getType();

		Map<String, String> config = new HashMap<>();
		config.put("bulk.flush.max.actions", "1");
		config.put("cluster.name", cluster);
		config.put("client.transport.sniff", "false");

		final InetSocketAddress socketAddress = new InetSocketAddress(host,port);

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(socketAddress);


		return new ElasticsearchSink<>(config, transports, new ResultSinkFunction(index, type));
	}