org.apache.flink.streaming.connectors.elasticsearch5.ElasticsearchSink Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.elasticsearch5.ElasticsearchSink. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Elasticsearch5SinkExample.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableSysoutLogging();
		env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
Example #2
Source File: Elasticsearch5SinkExample.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableSysoutLogging();
		env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
Example #3
Source File: Elasticsearch5TableSink.java    From alchemy with Apache License 2.0 5 votes vote down vote up 
private ElasticsearchSink<Row> createEsSink() {
    Map<String, String> userConfig = createUserConfig();
    List<InetSocketAddress> transports = new ArrayList<>();
    addTransportAddress(transports, this.elasticsearch5Properties.getTransports());
    ActionRequestFailureHandler actionRequestFailureHandler = ActionRequestFailureHandlerUtil.createFailureHandler(this.elasticsearch5Properties.getFailureHandler());
    Integer fieldIndex = findIndex(this.elasticsearch5Properties.getIndexField(), this.fieldNames);
    return new ElasticsearchSink<>(userConfig, transports,
            new Elasticsearch5TableFunction(
                    this.elasticsearch5Properties.getIndex(),
                    fieldIndex,
                    this.elasticsearch5Properties.getIndexType(),
                    jsonRowSchema),
            actionRequestFailureHandler);
}
Example #4
Source File: Elasticsearch5SinkExample.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

		final ParameterTool parameterTool = ParameterTool.fromArgs(args);

		if (parameterTool.getNumberOfParameters() < 3) {
			System.out.println("Missing parameters!\n" +
				"Usage: --numRecords <numRecords> --index <index> --type <type>");
			return;
		}

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
				env.enableCheckpointing(5000);

		DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1)
			.map(new MapFunction<Long, String>() {
				@Override
				public String map(Long value) throws Exception {
					return "message #" + value;
				}
			});

		Map<String, String> userConfig = new HashMap<>();
		userConfig.put("cluster.name", "elasticsearch");
		// This instructs the sink to emit after every element, otherwise they would be buffered
		userConfig.put(ElasticsearchSink.CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, "1");

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 9300));

		source.addSink(new ElasticsearchSink<>(userConfig, transports, new ElasticsearchSinkFunction<String>() {
			@Override
			public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
				indexer.add(createIndexRequest(element, parameterTool));
			}
		}));

		env.execute("Elasticsearch5.x end to end sink test example");
	}
Example #5
Source File: Elasticsearch5TableSink.java    From alchemy with Apache License 2.0 4 votes vote down vote up 
@Override
public void emitDataStream(DataStream<Row> dataStream) {
    ElasticsearchSink<Row> elasticsearchSink = createEsSink();
    dataStream.addSink(elasticsearchSink);
}
Example #6
Source File: PravegaAnomalyDetectionProcessor.java    From pravega-samples with Apache License 2.0 4 votes vote down vote up 
@Override
public void run() throws Exception {

	// Configure the Pravega event reader
	FlinkPravegaReader<Event> flinkPravegaReader = FlinkPravegaReader.<Event>builder()
			.withPravegaConfig(getPravegaConfig())
			.forStream(getStreamId())
			.withDeserializationSchema(PravegaSerialization.deserializationFor(Event.class))
			.build();

	// Configure the Flink job environment
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.setParallelism(appConfiguration.getPipeline().getParallelism());
	if(!appConfiguration.getPipeline().isDisableCheckpoint()) {
		long checkpointInterval = appConfiguration.getPipeline().getCheckpointIntervalInMilliSec();
		env.enableCheckpointing(checkpointInterval, CheckpointingMode.EXACTLY_ONCE);
	}

	// Construct a dataflow program:

	// 1. read network events
	DataStream<Event> events = env.addSource(flinkPravegaReader).name("AnomalyEventReader");
	events.print();

	// 2. detect anomalies in event sequences
	DataStream<Event.Alert> anomalies = events
			.keyBy("sourceAddress")
			.flatMap(new EventStateMachineMapper())
			.name("AnomalyDetector");
	anomalies.print();

	// 3. aggregate the alerts by network over time
	long maxOutOfOrderness = appConfiguration.getPipeline().getWatermarkOffsetInSec();
	DataStream<Event.Alert> timestampedAnomalies = anomalies
			.assignTimestampsAndWatermarks(new EventTimeExtractor(Time.seconds(maxOutOfOrderness)))
			.name("TimeExtractor");

	long windowIntervalInSeconds = appConfiguration.getPipeline().getWindowIntervalInSeconds();
	DataStream<Result> aggregate = timestampedAnomalies
			.keyBy("networkId")
			.window(TumblingEventTimeWindows.of(Time.seconds(windowIntervalInSeconds)))
			.fold(new Result(), new FoldAlertsToResult())
			.name("Aggregate");
	aggregate.print();

	// 4. emit the alerts to Elasticsearch (optional)
	if(appConfiguration.getPipeline().getElasticSearch().isSinkResults()) {
		ElasticsearchSink<Result> elasticSink = sinkToElasticSearch(appConfiguration);
		aggregate.addSink(elasticSink).name("ElasticSearchSink");
	}

	// Execute the program in the Flink environment
	env.execute(appConfiguration.getName());
}
Example #7
Source File: PravegaAnomalyDetectionProcessor.java    From pravega-samples with Apache License 2.0 4 votes vote down vote up 
private ElasticsearchSink<Result> sinkToElasticSearch(AppConfiguration appConfiguration) throws Exception {

		String host = appConfiguration.getPipeline().getElasticSearch().getHost();
		int port = appConfiguration.getPipeline().getElasticSearch().getPort();
		String cluster = appConfiguration.getPipeline().getElasticSearch().getCluster();

		String index = appConfiguration.getPipeline().getElasticSearch().getIndex();
		String type = appConfiguration.getPipeline().getElasticSearch().getType();

		Map<String, String> config = new HashMap<>();
		config.put("bulk.flush.max.actions", "1");
		config.put("cluster.name", cluster);
		config.put("client.transport.sniff", "false");

		final InetSocketAddress socketAddress = new InetSocketAddress(host,port);

		List<InetSocketAddress> transports = new ArrayList<>();
		transports.add(socketAddress);


		return new ElasticsearchSink<>(config, transports, new ResultSinkFunction(index, type));
	}