org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ESSink.java    From Mastering-Distributed-Tracing with MIT License 6 votes vote down vote up 
public static ElasticsearchSink<TraceSummary> build() {
    List<HttpHost> httpHosts = new ArrayList<>();
    httpHosts.add(new HttpHost("127.0.0.1", 9200, "http"));

    ElasticsearchSink.Builder<TraceSummary> esSinkBuilder = new ElasticsearchSink.Builder<>(httpHosts,
            new ElasticsearchSinkFunction<TraceSummary>() {

                @Override
                public void process(TraceSummary summary, RuntimeContext ctx, RequestIndexer indexer) {
                    indexer.add(Requests.indexRequest()//
                        .index("trace-summaries") //
                        .type("trace-summaries") //
                        .id(summary.traceId) //
                        .source(asJson(summary)));
                }
            });

    // configuration for the bulk requests; this instructs the sink to emit after
    // every element, otherwise they would be buffered
    esSinkBuilder.setBulkFlushMaxActions(1);

    return esSinkBuilder.build();
}
Example #2
Source File: Elasticsearch6DynamicSinkTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public ElasticsearchSink.Builder<RowData> createBuilder(
		List<HttpHost> httpHosts,
		RowElasticsearchSinkFunction upsertSinkFunction) {
	builderSpy = Mockito.spy(new ElasticsearchSink.Builder<>(httpHosts, upsertSinkFunction));
	doAnswer(
		invocation -> {
			sinkSpy = Mockito.spy((ElasticsearchSink<RowData>) invocation.callRealMethod());
			return sinkSpy;
		}
	).when(builderSpy).build();

	return builderSpy;
}
Example #3
Source File: Elasticsearch6TableSink.java    From alchemy with Apache License 2.0 4 votes vote down vote up 
private SinkFunction<Tuple2<Boolean, Row>> createSinkFunction(List<ElasticsearchUpsertTableSinkBase.Host> hosts, ActionRequestFailureHandler failureHandler, Elasticsearch6SinkFunction upsertFunction) {
    final List<HttpHost> httpHosts = hosts.stream()
            .map((host) -> new HttpHost(host.hostname, host.port, host.protocol))
            .collect(Collectors.toList());

    final ElasticsearchSink.Builder<Tuple2<Boolean, Row>> builder = new ElasticsearchSink.Builder(httpHosts, upsertFunction);

    builder.setFailureHandler(failureHandler);
    final Map<String, String> bulkConfig = elasticsearch6Properties.getConfig();
    if (bulkConfig != null) {
        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_MAX_ACTIONS))
                .ifPresent(v -> builder.setBulkFlushMaxActions(Integer.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_MAX_SIZE))
                .ifPresent(v -> builder.setBulkFlushMaxSizeMb(MemorySize.parse(v).getMebiBytes()));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_INTERVAL))
                .ifPresent(v -> builder.setBulkFlushInterval(Long.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF))
                .ifPresent(v -> builder.setBulkFlushBackoff(Boolean.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_TYPE))
                .ifPresent(v -> builder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_RETRIES))
                .ifPresent(v -> builder.setBulkFlushBackoffRetries(Integer.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_DELAY))
                .ifPresent(v -> builder.setBulkFlushBackoffDelay(Long.valueOf(v)));
    }
    builder.setRestClientFactory(
            new DefaultRestClientFactory(
                    elasticsearch6Properties.getMaxRetryTimeoutMills(),
                    elasticsearch6Properties.getPathPrefix()));

    final ElasticsearchSink<Tuple2<Boolean, Row>> sink = builder.build();

    Optional.ofNullable(elasticsearch6Properties.getDisableFlushOnCheckpoint())
            .ifPresent(v -> {
                if (Boolean.valueOf(v)) {
                    sink.disableFlushOnCheckpoint();
                }
            });

    return sink;
}
Example #4
Source File: Elasticsearch6DynamicSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
public Elasticsearch6DynamicSink(
		EncodingFormat<SerializationSchema<RowData>> format,
		Elasticsearch6Configuration config,
		TableSchema schema) {
	this(format, config, schema, (ElasticsearchSink.Builder::new));
}
Example #5
Source File: Elasticsearch6DynamicSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
ElasticsearchSink.Builder<RowData> createBuilder(
List<HttpHost> httpHosts,
RowElasticsearchSinkFunction upsertSinkFunction);
Example #6
Source File: Elasticsearch6DynamicSink.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public SinkFunctionProvider getSinkRuntimeProvider(Context context) {
	return () -> {
		SerializationSchema<RowData> format = this.format.createRuntimeEncoder(context, schema.toRowDataType());

		final RowElasticsearchSinkFunction upsertFunction =
			new RowElasticsearchSinkFunction(
				IndexGeneratorFactory.createIndexGenerator(config.getIndex(), schema),
				config.getDocumentType(),
				format,
				XContentType.JSON,
				REQUEST_FACTORY,
				KeyExtractor.createKeyExtractor(schema, config.getKeyDelimiter())
			);

		final ElasticsearchSink.Builder<RowData> builder = builderProvider.createBuilder(
			config.getHosts(),
			upsertFunction);

		builder.setFailureHandler(config.getFailureHandler());
		builder.setBulkFlushMaxActions(config.getBulkFlushMaxActions());
		builder.setBulkFlushMaxSizeMb((int) (config.getBulkFlushMaxByteSize() >> 20));
		builder.setBulkFlushInterval(config.getBulkFlushInterval());
		builder.setBulkFlushBackoff(config.isBulkFlushBackoffEnabled());
		config.getBulkFlushBackoffType().ifPresent(builder::setBulkFlushBackoffType);
		config.getBulkFlushBackoffRetries().ifPresent(builder::setBulkFlushBackoffRetries);
		config.getBulkFlushBackoffDelay().ifPresent(builder::setBulkFlushBackoffDelay);

		// we must overwrite the default factory which is defined with a lambda because of a bug
		// in shading lambda serialization shading see FLINK-18006
		builder.setRestClientFactory(new DefaultRestClientFactory(config.getPathPrefix().orElse(null)));

		final ElasticsearchSink<RowData> sink = builder.build();

		if (config.isDisableFlushOnCheckpoint()) {
			sink.disableFlushOnCheckpoint();
		}

		return sink;
	};
}
Example #7
Source File: ESSinkUtil.java    From flink-learning with Apache License 2.0 3 votes vote down vote up 
/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}
Example #8
Source File: ESSinkUtil.java    From flink-learning with Apache License 2.0 3 votes vote down vote up 
/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}
Example #9
Source File: ESSinkUtil.java    From flink-learning with Apache License 2.0 3 votes vote down vote up 
/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}
Example #10
Source File: ESSinkUtil.java    From flink-learning with Apache License 2.0 3 votes vote down vote up 
/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}