org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ESSink.java From Mastering-Distributed-Tracing with MIT License

6 votes

public static ElasticsearchSink<TraceSummary> build() {
    List<HttpHost> httpHosts = new ArrayList<>();
    httpHosts.add(new HttpHost("127.0.0.1", 9200, "http"));

    ElasticsearchSink.Builder<TraceSummary> esSinkBuilder = new ElasticsearchSink.Builder<>(httpHosts,
            new ElasticsearchSinkFunction<TraceSummary>() {

                @Override
                public void process(TraceSummary summary, RuntimeContext ctx, RequestIndexer indexer) {
                    indexer.add(Requests.indexRequest()//
                        .index("trace-summaries") //
                        .type("trace-summaries") //
                        .id(summary.traceId) //
                        .source(asJson(summary)));
                }
            });

    // configuration for the bulk requests; this instructs the sink to emit after
    // every element, otherwise they would be buffered
    esSinkBuilder.setBulkFlushMaxActions(1);

    return esSinkBuilder.build();
}

Example #2

Source File: Elasticsearch6DynamicSinkTest.java From flink with Apache License 2.0

5 votes

@Override
public ElasticsearchSink.Builder<RowData> createBuilder(
		List<HttpHost> httpHosts,
		RowElasticsearchSinkFunction upsertSinkFunction) {
	builderSpy = Mockito.spy(new ElasticsearchSink.Builder<>(httpHosts, upsertSinkFunction));
	doAnswer(
		invocation -> {
			sinkSpy = Mockito.spy((ElasticsearchSink<RowData>) invocation.callRealMethod());
			return sinkSpy;
		}
	).when(builderSpy).build();

	return builderSpy;
}

Example #3

Source File: Elasticsearch6TableSink.java From alchemy with Apache License 2.0

4 votes

private SinkFunction<Tuple2<Boolean, Row>> createSinkFunction(List<ElasticsearchUpsertTableSinkBase.Host> hosts, ActionRequestFailureHandler failureHandler, Elasticsearch6SinkFunction upsertFunction) {
    final List<HttpHost> httpHosts = hosts.stream()
            .map((host) -> new HttpHost(host.hostname, host.port, host.protocol))
            .collect(Collectors.toList());

    final ElasticsearchSink.Builder<Tuple2<Boolean, Row>> builder = new ElasticsearchSink.Builder(httpHosts, upsertFunction);

    builder.setFailureHandler(failureHandler);
    final Map<String, String> bulkConfig = elasticsearch6Properties.getConfig();
    if (bulkConfig != null) {
        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_MAX_ACTIONS))
                .ifPresent(v -> builder.setBulkFlushMaxActions(Integer.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_MAX_SIZE))
                .ifPresent(v -> builder.setBulkFlushMaxSizeMb(MemorySize.parse(v).getMebiBytes()));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_INTERVAL))
                .ifPresent(v -> builder.setBulkFlushInterval(Long.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF))
                .ifPresent(v -> builder.setBulkFlushBackoff(Boolean.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_TYPE))
                .ifPresent(v -> builder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_RETRIES))
                .ifPresent(v -> builder.setBulkFlushBackoffRetries(Integer.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_DELAY))
                .ifPresent(v -> builder.setBulkFlushBackoffDelay(Long.valueOf(v)));
    }
    builder.setRestClientFactory(
            new DefaultRestClientFactory(
                    elasticsearch6Properties.getMaxRetryTimeoutMills(),
                    elasticsearch6Properties.getPathPrefix()));

    final ElasticsearchSink<Tuple2<Boolean, Row>> sink = builder.build();

    Optional.ofNullable(elasticsearch6Properties.getDisableFlushOnCheckpoint())
            .ifPresent(v -> {
                if (Boolean.valueOf(v)) {
                    sink.disableFlushOnCheckpoint();
                }
            });

    return sink;
}

Example #4

Source File: Elasticsearch6DynamicSink.java From flink with Apache License 2.0

4 votes

public Elasticsearch6DynamicSink(
		EncodingFormat<SerializationSchema<RowData>> format,
		Elasticsearch6Configuration config,
		TableSchema schema) {
	this(format, config, schema, (ElasticsearchSink.Builder::new));
}

Example #5

Source File: Elasticsearch6DynamicSink.java From flink with Apache License 2.0

4 votes

ElasticsearchSink.Builder<RowData> createBuilder(
List<HttpHost> httpHosts,
RowElasticsearchSinkFunction upsertSinkFunction);

Example #6

Source File: Elasticsearch6DynamicSink.java From flink with Apache License 2.0

4 votes

@Override
public SinkFunctionProvider getSinkRuntimeProvider(Context context) {
	return () -> {
		SerializationSchema<RowData> format = this.format.createRuntimeEncoder(context, schema.toRowDataType());

		final RowElasticsearchSinkFunction upsertFunction =
			new RowElasticsearchSinkFunction(
				IndexGeneratorFactory.createIndexGenerator(config.getIndex(), schema),
				config.getDocumentType(),
				format,
				XContentType.JSON,
				REQUEST_FACTORY,
				KeyExtractor.createKeyExtractor(schema, config.getKeyDelimiter())
			);

		final ElasticsearchSink.Builder<RowData> builder = builderProvider.createBuilder(
			config.getHosts(),
			upsertFunction);

		builder.setFailureHandler(config.getFailureHandler());
		builder.setBulkFlushMaxActions(config.getBulkFlushMaxActions());
		builder.setBulkFlushMaxSizeMb((int) (config.getBulkFlushMaxByteSize() >> 20));
		builder.setBulkFlushInterval(config.getBulkFlushInterval());
		builder.setBulkFlushBackoff(config.isBulkFlushBackoffEnabled());
		config.getBulkFlushBackoffType().ifPresent(builder::setBulkFlushBackoffType);
		config.getBulkFlushBackoffRetries().ifPresent(builder::setBulkFlushBackoffRetries);
		config.getBulkFlushBackoffDelay().ifPresent(builder::setBulkFlushBackoffDelay);

		// we must overwrite the default factory which is defined with a lambda because of a bug
		// in shading lambda serialization shading see FLINK-18006
		builder.setRestClientFactory(new DefaultRestClientFactory(config.getPathPrefix().orElse(null)));

		final ElasticsearchSink<RowData> sink = builder.build();

		if (config.isDisableFlushOnCheckpoint()) {
			sink.disableFlushOnCheckpoint();
		}

		return sink;
	};
}

Example #7

Source File: ESSinkUtil.java From flink-learning with Apache License 2.0

3 votes

/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}

Example #8

Source File: ESSinkUtil.java From flink-learning with Apache License 2.0

3 votes

/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}

Example #9

Source File: ESSinkUtil.java From flink-learning with Apache License 2.0

3 votes

/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}

Example #10

Source File: ESSinkUtil.java From flink-learning with Apache License 2.0

3 votes

/**
 * es sink
 *
 * @param hosts               es hosts
 * @param bulkFlushMaxActions bulk flush size
 * @param parallelism         并行数
 * @param data                数据
 * @param func
 * @param <T>
 */
public static <T> void addSink(List<HttpHost> hosts, int bulkFlushMaxActions, int parallelism,
                               SingleOutputStreamOperator<T> data, ElasticsearchSinkFunction<T> func,
                               ParameterTool parameterTool) {
    ElasticsearchSink.Builder<T> esSinkBuilder = new ElasticsearchSink.Builder<>(hosts, func);
    esSinkBuilder.setBulkFlushMaxActions(bulkFlushMaxActions);
    esSinkBuilder.setFailureHandler(new RetryRequestFailureHandler());
    //todo:xpack security
    data.addSink(esSinkBuilder.build()).setParallelism(parallelism);
}