Java Code Examples for org.apache.flink.api.java.utils.ParameterTool#getBoolean()

private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
	final String stateBackend = pt.get(
		STATE_BACKEND.key(),
		STATE_BACKEND.defaultValue());

	final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());

	if ("file".equalsIgnoreCase(stateBackend)) {
		boolean asyncCheckpoints = pt.getBoolean(
			STATE_BACKEND_FILE_ASYNC.key(),
			STATE_BACKEND_FILE_ASYNC.defaultValue());

		env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
	} else if ("rocks".equalsIgnoreCase(stateBackend)) {
		boolean incrementalCheckpoints = pt.getBoolean(
			STATE_BACKEND_ROCKS_INCREMENTAL.key(),
			STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());

		env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
	} else {
		throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
	}
}
 

private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
	final String stateBackend = pt.get(
		STATE_BACKEND.key(),
		STATE_BACKEND.defaultValue());

	final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());

	if ("file".equalsIgnoreCase(stateBackend)) {
		boolean asyncCheckpoints = pt.getBoolean(
			STATE_BACKEND_FILE_ASYNC.key(),
			STATE_BACKEND_FILE_ASYNC.defaultValue());

		env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
	} else if ("rocks".equalsIgnoreCase(stateBackend)) {
		boolean incrementalCheckpoints = pt.getBoolean(
			STATE_BACKEND_ROCKS_INCREMENTAL.key(),
			STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());

		env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
	} else {
		throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
	}
}
 

private static void setupCheckpointing(final StreamExecutionEnvironment env, final ParameterTool pt) {
	String semantics = pt.get(TEST_SEMANTICS.key(), TEST_SEMANTICS.defaultValue());
	long checkpointInterval = pt.getLong(ENVIRONMENT_CHECKPOINT_INTERVAL.key(), ENVIRONMENT_CHECKPOINT_INTERVAL.defaultValue());
	CheckpointingMode checkpointingMode = semantics.equalsIgnoreCase("exactly-once")
		? CheckpointingMode.EXACTLY_ONCE
		: CheckpointingMode.AT_LEAST_ONCE;

	env.enableCheckpointing(checkpointInterval, checkpointingMode);

	boolean enableExternalizedCheckpoints = pt.getBoolean(
		ENVIRONMENT_EXTERNALIZE_CHECKPOINT.key(),
		ENVIRONMENT_EXTERNALIZE_CHECKPOINT.defaultValue());

	if (enableExternalizedCheckpoints) {
		String cleanupModeConfig = pt.get(
			ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.key(),
			ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.defaultValue());

		CheckpointConfig.ExternalizedCheckpointCleanup cleanupMode;
		switch (cleanupModeConfig) {
			case "retain":
				cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION;
				break;
			case "delete":
				cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION;
				break;
			default:
				throw new IllegalArgumentException("Unknown clean up mode for externalized checkpoints: " + cleanupModeConfig);
		}
		env.getCheckpointConfig().enableExternalizedCheckpoints(cleanupMode);

		final int tolerableDeclinedCheckpointNumber = pt.getInt(
			ENVIRONMENT_TOLERABLE_DECLINED_CHECKPOINT_NUMBER.key(),
			ENVIRONMENT_TOLERABLE_DECLINED_CHECKPOINT_NUMBER.defaultValue());
		env.getCheckpointConfig().setTolerableCheckpointFailureNumber(tolerableDeclinedCheckpointNumber);
	}
}
 

public static StreamExecutionEnvironment prepare(ParameterTool parameterTool) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(parameterTool.getInt(PropertiesConstants.STREAM_PARALLELISM, 5));
    env.getConfig().disableSysoutLogging();
    env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 60000));
    if (parameterTool.getBoolean(PropertiesConstants.STREAM_CHECKPOINT_ENABLE, true)) {
        env.enableCheckpointing(parameterTool.getLong(PropertiesConstants.STREAM_CHECKPOINT_INTERVAL, 10000));
    }
    env.getConfig().setGlobalJobParameters(parameterTool);
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    return env;
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final boolean useValueState = pt.getBoolean("useValueState", false);
	final boolean useListState = pt.getBoolean("useListState", false);
	final boolean useMapState = pt.getBoolean("useMapState", false);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);
	KeyedStream<Event, Integer> keyedStream = env
		.addSource(DataStreamAllroundTestJobFactory.createEventSource(pt))
		.name(EVENT_SOURCE.getName())
		.uid(EVENT_SOURCE.getUid())
		.assignTimestampsAndWatermarks(createTimestampExtractor(pt))
		.keyBy(Event::getKey);

	keyedStream.map(new ValueStateMapper(useValueState)).name("ValueStateMapper").uid("ValueStateMapper");
	keyedStream.map(new ListStateMapper(useListState)).name("ListStateMapper").uid("ListStateMapper");
	keyedStream.map(new MapStateMapper(useMapState)).name("MapStateMapper").uid("MapStateMapper");

	boolean useWindow = pt.getBoolean("useWindow", false);
	if (useWindow) {
		applyTumblingWindows(keyedStream, pt)
			.apply(new WindowFunction<Event, Event, Integer, TimeWindow>() {
				@Override
				public void apply(Integer integer, TimeWindow window, Iterable<Event> input, Collector<Event> out) {
					for (Event e : input) {
						out.collect(e);
					}
				}
			})
			.name(TIME_WINDOW_OPER.getName())
			.uid(TIME_WINDOW_OPER.getUid());
	}

	env.execute("RocksDB test job");
}
 

public SummarizationJobParameters(ParameterTool params) {
    timelyHostname = params.getRequired("timelyHostname");
    timelyTcpPort = params.getInt("timelyTcpPort", 4241);
    timelyHttpsPort = params.getInt("timelyHttpsPort", 4242);
    timelyWssPort = params.getInt("timelyWssPort", 4243);
    clientAuth = params.getBoolean("clientAuth", false);
    doLogin = params.getBoolean("doLogin", false);
    timelyUsername = params.get("timelyUsername", null);
    timelyPassword = params.get("timelyPassword", null);
    keyStoreFile = params.getRequired("keyStoreFile");
    keyStoreType = params.get("keyStoreType", "JKS");
    keyStorePass = params.getRequired("keyStorePass");
    trustStoreFile = params.getRequired("trustStoreFile");
    trustStoreType = params.get("trustStoreType", "JKS");
    trustStorePass = params.getRequired("trustStorePass");
    hostVerificationEnabled = params.getBoolean("hostVerificationEnabled", true);
    bufferSize = params.getInt("bufferSize", 10485760);
    String metricNames = params.getRequired("metrics");
    if (null != metricNames) {
        metrics = metricNames.split(",");
    } else {
        metrics = null;
    }
    startTime = params.getLong("startTime", 0L);
    endTime = params.getLong("endTime", 0L);
    interval = params.getRequired("interval");
    intervalUnits = params.getRequired("intervalUnits");
}
 

public AbstractHandler(String ... args) {
    ParameterTool params = ParameterTool.fromArgs(args);
    this.scope = params.get("scope", DEFAULT_SCOPE);
    this.stream = params.get("stream", DEFAULT_STREAM);
    this.controllerUri = params.get("controllerUri", DEFAULT_CONTROLLER_URI);
    this.create = params.getBoolean("create-stream", CREATE_STREAM);
    this.limit = params.getInt("threshold", DEFAULT_POPULAR_DEST_THRESHOLD);
}
 

public static StreamExecutionEnvironment prepare(ParameterTool parameterTool) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(parameterTool.getInt(PropertiesConstants.STREAM_PARALLELISM, 5));
    env.getConfig().disableSysoutLogging();
    env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 60000));
    if (parameterTool.getBoolean(PropertiesConstants.STREAM_CHECKPOINT_ENABLE, true)) {
        env.enableCheckpointing(parameterTool.getLong(PropertiesConstants.STREAM_CHECKPOINT_INTERVAL, 10000));
    }
    env.getConfig().setGlobalJobParameters(parameterTool);
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    return env;
}
 

public static void main(String[] args) throws Exception {
	if (args.length != 1) {
		throw new RuntimeException("Path to the properties file is expected as the only argument.");
	}
	ParameterTool params = ParameterTool.fromPropertiesFile(args[0]);

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<ItemTransaction> generatedInput =
			env.addSource(new ItemTransactionGeneratorSource(params))
					.name("Item Transaction Generator");

	FlinkKafkaProducer<ItemTransaction> kafkaSink = new FlinkKafkaProducer<>(
			params.getRequired(KafkaItemTransactionJob.TRANSACTION_INPUT_TOPIC_KEY),
			new TransactionSchema(),
			Utils.readKafkaProperties(params, false),
			Optional.empty());

	generatedInput.keyBy("itemId").addSink(kafkaSink).name("Transaction Kafka Sink");

	if (params.getBoolean(GENERATE_QUERIES, false)) {
		DataStream<Query> queries = env.addSource(new QueryGeneratorSource(params))
				.name("Query Generator");

		FlinkKafkaProducer<Query> querySink = new FlinkKafkaProducer<>(
				params.getRequired(KafkaItemTransactionJob.QUERY_INPUT_TOPIC_KEY),
				new QuerySchema(),
				Utils.readKafkaProperties(params, false),
				Optional.empty());

		queries.keyBy("itemId").addSink(querySink).name("Query Kafka Sink");
	}

	env.execute("Kafka Data generator");
}
 

public static void main(String[] args) throws Exception {
	ParameterTool params = ParameterTool.fromArgs(args);
	String sourceTablePath = params.getRequired("sourceTablePath");
	String queryPath = params.getRequired("queryPath");
	String sinkTablePath = params.getRequired("sinkTablePath");
	Boolean useTableStats = params.getBoolean("useTableStats");
	TableEnvironment tableEnvironment = prepareTableEnv(sourceTablePath, useTableStats);

	//execute TPC-DS queries
	for (String queryId : TPCDS_QUERIES) {
		System.out.println("[INFO]Run TPC-DS query " + queryId + " ...");
		String queryName = QUERY_PREFIX + queryId + QUERY_SUFFIX;
		String queryFilePath = queryPath + FILE_SEPARATOR + queryName;
		String queryString = loadFile2String(queryFilePath);
		Table resultTable = tableEnvironment.sqlQuery(queryString);

		//register sink table
		String sinkTableName = QUERY_PREFIX + queryId + "_sinkTable";
		((TableEnvironmentInternal) tableEnvironment).registerTableSinkInternal(sinkTableName,
				new CsvTableSink(
					sinkTablePath + FILE_SEPARATOR + queryId + RESULT_SUFFIX,
					COL_DELIMITER,
					1,
					FileSystem.WriteMode.OVERWRITE,
					resultTable.getSchema().getFieldNames(),
					resultTable.getSchema().getFieldDataTypes()
				));
		TableResult tableResult = resultTable.executeInsert(sinkTableName);
		// wait job finish
		tableResult.getJobClient().get()
				.getJobExecutionResult(Thread.currentThread().getContextClassLoader())
				.get();
		System.out.println("[INFO]Run TPC-DS query " + queryId + " success.");
	}
}
 

static boolean isSimulateFailures(ParameterTool pt) {
	return pt.getBoolean(TEST_SIMULATE_FAILURE.key(), TEST_SIMULATE_FAILURE.defaultValue());
}
 

public ElasticsearchSinkBase(
	ElasticsearchApiCallBridge<C> callBridge,
	Map<String, String> userConfig,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler) {

	this.callBridge = checkNotNull(callBridge);
	this.elasticsearchSinkFunction = checkNotNull(elasticsearchSinkFunction);
	this.failureHandler = checkNotNull(failureHandler);

	// we eagerly check if the user-provided sink function and failure handler is serializable;
	// otherwise, if they aren't serializable, users will merely get a non-informative error message
	// "ElasticsearchSinkBase is not serializable"

	checkArgument(InstantiationUtil.isSerializable(elasticsearchSinkFunction),
		"The implementation of the provided ElasticsearchSinkFunction is not serializable. " +
			"The object probably contains or references non-serializable fields.");

	checkArgument(InstantiationUtil.isSerializable(failureHandler),
		"The implementation of the provided ActionRequestFailureHandler is not serializable. " +
			"The object probably contains or references non-serializable fields.");

	// extract and remove bulk processor related configuration from the user-provided config,
	// so that the resulting user config only contains configuration related to the Elasticsearch client.

	checkNotNull(userConfig);

	// copy config so we can remove entries without side-effects
	userConfig = new HashMap<>(userConfig);

	ParameterTool params = ParameterTool.fromMap(userConfig);

	if (params.has(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS)) {
		bulkProcessorFlushMaxActions = params.getInt(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS);
	} else {
		bulkProcessorFlushMaxActions = null;
	}

	if (params.has(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB)) {
		bulkProcessorFlushMaxSizeMb = params.getInt(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB);
	} else {
		bulkProcessorFlushMaxSizeMb = null;
	}

	if (params.has(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS)) {
		bulkProcessorFlushIntervalMillis = params.getLong(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS);
	} else {
		bulkProcessorFlushIntervalMillis = null;
	}

	boolean bulkProcessorFlushBackoffEnable = params.getBoolean(CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, true);
	userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE);

	if (bulkProcessorFlushBackoffEnable) {
		this.bulkProcessorFlushBackoffPolicy = new BulkFlushBackoffPolicy();

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE)) {
			bulkProcessorFlushBackoffPolicy.setBackoffType(FlushBackoffType.valueOf(params.get(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE)));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE);
		}

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES)) {
			bulkProcessorFlushBackoffPolicy.setMaxRetryCount(params.getInt(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES);
		}

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY)) {
			bulkProcessorFlushBackoffPolicy.setDelayMillis(params.getLong(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY);
		}

	} else {
		bulkProcessorFlushBackoffPolicy = null;
	}

	this.userConfig = userConfig;
}
 

public final StreamExecutionEnvironment createApplicationPipeline(ParameterTool params) throws Exception {

		// Create and configure the StreamExecutionEnvironment
		StreamExecutionEnvironment env = createExecutionEnvironment(params);

		// Read transaction stream
		DataStream<ItemTransaction> transactionStream = readTransactionStream(params, env);

		// We read the query stream and exclude it from watermark tracking by assigning Long.MAX_VALUE watermark
		DataStream<Query> queryStream = readQueryStream(params, env)
				.assignTimestampsAndWatermarks(new MaxWatermark<>())
				.name("MaxWatermark");

		// Connect transactions with queries using the same itemId key and apply our transaction processor
		// The main output is the transaction result, query results are accessed as a side output.
		SingleOutputStreamOperator<TransactionResult> processedTransactions = transactionStream.keyBy("itemId")
				.connect(queryStream.keyBy("itemId"))
				.process(new TransactionProcessor())
				.name("Transaction Processor")
				.uid("Transaction Processor");

		// Query results are accessed as a sideoutput of the transaction processor
		DataStream<QueryResult> queryResultStream = processedTransactions.getSideOutput(QUERY_RESULT);

		if (params.getBoolean(ENABLE_DB_ENRICHMENT, false)) {
			queryResultStream = AsyncDataStream.unorderedWait(
					queryResultStream,
					new ItemInfoEnrichment(params.getInt(ASYNC_TP_SIZE, 5), params.getRequired(DB_CONN_STRING)),
					10, TimeUnit.SECONDS
			);
		}

		// Handle the output of transaction and query results separately
		writeTransactionResults(params, processedTransactions);
		writeQueryOutput(params, queryResultStream);

		// If needed we create a window computation of the transaction summaries by item and time window
		if (params.getBoolean(ENABLE_SUMMARIES_KEY, false)) {
			DataStream<TransactionSummary> transactionSummaryStream = processedTransactions
					.keyBy("transaction.itemId")
					.timeWindow(Time.minutes(10))
					.aggregate(new TransactionSummaryAggregator())
					.name("Create Transaction Summary")
					.uid("Create Transaction Summary")
					.filter(new SummaryAlertingCondition(params))
					.name("Filter High failure rate");

			writeTransactionSummaries(params, transactionSummaryStream);
		}

		return env;
	}
 

public static void main(String[] args) throws Exception {

		final ParameterTool pt = ParameterTool.fromArgs(args);

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		env.setParallelism(pt.getInt("parallelism", 1));
		env.setMaxParallelism(pt.getInt("maxParallelism", pt.getInt("parallelism", 1)));
		env.enableCheckpointing(pt.getInt("checkpointInterval", 1000));
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, pt.getInt("restartDelay", 0)));
		if (pt.getBoolean("externalizedCheckpoints", false)) {
			env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
		}

		String stateBackend = pt.get("stateBackend", "file");
		String checkpointDir = pt.getRequired("checkpointDir");

		boolean killJvmOnFail = pt.getBoolean("killJvmOnFail", false);

		if ("file".equals(stateBackend)) {
			boolean asyncCheckpoints = pt.getBoolean("asyncCheckpoints", true);
			env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints));
		} else if ("rocks".equals(stateBackend)) {
			boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false);
			env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
		} else {
			throw new IllegalArgumentException("Unknown backend: " + stateBackend);
		}

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(pt);

		// delay to throttle down the production of the source
		long delay = pt.getLong("delay", 0L);

		// the maximum number of attempts, before the job finishes with success
		int maxAttempts = pt.getInt("maxAttempts", 3);

		// size of one artificial value
		int valueSize = pt.getInt("valueSize", 10);

		env.addSource(new RandomLongSource(maxAttempts, delay))
			.keyBy((KeySelector<Long, Long>) aLong -> aLong)
			.flatMap(new StateCreatingFlatMap(valueSize, killJvmOnFail))
			.addSink(new PrintSinkFunction<>());

		env.execute("Sticky Allocation And Local Recovery Test");
	}
 

static boolean isSimulateFailures(ParameterTool pt) {
	return pt.getBoolean(TEST_SIMULATE_FAILURE.key(), TEST_SIMULATE_FAILURE.defaultValue());
}
 

public static void main(String[] args) throws Exception {

		final ParameterTool pt = ParameterTool.fromArgs(args);

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		env.setParallelism(pt.getInt("parallelism", 1));
		env.setMaxParallelism(pt.getInt("maxParallelism", pt.getInt("parallelism", 1)));
		env.enableCheckpointing(pt.getInt("checkpointInterval", 1000));
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, pt.getInt("restartDelay", 0)));
		if (pt.getBoolean("externalizedCheckpoints", false)) {
			env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
		}

		String stateBackend = pt.get("stateBackend", "file");
		String checkpointDir = pt.getRequired("checkpointDir");

		boolean killJvmOnFail = pt.getBoolean("killJvmOnFail", false);

		if ("file".equals(stateBackend)) {
			boolean asyncCheckpoints = pt.getBoolean("asyncCheckpoints", true);
			env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints));
		} else if ("rocks".equals(stateBackend)) {
			boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false);
			env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
		} else {
			throw new IllegalArgumentException("Unknown backend: " + stateBackend);
		}

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(pt);

		// delay to throttle down the production of the source
		long delay = pt.getLong("delay", 0L);

		// the maximum number of attempts, before the job finishes with success
		int maxAttempts = pt.getInt("maxAttempts", 3);

		// size of one artificial value
		int valueSize = pt.getInt("valueSize", 10);

		env.addSource(new RandomLongSource(maxAttempts, delay))
			.keyBy((KeySelector<Long, Long>) aLong -> aLong)
			.flatMap(new StateCreatingFlatMap(valueSize, killJvmOnFail))
			.addSink(new PrintSinkFunction<>());

		env.execute("Sticky Allocation And Local Recovery Test");
	}
 

public ElasticsearchSinkBase(
	ElasticsearchApiCallBridge<C> callBridge,
	Map<String, String> userConfig,
	ElasticsearchSinkFunction<T> elasticsearchSinkFunction,
	ActionRequestFailureHandler failureHandler) {

	this.callBridge = checkNotNull(callBridge);
	this.elasticsearchSinkFunction = checkNotNull(elasticsearchSinkFunction);
	this.failureHandler = checkNotNull(failureHandler);

	// we eagerly check if the user-provided sink function and failure handler is serializable;
	// otherwise, if they aren't serializable, users will merely get a non-informative error message
	// "ElasticsearchSinkBase is not serializable"

	checkArgument(InstantiationUtil.isSerializable(elasticsearchSinkFunction),
		"The implementation of the provided ElasticsearchSinkFunction is not serializable. " +
			"The object probably contains or references non-serializable fields.");

	checkArgument(InstantiationUtil.isSerializable(failureHandler),
		"The implementation of the provided ActionRequestFailureHandler is not serializable. " +
			"The object probably contains or references non-serializable fields.");

	// extract and remove bulk processor related configuration from the user-provided config,
	// so that the resulting user config only contains configuration related to the Elasticsearch client.

	checkNotNull(userConfig);

	// copy config so we can remove entries without side-effects
	userConfig = new HashMap<>(userConfig);

	ParameterTool params = ParameterTool.fromMap(userConfig);

	if (params.has(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS)) {
		bulkProcessorFlushMaxActions = params.getInt(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS);
	} else {
		bulkProcessorFlushMaxActions = null;
	}

	if (params.has(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB)) {
		bulkProcessorFlushMaxSizeMb = params.getInt(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB);
	} else {
		bulkProcessorFlushMaxSizeMb = null;
	}

	if (params.has(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS)) {
		bulkProcessorFlushIntervalMillis = params.getLong(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS);
		userConfig.remove(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS);
	} else {
		bulkProcessorFlushIntervalMillis = null;
	}

	boolean bulkProcessorFlushBackoffEnable = params.getBoolean(CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, true);
	userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE);

	if (bulkProcessorFlushBackoffEnable) {
		this.bulkProcessorFlushBackoffPolicy = new BulkFlushBackoffPolicy();

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE)) {
			bulkProcessorFlushBackoffPolicy.setBackoffType(FlushBackoffType.valueOf(params.get(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE)));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE);
		}

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES)) {
			bulkProcessorFlushBackoffPolicy.setMaxRetryCount(params.getInt(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES);
		}

		if (params.has(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY)) {
			bulkProcessorFlushBackoffPolicy.setDelayMillis(params.getLong(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY));
			userConfig.remove(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY);
		}

	} else {
		bulkProcessorFlushBackoffPolicy = null;
	}

	this.userConfig = userConfig;
}
 

public static void main(String[] args) throws Exception {

		final ParameterTool pt = ParameterTool.fromArgs(args);

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		env.setParallelism(pt.getInt("parallelism", 1));
		env.setMaxParallelism(pt.getInt("maxParallelism", pt.getInt("parallelism", 1)));
		env.enableCheckpointing(pt.getInt("checkpointInterval", 1000));
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, pt.getInt("restartDelay", 0)));
		if (pt.getBoolean("externalizedCheckpoints", false)) {
			env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
		}

		String stateBackend = pt.get("stateBackend", "file");
		String checkpointDir = pt.getRequired("checkpointDir");

		boolean killJvmOnFail = pt.getBoolean("killJvmOnFail", false);

		if ("file".equals(stateBackend)) {
			boolean asyncCheckpoints = pt.getBoolean("asyncCheckpoints", true);
			env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints));
		} else if ("rocks".equals(stateBackend)) {
			boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false);
			env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
		} else {
			throw new IllegalArgumentException("Unknown backend: " + stateBackend);
		}

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(pt);

		// delay to throttle down the production of the source
		long delay = pt.getLong("delay", 0L);

		// the maximum number of attempts, before the job finishes with success
		int maxAttempts = pt.getInt("maxAttempts", 3);

		// size of one artificial value
		int valueSize = pt.getInt("valueSize", 10);

		env.addSource(new RandomLongSource(maxAttempts, delay))
			.keyBy((KeySelector<Long, Long>) aLong -> aLong)
			.flatMap(new StateCreatingFlatMap(valueSize, killJvmOnFail))
			.addSink(new PrintSinkFunction<>());

		env.execute("Sticky Allocation And Local Recovery Test");
	}
 

static boolean isSimulateFailures(ParameterTool pt) {
	return pt.getBoolean(TEST_SIMULATE_FAILURE.key(), TEST_SIMULATE_FAILURE.defaultValue());
}
 

public static void setupEnvironment(StreamExecutionEnvironment env, ParameterTool pt) throws Exception {

		// set checkpointing semantics
		String semantics = pt.get(TEST_SEMANTICS.key(), TEST_SEMANTICS.defaultValue());
		long checkpointInterval = pt.getLong(ENVIRONMENT_CHECKPOINT_INTERVAL.key(), ENVIRONMENT_CHECKPOINT_INTERVAL.defaultValue());
		CheckpointingMode checkpointingMode = semantics.equalsIgnoreCase("exactly-once")
			? CheckpointingMode.EXACTLY_ONCE
			: CheckpointingMode.AT_LEAST_ONCE;

		env.enableCheckpointing(checkpointInterval, checkpointingMode);

		// use event time
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		// parallelism
		env.setParallelism(pt.getInt(ENVIRONMENT_PARALLELISM.key(), ENVIRONMENT_PARALLELISM.defaultValue()));
		env.setMaxParallelism(pt.getInt(ENVIRONMENT_MAX_PARALLELISM.key(), ENVIRONMENT_MAX_PARALLELISM.defaultValue()));

		// restart strategy
		String restartStrategyConfig = pt.get(ENVIRONMENT_RESTART_STRATEGY.key());
		if (restartStrategyConfig != null) {
			RestartStrategies.RestartStrategyConfiguration restartStrategy;
			switch (restartStrategyConfig) {
				case "fixed_delay":
					restartStrategy = RestartStrategies.fixedDelayRestart(
						pt.getInt(
							ENVIRONMENT_RESTART_STRATEGY_FIXED_ATTEMPTS.key(),
							ENVIRONMENT_RESTART_STRATEGY_FIXED_ATTEMPTS.defaultValue()),
						pt.getLong(
							ENVIRONMENT_RESTART_STRATEGY_FIXED_DELAY.key(),
							ENVIRONMENT_RESTART_STRATEGY_FIXED_DELAY.defaultValue()));
					break;
				case "no_restart":
					restartStrategy = RestartStrategies.noRestart();
					break;
				default:
					throw new IllegalArgumentException("Unkown restart strategy: " + restartStrategyConfig);
			}
			env.setRestartStrategy(restartStrategy);
		}

		// state backend
		final String stateBackend = pt.get(
			STATE_BACKEND.key(),
			STATE_BACKEND.defaultValue());

		final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());

		if ("file".equalsIgnoreCase(stateBackend)) {
			boolean asyncCheckpoints = pt.getBoolean(
				STATE_BACKEND_FILE_ASYNC.key(),
				STATE_BACKEND_FILE_ASYNC.defaultValue());

			env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
		} else if ("rocks".equalsIgnoreCase(stateBackend)) {
			boolean incrementalCheckpoints = pt.getBoolean(
				STATE_BACKEND_ROCKS_INCREMENTAL.key(),
				STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());

			env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
		} else {
			throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
		}

		boolean enableExternalizedCheckpoints = pt.getBoolean(
			ENVIRONMENT_EXTERNALIZE_CHECKPOINT.key(),
			ENVIRONMENT_EXTERNALIZE_CHECKPOINT.defaultValue());

		if (enableExternalizedCheckpoints) {
			String cleanupModeConfig = pt.get(
				ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.key(),
				ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.defaultValue());

			CheckpointConfig.ExternalizedCheckpointCleanup cleanupMode;
			switch (cleanupModeConfig) {
				case "retain":
					cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION;
					break;
				case "delete":
					cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION;
					break;
				default:
					throw new IllegalArgumentException("Unknown clean up mode for externalized checkpoints: " + cleanupModeConfig);
			}

			env.getCheckpointConfig().enableExternalizedCheckpoints(cleanupMode);
		}

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(pt);
	}