org.apache.flink.api.java.utils.ParameterTool Java Examples
The following examples show how to use
org.apache.flink.api.java.utils.ParameterTool.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AvroDataGeneratorJob.java From flink-tutorials with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { ParameterTool params = Utils.parseArgs(args); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); KafkaSerializationSchema<Message> schema = ClouderaRegistryKafkaSerializationSchema.<Message> builder(params.getRequired(K_KAFKA_TOPIC)) .setConfig(Utils.readSchemaRegistryProperties(params)) .setKey(Message::getId) .build(); FlinkKafkaProducer<Message> kafkaSink = new FlinkKafkaProducer<>( "default", schema, Utils.readKafkaProperties(params), FlinkKafkaProducer.Semantic.AT_LEAST_ONCE); DataStream<Message> input = env.addSource(new DataGeneratorSource()).name("Data Generator Source"); input.addSink(kafkaSink) .name("Kafka Sink") .uid("Kafka Sink"); input.print(); env.execute("Data Generator Job"); }
Example #2
Source File: StreamWordCount.java From jstorm with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // env.addS ParameterTool tool = ParameterTool.fromArgs(args); int sourceParallel = Integer.parseInt(tool.get("s")); int operatorParallel = Integer.parseInt(tool.get("p")); System.out.println("sourceParallel: " + sourceParallel + ", operatorParallel: " + operatorParallel); env.setParallelism(operatorParallel); // get input data DataStream<String> text = env.addSource(new WordSource()).setParallelism(sourceParallel); DataStream<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1) text.flatMap(new LineSplitter()) // group by the tuple field "0" and sum up tuple field "1" .keyBy(0) .sum(1); // execute program // env.execute("StreamWordCount"); System.out.println(env.getExecutionPlan()); }
Example #3
Source File: RideCleansingSolution.java From flink-training-exercises with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", pathToRideData); final int maxEventDelay = 60; // events are out of order by max 60 seconds final int servingSpeedFactor = 600; // events of 10 minutes are served in 1 second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(ExerciseBase.parallelism); // start the data generator DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor))); DataStream<TaxiRide> filteredRides = rides // keep only those rides and both start and end in NYC .filter(new NYCFilter()); // print the filtered stream printOrTest(filteredRides); // run the cleansing pipeline env.execute("Taxi Ride Cleansing"); }
Example #4
Source File: DataSetFineGrainedRecoveryTestProgram.java From flink with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final String latchFilePath = params.getRequired("latchFilePath"); final String outputPath = params.getRequired("outputPath"); final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setExecutionMode(ExecutionMode.BATCH_FORCED); env.setParallelism(4); env.generateSequence(0, 1000) .map(new BlockingIncrementingMapFunction(latchFilePath)) .writeAsText(outputPath, FileSystem.WriteMode.OVERWRITE) .setParallelism(1); env.execute(); }
Example #5
Source File: FlinkKafkaConsumerTest1.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args); StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool); Properties props = buildKafkaProps(parameterTool); //kafka topic list List<String> topics = Arrays.asList(parameterTool.get("metrics.topic"), parameterTool.get("logs.topic")); FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(topics, new MetricSchema(), props); //kafka topic Pattern //FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(java.utils.regex.Pattern.compile("test-topic-[0-9]"), new MetricSchema(), props); // consumer.setStartFromLatest(); // consumer.setStartFromEarliest() DataStreamSource<MetricEvent> data = env.addSource(consumer); data.print(); env.execute("flink kafka connector test"); }
Example #6
Source File: DataStreamAllroundTestJobFactory.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
static SourceFunction<Event> createEventSource(ParameterTool pt) { return new SequenceGeneratorSource( pt.getInt( SEQUENCE_GENERATOR_SRC_KEYSPACE.key(), SEQUENCE_GENERATOR_SRC_KEYSPACE.defaultValue()), pt.getInt( SEQUENCE_GENERATOR_SRC_PAYLOAD_SIZE.key(), SEQUENCE_GENERATOR_SRC_PAYLOAD_SIZE.defaultValue()), pt.getLong( SEQUENCE_GENERATOR_SRC_EVENT_TIME_MAX_OUT_OF_ORDERNESS.key(), SEQUENCE_GENERATOR_SRC_EVENT_TIME_MAX_OUT_OF_ORDERNESS.defaultValue()), pt.getLong( SEQUENCE_GENERATOR_SRC_EVENT_TIME_CLOCK_PROGRESS.key(), SEQUENCE_GENERATOR_SRC_EVENT_TIME_CLOCK_PROGRESS.defaultValue()), pt.getLong( SEQUENCE_GENERATOR_SRC_SLEEP_TIME.key(), SEQUENCE_GENERATOR_SRC_SLEEP_TIME.defaultValue()), pt.getLong( SEQUENCE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.key(), SEQUENCE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.defaultValue())); }
Example #7
Source File: Main.java From flink-learning with Apache License 2.0 | 6 votes |
private static void writeEventToHbase(String string, ParameterTool parameterTool) throws IOException { Configuration configuration = HBaseConfiguration.create(); configuration.set(HBASE_ZOOKEEPER_QUORUM, parameterTool.get(HBASE_ZOOKEEPER_QUORUM)); configuration.set(HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT, parameterTool.get(HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT)); configuration.set(HBASE_RPC_TIMEOUT, parameterTool.get(HBASE_RPC_TIMEOUT)); configuration.set(HBASE_CLIENT_OPERATION_TIMEOUT, parameterTool.get(HBASE_CLIENT_OPERATION_TIMEOUT)); configuration.set(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, parameterTool.get(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)); Connection connect = ConnectionFactory.createConnection(configuration); Admin admin = connect.getAdmin(); if (!admin.tableExists(HBASE_TABLE_NAME)) { //检查是否有该表,如果没有,创建 admin.createTable(new HTableDescriptor(HBASE_TABLE_NAME).addFamily(new HColumnDescriptor(INFO_STREAM))); } Table table = connect.getTable(HBASE_TABLE_NAME); TimeStamp ts = new TimeStamp(new Date()); Date date = ts.getDate(); Put put = new Put(Bytes.toBytes(date.getTime())); put.addColumn(Bytes.toBytes(INFO_STREAM), Bytes.toBytes("test"), Bytes.toBytes(string)); table.put(put); table.close(); connect.close(); }
Example #8
Source File: FailureRateRestartStrategyMain.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setGlobalJobParameters(ParameterTool.fromArgs(args)); //每隔 10s 重启一次,如果两分钟内重启过三次则停止 Job env.setRestartStrategy(RestartStrategies.failureRateRestart(3, Time.minutes(2), Time.seconds(10))); env.addSource(new SourceFunction<Long>() { @Override public void run(SourceContext<Long> sourceContext) throws Exception { while (true) { sourceContext.collect(null); } } @Override public void cancel() { } }) .map((MapFunction<Long, Long>) aLong -> aLong / 1) .print(); env.execute("zhisheng failureRate Restart Strategy example"); }
Example #9
Source File: KafkaExampleUtil.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public static StreamExecutionEnvironment prepareExecutionEnv(ParameterTool parameterTool) throws Exception { if (parameterTool.getNumberOfParameters() < 5) { System.out.println("Missing parameters!\n" + "Usage: Kafka --input-topic <topic> --output-topic <topic> " + "--bootstrap.servers <kafka brokers> " + "--zookeeper.connect <zk quorum> --group.id <some id>"); throw new Exception("Missing parameters!\n" + "Usage: Kafka --input-topic <topic> --output-topic <topic> " + "--bootstrap.servers <kafka brokers> " + "--zookeeper.connect <zk quorum> --group.id <some id>"); } StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000)); env.enableCheckpointing(5000); // create a checkpoint every 5 seconds env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); return env; }
Example #10
Source File: DataSetBrocastMain.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool params = ParameterTool.fromArgs(args); final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); //1. 待广播的数据 DataSet<Integer> toBroadcast = env.fromElements(1, 2, 3); env.fromElements("a", "b") .map(new RichMapFunction<String, String>() { List<Integer> broadcastData; @Override public void open(Configuration parameters) throws Exception { // 3. 获取广播的DataSet数据 作为一个Collection broadcastData = getRuntimeContext().getBroadcastVariable("zhisheng"); } @Override public String map(String value) throws Exception { return broadcastData.get(1) + value; } }).withBroadcastSet(toBroadcast, "zhisheng")// 2. 广播DataSet .print(); }
Example #11
Source File: DoubleParameter.java From flink with Apache License 2.0 | 6 votes |
@Override public void configure(ParameterTool parameterTool) { value = hasDefaultValue ? parameterTool.getDouble(name, defaultValue) : parameterTool.getDouble(name); if (hasMinimumValue) { if (minimumValueInclusive) { Util.checkParameter(value >= minimumValue, name + " must be greater than or equal to " + minimumValue); } else { Util.checkParameter(value > minimumValue, name + " must be greater than " + minimumValue); } } if (hasMaximumValue) { if (maximumValueInclusive) { Util.checkParameter(value <= maximumValue, name + " must be less than or equal to " + maximumValue); } else { Util.checkParameter(value < maximumValue, name + " must be less than " + maximumValue); } } }
Example #12
Source File: DoubleParameter.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public void configure(ParameterTool parameterTool) { value = hasDefaultValue ? parameterTool.getDouble(name, defaultValue) : parameterTool.getDouble(name); if (hasMinimumValue) { if (minimumValueInclusive) { Util.checkParameter(value >= minimumValue, name + " must be greater than or equal to " + minimumValue); } else { Util.checkParameter(value > minimumValue, name + " must be greater than " + minimumValue); } } if (hasMaximumValue) { if (maximumValueInclusive) { Util.checkParameter(value <= maximumValue, name + " must be less than or equal to " + maximumValue); } else { Util.checkParameter(value < maximumValue, name + " must be less than " + maximumValue); } } }
Example #13
Source File: GridGraph.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public void configure(ParameterTool parameterTool) throws ProgramParametrizationException { super.configure(parameterTool); // add dimensions as ordered by dimension ID (dim0, dim1, dim2, ...) Map<Integer, String> dimensionMap = new TreeMap<>(); // first parse all dimensions into a sorted map for (String key : parameterTool.toMap().keySet()) { if (key.startsWith(PREFIX)) { int dimensionId = Integer.parseInt(key.substring(PREFIX.length())); dimensionMap.put(dimensionId, parameterTool.get(key)); } } // then store dimensions in order for (String field : dimensionMap.values()) { dimensions.add(new Dimension(field)); } }
Example #14
Source File: Sink2ES6Main.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args); StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool); DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env); List<HttpHost> esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS)); int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40); int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 5); log.info("-----esAddresses = {}, parameterTool = {}, ", esAddresses, parameterTool); ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, data, (MetricEvent metric, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> { requestIndexer.add(Requests.indexRequest() .index(ZHISHENG + "_" + metric.getName()) .type(ZHISHENG) .source(GsonUtil.toJSONBytes(metric), XContentType.JSON)); }, parameterTool); env.execute("flink learning connectors es6"); }
Example #15
Source File: FlinkKafkaSchemaTest1.java From flink-learning with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args); StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool); Properties props = buildKafkaProps(parameterTool); //kafka topic list List<String> topics = Arrays.asList(parameterTool.get("metrics.topic")); FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(topics, new KafkaDeserializationSchemaWrapper<>(new MetricSchema()), props); DataStreamSource<MetricEvent> data = env.addSource(consumer); data.print(); env.execute("flink kafka connector test"); }
Example #16
Source File: IterationConvergence.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public void configure(ParameterTool parameterTool) { if (!parameterTool.has("iterations") && !parameterTool.has("convergence_threshold")) { // no configuration so use default iterations and maximum threshold value.iterations = defaultIterations; value.convergenceThreshold = Double.MAX_VALUE; } else { // use configured values and maximum default for unset values value.iterations = parameterTool.getInt("iterations", Integer.MAX_VALUE); Util.checkParameter(value.iterations > 0, "iterations must be greater than zero"); value.convergenceThreshold = parameterTool.getDouble("convergence_threshold", Double.MAX_VALUE); Util.checkParameter(value.convergenceThreshold > 0, "convergence threshold must be greater than zero"); } }
Example #17
Source File: KafkaExample.java From flink with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { // parse input arguments final ParameterTool parameterTool = ParameterTool.fromArgs(args); StreamExecutionEnvironment env = KafkaExampleUtil.prepareExecutionEnv(parameterTool); DataStream<KafkaEvent> input = env .addSource( new FlinkKafkaConsumer<>( parameterTool.getRequired("input-topic"), new KafkaEventSchema(), parameterTool.getProperties()) .assignTimestampsAndWatermarks(new CustomWatermarkExtractor())) .keyBy("word") .map(new RollingAdditionMapper()); input.addSink( new FlinkKafkaProducer<>( parameterTool.getRequired("output-topic"), new KeyedSerializationSchemaWrapper<>(new KafkaEventSchema()), parameterTool.getProperties(), FlinkKafkaProducer.Semantic.EXACTLY_ONCE)); env.execute("Modern Kafka Example"); }
Example #18
Source File: DoubleParameterTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testWithoutDefaultWithoutParameter() { Assert.assertEquals("--test TEST", parameter.getUsage()); expectedException.expect(RuntimeException.class); expectedException.expectMessage("No data for required key 'test'"); parameter.configure(ParameterTool.fromArgs(new String[]{})); }
Example #19
Source File: TtlTestConfig.java From flink with Apache License 2.0 | 5 votes |
static TtlTestConfig fromArgs(ParameterTool pt) { int keySpace = pt.getInt(UPDATE_GENERATOR_SRC_KEYSPACE.key(), UPDATE_GENERATOR_SRC_KEYSPACE.defaultValue()); long sleepAfterElements = pt.getLong(UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.key(), UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.defaultValue()); long sleepTime = pt.getLong(UPDATE_GENERATOR_SRC_SLEEP_TIME.key(), UPDATE_GENERATOR_SRC_SLEEP_TIME.defaultValue()); Time ttl = Time.milliseconds(pt.getLong(STATE_TTL_VERIFIER_TTL_MILLI.key(), STATE_TTL_VERIFIER_TTL_MILLI.defaultValue())); long reportStatAfterUpdatesNum = pt.getLong(REPORT_STAT_AFTER_UPDATES_NUM.key(), REPORT_STAT_AFTER_UPDATES_NUM.defaultValue()); return new TtlTestConfig(keySpace, sleepAfterElements, sleepTime, ttl, reportStatAfterUpdatesNum); }
Example #20
Source File: AlertRuleAsyncIOFunction.java From flink-learning with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { parameterTool = (ParameterTool) getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); connection = getConnection(); String sql = "select * from alert_rule where name = ?;"; if (connection != null) { ps = this.connection.prepareStatement(sql); } }
Example #21
Source File: KafkaConfigUtil.java From flink-learning with Apache License 2.0 | 5 votes |
private static Map<KafkaTopicPartition, Long> buildOffsetByTime(Properties props, ParameterTool parameterTool, Long time) { props.setProperty("group.id", "query_time_" + time); KafkaConsumer consumer = new KafkaConsumer(props); List<PartitionInfo> partitionsFor = consumer.partitionsFor(parameterTool.getRequired(PropertiesConstants.METRICS_TOPIC)); Map<TopicPartition, Long> partitionInfoLongMap = new HashMap<>(); for (PartitionInfo partitionInfo : partitionsFor) { partitionInfoLongMap.put(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()), time); } Map<TopicPartition, OffsetAndTimestamp> offsetResult = consumer.offsetsForTimes(partitionInfoLongMap); Map<KafkaTopicPartition, Long> partitionOffset = new HashMap<>(); offsetResult.forEach((key, value) -> partitionOffset.put(new KafkaTopicPartition(key.topic(), key.partition()), value.offset())); consumer.close(); return partitionOffset; }
Example #22
Source File: DoubleParameterTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMinAndMaxAtRangeMaximumInclusive() { parameter.setMinimumValue(-1, true); parameter.setMaximumValue(1, true); parameter.configure(ParameterTool.fromArgs(new String[]{"--test", "1"})); Assert.assertEquals(new Double(1), parameter.getValue()); }
Example #23
Source File: ChoiceParameterTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testWithDefaultWithoutParameter() { parameter.setDefaultValue("default").addChoices("c0", "c1", "c2"); Assert.assertEquals("[--choice <default | c0 | c1 | c2>]", parameter.getUsage()); parameter.configure(ParameterTool.fromArgs(new String[]{})); Assert.assertEquals("default", parameter.getValue()); }
Example #24
Source File: Elasticsearch5SinkExample.java From flink with Apache License 2.0 | 5 votes |
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
Example #25
Source File: DoubleParameterTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testWithDefaultWithParameter() { parameter.setDefaultValue(43.21); Assert.assertEquals("[--test TEST]", parameter.getUsage()); parameter.configure(ParameterTool.fromArgs(new String[]{"--test", "12.34"})); Assert.assertEquals(new Double(12.34), parameter.getValue()); }
Example #26
Source File: GetAlertRuleSourceFunction.java From flink-learning with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { parameterTool = (ParameterTool) getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); connection = getConnection(); String sql = "select * from alert_rule;"; if (connection != null) { ps = this.connection.prepareStatement(sql); } }
Example #27
Source File: Main.java From flink-learning with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL; Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool); SingleOutputStreamOperator<Tuple2<String, String>> product = env.addSource(new FlinkKafkaConsumer011<>( parameterTool.get(METRICS_TOPIC), //这个 kafka topic 需要和上面的工具类的 topic 一致 new SimpleStringSchema(), props)) .map(string -> GsonUtil.fromJson(string, ProductEvent.class)) //反序列化 JSON .flatMap(new FlatMapFunction<ProductEvent, Tuple2<String, String>>() { @Override public void flatMap(ProductEvent value, Collector<Tuple2<String, String>> out) throws Exception { //收集商品 id 和 price 两个属性 out.collect(new Tuple2<>(value.getId().toString(), value.getPrice().toString())); } }); // product.print(); //单个 Redis FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder().setHost(parameterTool.get("redis.host")).build(); product.addSink(new RedisSink<Tuple2<String, String>>(conf, new RedisSinkMapper())); //Redis 的 ip 信息一般都从配置文件取出来 //Redis 集群 /* FlinkJedisClusterConfig clusterConfig = new FlinkJedisClusterConfig.Builder() .setNodes(new HashSet<InetSocketAddress>( Arrays.asList(new InetSocketAddress("redis1", 6379)))).build();*/ //Redis Sentinels /* FlinkJedisSentinelConfig sentinelConfig = new FlinkJedisSentinelConfig.Builder() .setMasterName("master") .setSentinels(new HashSet<>(Arrays.asList("sentinel1", "sentinel2"))) .setPassword("") .setDatabase(1).build();*/ env.execute("flink redis connector"); }
Example #28
Source File: ChoiceParameterTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testWithDefaultWithParameter() { parameter.setDefaultValue("default").addChoices("c0", "c1", "c2"); Assert.assertEquals("[--choice <default | c0 | c1 | c2>]", parameter.getUsage()); parameter.configure(ParameterTool.fromArgs(new String[]{"--choice", "c1"})); Assert.assertEquals("c1", parameter.getValue()); }
Example #29
Source File: LongParameterTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testWithoutDefaultWithParameter() { Assert.assertEquals("--test TEST", parameter.getUsage()); parameter.configure(ParameterTool.fromArgs(new String[]{"--test", "42"})); Assert.assertEquals(new Long(42), parameter.getValue()); }
Example #30
Source File: DataStreamAllroundTestJobFactory.java From flink with Apache License 2.0 | 5 votes |
private static void setupCheckpointing(final StreamExecutionEnvironment env, final ParameterTool pt) { String semantics = pt.get(TEST_SEMANTICS.key(), TEST_SEMANTICS.defaultValue()); long checkpointInterval = pt.getLong(ENVIRONMENT_CHECKPOINT_INTERVAL.key(), ENVIRONMENT_CHECKPOINT_INTERVAL.defaultValue()); CheckpointingMode checkpointingMode = semantics.equalsIgnoreCase("exactly-once") ? CheckpointingMode.EXACTLY_ONCE : CheckpointingMode.AT_LEAST_ONCE; env.enableCheckpointing(checkpointInterval, checkpointingMode); boolean enableExternalizedCheckpoints = pt.getBoolean( ENVIRONMENT_EXTERNALIZE_CHECKPOINT.key(), ENVIRONMENT_EXTERNALIZE_CHECKPOINT.defaultValue()); if (enableExternalizedCheckpoints) { String cleanupModeConfig = pt.get( ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.key(), ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.defaultValue()); CheckpointConfig.ExternalizedCheckpointCleanup cleanupMode; switch (cleanupModeConfig) { case "retain": cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION; break; case "delete": cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION; break; default: throw new IllegalArgumentException("Unknown clean up mode for externalized checkpoints: " + cleanupModeConfig); } env.getCheckpointConfig().enableExternalizedCheckpoints(cleanupMode); final int tolerableDeclinedCheckpointNumber = pt.getInt( ENVIRONMENT_TOLERABLE_DECLINED_CHECKPOINT_NUMBER.key(), ENVIRONMENT_TOLERABLE_DECLINED_CHECKPOINT_NUMBER.defaultValue()); env.getCheckpointConfig().setTolerableCheckpointFailureNumber(tolerableDeclinedCheckpointNumber); } }