Java Code Examples for org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setNumberOfExecutionRetries()

The following examples show how to use org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setNumberOfExecutionRetries() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RestartStrategyTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
Example 2
Source File: RestartStrategyTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
 */
@Test
public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(42);
	env.getConfig().setExecutionRetryDelay(1337);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
	Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
	Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
}
Example 3
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
Example 4
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
 */
@Test
public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(42);
	env.getConfig().setExecutionRetryDelay(1337);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
	Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
	Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
}
Example 5
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
Example 6
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
 */
@Test
public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(42);
	env.getConfig().setExecutionRetryDelay(1337);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
	Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
	Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
}
Example 7
Source File: Throughput.java    From flink-perf with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.getConfig().setGlobalJobParameters(pt);
	see.setNumberOfExecutionRetries(0);

	if(pt.has("timeout")) {
		see.setBufferTimeout(pt.getLong("timeout"));
	}

	if(pt.has("ft")) {
		see.enableCheckpointing(pt.getLong("ft"));
	}

	DataStream<Type> source = see.addSource(new Source(pt) );

	DataStream<Type> repartitioned = source.partitionByHash(0);
	for(int i = 0; i < pt.getInt("repartitions", 1) - 1;i++) {
		repartitioned = repartitioned.map(new MapFunction<Type, Type>() {
			@Override
			public Type map(Type in) throws Exception {
				Type out = in.copy();
				out.f0++;
				return out;
			}
		}).partitionByHash(0);
	}
	repartitioned.flatMap(new FlatMapFunction<Type, Integer>() {
		public int host = -2;
		long received = 0;
		long start = 0;
		long logfreq = pt.getInt("logfreq");
		long lastLog = -1;
		long lastElements = 0;

		@Override
		public void flatMap(Type element, Collector<Integer> collector) throws Exception {
			if(host == -2) {
				host = convertHostnameToInt(InetAddress.getLocalHost().getHostName());
			}
			if (start == 0) {
				start = System.currentTimeMillis();
			}

			received++;
			if (received % logfreq == 0) {
				// throughput over entire time
				long now = System.currentTimeMillis();
				long sinceSec = ((now - start) / 1000);
				if (sinceSec == 0) return;
				LOG.info("Received {} elements since {}. Elements per second {}, GB received {}",
						received,
						sinceSec,
						received / sinceSec,
						(received * (8 + 8 + 4 + pt.getInt("payload"))) / 1024 / 1024 / 1024);

				// throughput for the last "logfreq" elements
				if(lastLog == -1) {
					// init (the first)
					lastLog = now;
					lastElements = received;
				} else {
					long timeDiff = now - lastLog;
					long elementDiff = received - lastElements;
					double ex = (1000/(double)timeDiff);
					LOG.info("During the last {} ms, we received {} elements. That's {} elements/second/core", timeDiff, elementDiff, elementDiff*ex);
					// reinit
					lastLog = now;
					lastElements = received;
				}
			}
			if (element.f2 != 0 /* && element.f1.equals(host) */) {
				long lat = System.currentTimeMillis() - element.f2;
				LOG.info("Latency {} ms from machine " + element.f1, lat);
			}
		}
	});
	//System.out.println("plan = "+see.getExecutionPlan());;
	see.execute("Flink Throughput Job with: "+pt.toMap());
}