org.apache.flink.optimizer.DataStatistics Java Examples

/**
 * We assume that the cardinality is the product of  the input cardinalities
 * and that the result width is the sum of the input widths.
 * 
 * @param statistics The statistics object to optionally access.
 */
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 * card2;
	
	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
		
		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();

	if (card1 < 0 || card2 < 0) {
		this.estimatedNumRecords = -1;
	} else {
		this.estimatedNumRecords = Math.max(card1, card2);
	}

	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;

		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

/**
 * We assume that the cardinality is the product of  the input cardinalities
 * and that the result width is the sum of the input widths.
 * 
 * @param statistics The statistics object to optionally access.
 */
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 * card2;
	
	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
		
		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

/**
 * Creates a instance that submits the programs to the JobManager defined in the
 * configuration. This method will try to resolve the JobManager hostname and throw an exception
 * if that is not possible.
 *
 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval
 * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down
 */
public ClusterClient(
		Configuration flinkConfig,
		HighAvailabilityServices highAvailabilityServices,
		boolean sharedHaServices) {
	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);

	this.timeout = AkkaUtils.getClientTimeout(flinkConfig);
	this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig);

	this.actorSystemLoader = new LazyActorSystemLoader(
		highAvailabilityServices,
		Time.milliseconds(lookupTimeout.toMillis()),
		flinkConfig,
		log);

	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
	this.sharedHaServices = sharedHaServices;
}
 

public ClusterClient(
		Configuration flinkConfig,
		HighAvailabilityServices highAvailabilityServices,
		boolean sharedHaServices,
		ActorSystemLoader actorSystemLoader) {
	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);

	this.timeout = AkkaUtils.getClientTimeout(flinkConfig);
	this.lookupTimeout = AkkaUtils.getLookupTimeout(flinkConfig);

	this.actorSystemLoader = Preconditions.checkNotNull(actorSystemLoader);

	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
	this.sharedHaServices = sharedHaServices;
}
 

@Test
public void testGetExecutionPlan() throws ProgramInvocationException {
	PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
	assertNotNull(prg.getPreviewPlan());

	Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
	OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1);
	assertNotNull(op);

	PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
	assertNotNull(dumper.getOptimizerPlanAsJSON(op));

	// test HTML escaping
	PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
	dumper2.setEncodeForHTML(true);
	String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);

	assertEquals(-1, htmlEscaped.indexOf('\\'));
}
 

@Test
public void testGetExecutionPlan() throws ProgramInvocationException {
	PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
	assertNotNull(prg.getPreviewPlan());

	Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
	OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1);
	assertNotNull(op);

	PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
	assertNotNull(dumper.getOptimizerPlanAsJSON(op));

	// test HTML escaping
	PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
	dumper2.setEncodeForHTML(true);
	String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);

	assertEquals(-1, htmlEscaped.indexOf('\\'));
}
 

/**
 * The default estimates build on the principle of inclusion: The smaller input key domain is included in the larger
 * input key domain. We also assume that every key from the larger input has one join partner in the smaller input.
 * The result cardinality is hence the larger one.
 */
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : Math.max(card1, card2);
	
	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
		
		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

/**
 * The default estimates build on the principle of inclusion: The smaller input key domain is included in the larger
 * input key domain. We also assume that every key from the larger input has one join partner in the smaller input.
 * The result cardinality is hence the larger one.
 */
@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : Math.max(card1, card2);
	
	if (this.estimatedNumRecords >= 0) {
		float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
		float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
		
		if (width > 0) {
			this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
		}
	}
}
 

@Test
public void testGetExecutionPlan() {
	try {
		PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
		assertNotNull(prg.getPreviewPlan());

		InetAddress mockAddress = InetAddress.getLocalHost();
		InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345);

		Configuration config = new Configuration();

		config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName());
		config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort());

		Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
		OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1);
		assertNotNull(op);

		PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
		assertNotNull(dumper.getOptimizerPlanAsJSON(op));

		// test HTML escaping
		PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
		dumper2.setEncodeForHTML(true);
		String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);

		assertEquals(-1, htmlEscaped.indexOf('\\'));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration);
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 

@Override
public void computeOutputEstimates(DataStatistics statistics) {
	OptimizerNode in1 = getFirstPredecessorNode();
	OptimizerNode in2 = getSecondPredecessorNode();
	
	this.estimatedNumRecords = in1.estimatedNumRecords > 0 && in2.estimatedNumRecords > 0 ?
			in1.estimatedNumRecords + in2.estimatedNumRecords : -1;
	this.estimatedOutputSize = in1.estimatedOutputSize > 0 && in2.estimatedOutputSize > 0 ?
		in1.estimatedOutputSize + in2.estimatedOutputSize : -1;
}
 

/**
 * Helpers to generate the JobGraph.
 */
private static JobGraph getJobGraph(Plan plan) {
	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	JobGraphGenerator jgg = new JobGraphGenerator();
	OptimizedPlan op = pc.compile(plan);
	return jgg.compileJobGraph(op);
}
 

@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 + card2;
	
	long size1 = getFirstPredecessorNode().getEstimatedOutputSize();
	long size2 = getSecondPredecessorNode().getEstimatedOutputSize();
	this.estimatedOutputSize = (size1 < 0 || size2 < 0) ? -1 : size1 + size2;
}
 

@Test
public void testGetExecutionPlan() {
	try {
		PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp");
		assertNotNull(prg.getPreviewPlan());

		InetAddress mockAddress = InetAddress.getLocalHost();
		InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345);

		Configuration config = new Configuration();

		config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName());
		config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort());

		Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config);
		OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1);
		assertNotNull(op);

		PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
		assertNotNull(dumper.getOptimizerPlanAsJSON(op));

		// test HTML escaping
		PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator();
		dumper2.setEncodeForHTML(true);
		String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op);

		assertEquals(-1, htmlEscaped.indexOf('\\'));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Override
public void computeOutputEstimates(DataStatistics statistics) {
	OptimizerNode in1 = getFirstPredecessorNode();
	OptimizerNode in2 = getSecondPredecessorNode();
	
	this.estimatedNumRecords = in1.estimatedNumRecords > 0 && in2.estimatedNumRecords > 0 ?
			in1.estimatedNumRecords + in2.estimatedNumRecords : -1;
	this.estimatedOutputSize = in1.estimatedOutputSize > 0 && in2.estimatedOutputSize > 0 ?
		in1.estimatedOutputSize + in2.estimatedOutputSize : -1;
}
 

@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
	long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
	this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 + card2;
	
	long size1 = getFirstPredecessorNode().getEstimatedOutputSize();
	long size2 = getSecondPredecessorNode().getEstimatedOutputSize();
	this.estimatedOutputSize = (size1 < 0 || size2 < 0) ? -1 : size1 + size2;
}
 

/**
 * Creates a instance that submits the programs to the JobManager defined in the
 * configuration. This method will try to resolve the JobManager hostname and throw an exception
 * if that is not possible.
 *
 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval
 * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down
 */
public ClusterClient(
		Configuration flinkConfig,
		HighAvailabilityServices highAvailabilityServices,
		boolean sharedHaServices) {
	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);

	this.timeout = AkkaUtils.getClientTimeout(flinkConfig);

	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
	this.sharedHaServices = sharedHaServices;
}
 

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration);
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 

/**
 * Helpers to generate the JobGraph.
 */
private static JobGraph getJobGraph(Plan plan) {
	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	JobGraphGenerator jgg = new JobGraphGenerator();
	OptimizedPlan op = pc.compile(plan);
	return jgg.compileJobGraph(op);
}
 

private FlinkPlan createPlan(String name, Configuration flinkConfig) {
	if (streamExecEnv != null) {
		final StreamGraph graph = streamExecEnv.getStreamGraph();
		graph.setJobName(name);
		return graph;
	} else {
		final int parallelism = execEnv.getParallelism();
		final Plan unoptimizedPlan = execEnv.createProgramPlan();
		unoptimizedPlan.setJobName(name);
		final Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);
		return ClusterClient.getOptimizedPlan(compiler, unoptimizedPlan, parallelism);
	}
}
 

@Before
public void setup() {
	Configuration flinkConf = new Configuration();
	this.dataStats = new DataStatistics();
	this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator(), flinkConf);
	this.withStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM);

	this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator(), flinkConf);
	this.noStatsCompiler.setDefaultParallelism(DEFAULT_PARALLELISM);
}
 

@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
	Optimizer opt = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
	OptimizedPlan optPlan = opt.compile(plan);
	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(optPlan);
}
 

/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws CliArgsException, FileNotFoundException, ProgramInvocationException {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	InfoOptions infoOptions = new InfoOptions(commandLine);

	// evaluate help flag
	if (infoOptions.isPrintHelp()) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	if (infoOptions.getJarFilePath() == null) {
		throw new CliArgsException("The program JAR file was not specified.");
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(infoOptions);

	try {
		int parallelism = infoOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
		FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);

		String jsonPlan = null;
		if (flinkPlan instanceof OptimizedPlan) {
			jsonPlan = new PlanJSONDumpGenerator().getOptimizerPlanAsJSON((OptimizedPlan) flinkPlan);
		} else if (flinkPlan instanceof StreamingPlan) {
			jsonPlan = ((StreamingPlan) flinkPlan).getStreamingPlanAsJSON();
		}

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 

@Override
public void computeOutputEstimates(DataStatistics statistics) {
	// nothing to be done here
}
 

@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	// no estimates needed at this point
}
 

/**
 * Executes the given program on a local runtime and waits for the job to finish.
 *
 * <p>If the executor has not been started before, this starts the executor and shuts it down
 * after the job finished. If the job runs in session mode, the executor is kept alive until
 * no more references to the executor exist.</p>
 *
 * @param plan The plan of the program to execute.
 * @return The net runtime of the program, in milliseconds.
 *
 * @throws Exception Thrown, if either the startup of the local execution context, or the execution
 *                   caused an exception.
 */
@Override
public JobExecutionResult executePlan(Plan plan) throws Exception {
	if (plan == null) {
		throw new IllegalArgumentException("The plan may not be null.");
	}

	synchronized (this.lock) {

		// check if we start a session dedicated for this execution
		final boolean shutDownAtEnd;

		if (jobExecutorService == null) {
			shutDownAtEnd = true;

			// configure the number of local slots equal to the parallelism of the local plan
			if (this.taskManagerNumSlots == DEFAULT_TASK_MANAGER_NUM_SLOTS) {
				int maxParallelism = plan.getMaximumParallelism();
				if (maxParallelism > 0) {
					this.taskManagerNumSlots = maxParallelism;
				}
			}

			// start the cluster for us
			start();
		}
		else {
			// we use the existing session
			shutDownAtEnd = false;
		}

		try {
			// TODO: Set job's default parallelism to max number of slots
			final int slotsPerTaskManager = jobExecutorServiceConfiguration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, taskManagerNumSlots);
			final int numTaskManagers = jobExecutorServiceConfiguration.getInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
			plan.setDefaultParallelism(slotsPerTaskManager * numTaskManagers);

			Optimizer pc = new Optimizer(new DataStatistics(), jobExecutorServiceConfiguration);
			OptimizedPlan op = pc.compile(plan);

			JobGraphGenerator jgg = new JobGraphGenerator(jobExecutorServiceConfiguration);
			JobGraph jobGraph = jgg.compileJobGraph(op, plan.getJobId());

			return jobExecutorService.executeJobBlocking(jobGraph);
		}
		finally {
			if (shutDownAtEnd) {
				stop();
			}
		}
	}
}
 

private OptimizedPlan compileProgram(String jobName) {
	Plan p = createProgramPlan(jobName);

	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	return pc.compile(p);
}
 

@Override
protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
	// for CoGroup, we currently make no reasonable default estimates
}