org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopInputSplit Java Examples

@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 

@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 

@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 

@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 

@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {

	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 

@Override
public HadoopInputSplit[] createInputSplits(int minNumSplits)
		throws IOException {
	org.apache.hadoop.mapred.InputSplit[] splitArray = mapredInputFormat.getSplits(jobConf, minNumSplits);
	HadoopInputSplit[] hiSplit = new HadoopInputSplit[splitArray.length];
	for (int i = 0; i < splitArray.length; i++) {
		hiSplit[i] = new HadoopInputSplit(i, splitArray[i], jobConf);
	}
	return hiSplit;
}
 

@Override
public void open(HadoopInputSplit split) throws IOException {

	// enforce sequential open() calls
	synchronized (OPEN_MUTEX) {

		this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());
		if (this.recordReader instanceof Configurable) {
			((Configurable) this.recordReader).setConf(jobConf);
		}
		key = this.recordReader.createKey();
		value = this.recordReader.createValue();
		this.fetched = false;
	}
}
 

@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 

private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}
 

@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 

private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}
 

@Override
public void open(HadoopInputSplit split) throws IOException {

	this.jobConf = split.getJobConf();
	this.flowProcess = new FlinkFlowProcess(this.jobConf, this.getRuntimeContext(), flowNode.getID());

	processBeginTime = System.currentTimeMillis();
	flowProcess.increment( SliceCounters.Process_Begin_Time, processBeginTime );

	try {

		Set<FlowElement> sources = flowNode.getSourceElements();
		if(sources.size() != 1) {
			throw new RuntimeException("FlowNode for TapInputFormat may only have a single source");
		}
		FlowElement sourceElement = sources.iterator().next();
		if(!(sourceElement instanceof Tap)) {
			throw new RuntimeException("Source of TapInputFormat must be a Tap");
		}
		Tap source = (Tap)sourceElement;

		streamGraph = new SourceStreamGraph( flowProcess, flowNode, source );

		sourceStage = this.streamGraph.getSourceStage();
		sinkStage = this.streamGraph.getSinkStage();

		for( Duct head : streamGraph.getHeads() ) {
			LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement());
		}

		for( Duct tail : streamGraph.getTails() ) {
			LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement());
		}

	}
	catch( Throwable throwable ) {

		if( throwable instanceof CascadingException) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during TapInputFormat configuration", throwable );
	}

	RecordReader<?, ?> recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter());

	if (recordReader instanceof Configurable) {
		((Configurable) recordReader).setConf(jobConf);
	}
	else if (recordReader instanceof JobConfigurable) {
		((JobConfigurable) recordReader).configure(jobConf);
	}

	try {
		this.sourceStage.setRecordReader(recordReader);
	} catch(Throwable t) {
		if(t instanceof IOException) {
			throw (IOException)t;
		}
		else {
			throw new RuntimeException(t);
		}
	}

}
 

@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 

@Override
public InputSplitAssigner getInputSplitAssigner(HadoopInputSplit[] inputSplits) {
	return new LocatableInputSplitAssigner(inputSplits);
}
 

private HadoopInputSplit getHadoopInputSplit() {
	return new HadoopInputSplit(1, getFileSplit(), new JobConf());
}