org.apache.flink.api.common.operators.util.UserCodeWrapper Java Examples

private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
	final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	final OperatorID operatorID = new OperatorID();

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSinkTask.class);
	vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, (UserCodeWrapper<? extends OutputFormat<?>>) node.getProgramOperator().getUserCodeWrapper())
		.addParameters(operatorID, node.getProgramOperator().getParameters())
		.write(config);

	return vertex;
}
 

private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
	final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	final OperatorID operatorID = new OperatorID();

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSinkTask.class);
	vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, (UserCodeWrapper<? extends OutputFormat<?>>) node.getProgramOperator().getUserCodeWrapper())
		.addParameters(operatorID, node.getProgramOperator().getParameters())
		.write(config);

	return vertex;
}
 

private JobVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
	final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	final OperatorID operatorID = new OperatorID();

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSourceTask.class);
	vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addInputFormat(operatorID, (UserCodeWrapper<? extends InputFormat<?, ?>>) node.getProgramOperator().getUserCodeWrapper())
		.addParameters(operatorID, node.getProgramOperator().getParameters())
		.write(config);

	config.setOutputSerializer(node.getSerializer());
	return vertex;
}
 

@Test
public void testOnlyInputFormat() {
	InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());

	OperatorID operatorID = new OperatorID();
	formatContainer.addInputFormat(operatorID, new TestInputFormat("test input format"));
	formatContainer.addParameters(operatorID, "parameter1", "abc123");

	TaskConfig taskConfig = new TaskConfig(new Configuration());
	formatContainer.write(taskConfig);

	InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());

	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = loadedFormatContainer.getInputFormats();
	assertEquals(1, inputFormats.size());
	assertEquals(0, loadedFormatContainer.getOutputFormats().size());

	TestInputFormat inputFormat = (TestInputFormat) inputFormats.get(operatorID).getUserCodeObject();
	assertEquals("test input format", inputFormat.getName());

	Configuration parameters = loadedFormatContainer.getParameters(operatorID);
	assertEquals(1, parameters.keySet().size());
	assertEquals("abc123", parameters.getString("parameter1", null));
}
 

@SuppressWarnings("unchecked")
public <OT, T extends InputSplit> Pair<OperatorID, InputFormat<OT, T>> getUniqueInputFormat() {
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formats.getInputFormats();
	Preconditions.checkState(inputFormats.size() == 1);

	Map.Entry<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> entry = inputFormats.entrySet().iterator().next();

	return new ImmutablePair<>(entry.getKey(),
		(InputFormat<OT, T>) entry.getValue().getUserCodeObject(InputFormat.class, userCodeClassLoader));
}
 

public CrossOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
	
	if (this instanceof CrossWithSmall) {
		setCrossHint(CrossHint.SECOND_IS_SMALL);
	}
	else if (this instanceof CrossWithLarge) {
		setCrossHint(CrossHint.FIRST_IS_SMALL);
	}
}
 

public void setStubWrapper(UserCodeWrapper<?> wrapper) {
	try {
		InstantiationUtil.writeObjectToConfig(wrapper, this.config, STUB_OBJECT);
	} catch (IOException e) {
		throw new CorruptConfigurationException("Could not write the user code wrapper " + wrapper.getClass() + " : " + e.toString(), e);
	}
}
 

@Override
public void finalizeOnMaster(ClassLoader loader) throws Exception {
	final InputOutputFormatContainer formatContainer = initInputOutputformatContainer(loader);

	final ClassLoader original = Thread.currentThread().getContextClassLoader();
	try {
		// set user classloader before calling user code
		Thread.currentThread().setContextClassLoader(loader);

		// configure input formats and invoke finalizeGlobal()
		Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
		for (Map.Entry<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> entry : outputFormats.entrySet()) {
			final OutputFormat<?> outputFormat;

			try {
				outputFormat = entry.getValue().getUserCodeObject();
				outputFormat.configure(formatContainer.getParameters(entry.getKey()));
			} catch (Throwable t) {
				throw new Exception("Configuring the output format (" + getFormatDescription(entry.getKey()) + ") failed: "
					+ t.getMessage(), t);
			}

			if (outputFormat instanceof FinalizeOnMaster) {
				((FinalizeOnMaster) outputFormat).finalizeGlobal(getParallelism());
			}
		}

	} finally {
		// restore original classloader
		Thread.currentThread().setContextClassLoader(original);
	}
}
 

public CrossOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
	
	if (this instanceof CrossWithSmall) {
		setCrossHint(CrossHint.SECOND_IS_SMALL);
	}
	else if (this instanceof CrossWithLarge) {
		setCrossHint(CrossHint.FIRST_IS_SMALL);
	}
}
 

@SuppressWarnings("unchecked")
public <T> UserCodeWrapper<T> getStubWrapper(ClassLoader cl) {
	try {
		return (UserCodeWrapper<T>) InstantiationUtil.readObjectFromConfig(this.config, STUB_OBJECT, cl);
	} catch (ClassNotFoundException | IOException e) {
		throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
	}
}
 

@SuppressWarnings("unchecked")
public <OT, T extends InputSplit> Pair<OperatorID, InputFormat<OT, T>> getUniqueInputFormat() {
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formats.getInputFormats();
	Preconditions.checkState(inputFormats.size() == 1);

	Map.Entry<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> entry = inputFormats.entrySet().iterator().next();

	return new ImmutablePair<>(entry.getKey(),
		(InputFormat<OT, T>) entry.getValue().getUserCodeObject(InputFormat.class, userCodeClassLoader));
}
 

public CrossOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
	
	if (this instanceof CrossWithSmall) {
		setCrossHint(CrossHint.SECOND_IS_SMALL);
	}
	else if (this instanceof CrossWithLarge) {
		setCrossHint(CrossHint.FIRST_IS_SMALL);
	}
}
 

public void addOutputFormat(OperatorID operatorId, UserCodeWrapper<? extends OutputFormat<?>> wrapper) {
	if (outputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The output format has been set for the operator: " + operatorId);
	}

	outputFormats.put(operatorId, checkNotNull(wrapper));
}
 

public void addOutputFormat(OperatorID operatorId, UserCodeWrapper<? extends OutputFormat<?>> wrapper) {
	if (outputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The output format has been set for the operator: " + operatorId);
	}

	outputFormats.put(operatorId, checkNotNull(wrapper));
}
 

public void addInputFormat(OperatorID operatorId, UserCodeWrapper<? extends InputFormat<?, ?>> wrapper) {
	if (inputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The input format has been set for the operator: " + operatorId);
	}

	inputFormats.put(operatorId, checkNotNull(wrapper));
}
 

@SuppressWarnings("unchecked")
public <T> UserCodeWrapper<T> getStubWrapper(ClassLoader cl) {
	try {
		return (UserCodeWrapper<T>) InstantiationUtil.readObjectFromConfig(this.config, STUB_OBJECT, cl);
	} catch (ClassNotFoundException | IOException e) {
		throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
	}
}
 

@SuppressWarnings("unchecked")
public <T> UserCodeWrapper<T> getStubWrapper(ClassLoader cl) {
	try {
		return (UserCodeWrapper<T>) InstantiationUtil.readObjectFromConfig(this.config, STUB_OBJECT, cl);
	} catch (ClassNotFoundException | IOException e) {
		throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
	}
}
 

public void addInputFormat(OperatorID operatorId, UserCodeWrapper<? extends InputFormat<?, ?>> wrapper) {
	if (inputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The input format has been set for the operator: " + operatorId);
	}

	inputFormats.put(operatorId, checkNotNull(wrapper));
}
 

public FlatMapOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 

public Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> getOutputFormats() {
	return Collections.unmodifiableMap(outputFormats);
}
 

public Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> getInputFormats() {
	return Collections.unmodifiableMap(inputFormats);
}
 

public Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> getInputFormats() {
	return formats.getInputFormats();
}
 

public MapOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 

public Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> getInputFormats() {
	return Collections.unmodifiableMap(inputFormats);
}
 

public GroupReduceOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 

public GroupReduceOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, int[] keyPositions, String name) {
	super(udf, operatorInfo, keyPositions, name);
}
 

public MapPartitionOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 

public Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> getOutputFormats() {
	return formats.getOutputFormats();
}
 

public FilterOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<T, T> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 

public InnerJoinOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo,
		int[] keyPositions1, int[] keyPositions2, String name) {
	super(udf, operatorInfo, keyPositions1, keyPositions2, name);
}