org.dmg.pmml.DefineFunction Java Examples

@Override
public DefineFunction encodeDefineFunction(){
	TfidfTransformer transformer = getTransformer();

	DefineFunction defineFunction = super.encodeDefineFunction();

	Expression expression = defineFunction.getExpression();

	Boolean sublinearTf = transformer.getSublinearTf();
	if(sublinearTf){
		expression = PMMLUtil.createApply(PMMLFunctions.ADD, PMMLUtil.createApply(PMMLFunctions.LN, expression), PMMLUtil.createConstant(1d));
	} // End if

	Boolean useIdf = transformer.getUseIdf();
	if(useIdf){
		ParameterField weight = new ParameterField(FieldName.create("weight"));

		defineFunction.addParameterFields(weight);

		expression = PMMLUtil.createApply(PMMLFunctions.MULTIPLY, expression, new FieldRef(weight.getName()));
	}

	defineFunction.setExpression(expression);

	return defineFunction;
}
 

private ParameterField findParameterField(FieldName name){
	DefineFunction defineFunction = getDefineFunction();

	if(defineFunction.hasParameterFields()){
		List<ParameterField> parameterFields = defineFunction.getParameterFields();

		for(ParameterField parameterField : parameterFields){

			if(Objects.equals(parameterField.getName(), name)){
				return parameterField;
			}
		}
	}

	return null;
}
 

static
public FieldValue evaluate(DefineFunction defineFunction, List<FieldValue> values, EvaluationContext context){
	List<ParameterField> parameterFields = defineFunction.getParameterFields();

	if(parameterFields.size() != values.size()){
		throw new EvaluationException("Function " + PMMLException.formatKey(defineFunction.getName()) + " expects " + parameterFields.size() + " arguments, got " + values.size() + " arguments");
	}

	DefineFunctionEvaluationContext functionContext = new DefineFunctionEvaluationContext(defineFunction, context);

	for(int i = 0; i < parameterFields.size(); i++){
		ParameterField parameterField = parameterFields.get(i);
		FieldValue value = values.get(i);

		FieldName name = parameterField.getName();
		if(name == null){
			throw new MissingAttributeException(parameterField, PMMLAttributes.PARAMETERFIELD_NAME);
		}

		value = value.cast(parameterField);

		functionContext.declare(name, value);
	}

	return ExpressionUtil.evaluateTypedExpressionContainer(defineFunction, functionContext);
}
 

@Override
public List<Feature> encodeFeatures(SparkMLEncoder encoder){
	CountVectorizerModel transformer = getTransformer();

	DocumentFeature documentFeature = (DocumentFeature)encoder.getOnlyFeature(transformer.getInputCol());

	ParameterField documentField = new ParameterField(FieldName.create("document"));

	ParameterField termField = new ParameterField(FieldName.create("term"));

	TextIndex textIndex = new TextIndex(documentField.getName(), new FieldRef(termField.getName()))
		.setTokenize(Boolean.TRUE)
		.setWordSeparatorCharacterRE(documentFeature.getWordSeparatorRE())
		.setLocalTermWeights(transformer.getBinary() ? TextIndex.LocalTermWeights.BINARY : null);

	Set<DocumentFeature.StopWordSet> stopWordSets = documentFeature.getStopWordSets();
	for(DocumentFeature.StopWordSet stopWordSet : stopWordSets){

		if(stopWordSet.isEmpty()){
			continue;
		}

		String tokenRE;

		String wordSeparatorRE = documentFeature.getWordSeparatorRE();
		switch(wordSeparatorRE){
			case "\\s+":
				tokenRE = "(^|\\s+)\\p{Punct}*(" + JOINER.join(stopWordSet) + ")\\p{Punct}*(\\s+|$)";
				break;
			case "\\W+":
				tokenRE = "(\\W+)(" + JOINER.join(stopWordSet) + ")(\\W+)";
				break;
			default:
				throw new IllegalArgumentException("Expected \"\\s+\" or \"\\W+\" as splitter regex pattern, got \"" + wordSeparatorRE + "\"");
		}

		Map<String, List<String>> data = new LinkedHashMap<>();
		data.put("string", Collections.singletonList(tokenRE));
		data.put("stem", Collections.singletonList(" "));
		data.put("regex", Collections.singletonList("true"));

		TextIndexNormalization textIndexNormalization = new TextIndexNormalization(null, PMMLUtil.createInlineTable(data))
			.setCaseSensitive(stopWordSet.isCaseSensitive())
			.setRecursive(Boolean.TRUE); // Handles consecutive matches. See http://stackoverflow.com/a/25085385

		textIndex.addTextIndexNormalizations(textIndexNormalization);
	}

	DefineFunction defineFunction = new DefineFunction("tf" + "@" + String.valueOf(CountVectorizerModelConverter.SEQUENCE.getAndIncrement()), OpType.CONTINUOUS, DataType.INTEGER, null, textIndex)
		.addParameterFields(documentField, termField);

	encoder.addDefineFunction(defineFunction);

	List<Feature> result = new ArrayList<>();

	String[] vocabulary = transformer.vocabulary();
	for(int i = 0; i < vocabulary.length; i++){
		String term = vocabulary[i];

		if(TermUtil.hasPunctuation(term)){
			throw new IllegalArgumentException("Punctuated vocabulary terms (" + term + ") are not supported");
		}

		result.add(new TermFeature(encoder, defineFunction, documentFeature, term));
	}

	return result;
}
 

public TermFeature(PMMLEncoder encoder, DefineFunction defineFunction, Feature feature, String value){
	super(encoder, FieldName.create(defineFunction.getName() + "(" + value + ")"), defineFunction.getDataType());

	setDefineFunction(defineFunction);

	setFeature(feature);
	setValue(value);
}
 

public Apply createApply(){
	DefineFunction defineFunction = getDefineFunction();
	Feature feature = getFeature();
	String value = getValue();

	Constant constant = PMMLUtil.createConstant(value, DataType.STRING);

	return PMMLUtil.createApply(defineFunction.getName(), feature.ref(), constant);
}
 

/**
 * https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.BSpline.html
 */
static
private DefineFunction createBSplineFunction(BSpline bspline, SkLearnEncoder encoder){
	int k = bspline.getK();

	List<Number> c = bspline.getC();
	List<Number> t = bspline.getT();

	int n = (t.size() - k - 1);

	ParameterField paramterField = new ParameterField()
		.setName(FieldName.create("x"))
		.setOpType(OpType.CONTINUOUS)
		.setDataType(DataType.DOUBLE);

	Apply sumApply = PMMLUtil.createApply(PMMLFunctions.SUM);

	for(int i = 0; i < n; i++){

		for(int j = k; j >= 0; j--){
			createBFunction(t, i, j, encoder);
		}

		Apply apply = PMMLUtil.createApply(PMMLFunctions.MULTIPLY)
			.addExpressions(PMMLUtil.createConstant(c.get(i)))
			.addExpressions(PMMLUtil.createApply(formatBFunction(i, k), new FieldRef(paramterField.getName())));

		sumApply.addExpressions(apply);
	}

	DefineFunction defineFunction = new DefineFunction(formatBSplineFunction(k), OpType.CONTINUOUS, DataType.DOUBLE, null, sumApply)
		.addParameterFields(paramterField);

	encoder.addDefineFunction(defineFunction);

	return defineFunction;
}
 

static
private FieldValue evaluateFunction(String name, List<FieldValue> values, EvaluationContext context){
	Function function = FunctionRegistry.getFunction(name);
	if(function != null){
		return function.evaluate(values);
	}

	DefineFunction defineFunction = context.getDefineFunction(name);
	if(defineFunction != null){
		return evaluate(defineFunction, values, context);
	}

	throw new EvaluationException("Function " + PMMLException.formatKey(name) + " is not defined");
}
 

@Override
protected DefineFunction getDefineFunction(String name){
	ModelEvaluator<?> modelEvaluator = getModelEvaluator();

	DefineFunction defineFunction = modelEvaluator.getDefineFunction(name);

	return defineFunction;
}
 

public WeightedTermFeature(PMMLEncoder encoder, DefineFunction defineFunction, Feature feature, String value, Number weight){
	super(encoder, defineFunction, feature, value);

	setWeight(weight);
}
 

protected DefineFunction getDefineFunction(String name){
	throw new UnsupportedOperationException();
}
 

private void setDefineFunction(DefineFunction defineFunction){
	this.defineFunction = defineFunction;
}
 

public DefineFunction getDefineFunction(){
	return this.defineFunction;
}
 

@Override
protected DefineFunction getDefineFunction(String name){
	EvaluationContext parent = getParent();

	return parent.getDefineFunction(name);
}
 

public DefineFunctionEvaluationContext(DefineFunction defineFunction, EvaluationContext parent){
	setDefineFunction(Objects.requireNonNull(defineFunction));
	setParent(Objects.requireNonNull(parent));
}
 

@Override
public Map<String, DefineFunction> load(TransformationDictionary transformationDictionary){
	return IndexableUtil.buildMap(transformationDictionary.getDefineFunctions());
}
 

public DefineFunction getDefineFunction(String name){
	return this.defineFunctions.get(name);
}
 

@Override
public VisitorAction visit(DefineFunction defineFunction){
	declareFields(defineFunction, defineFunction.hasParameterFields() ? defineFunction.getParameterFields() : Collections.emptyList());

	return super.visit(defineFunction);
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	BSpline bspline = getBSpline();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);

	ContinuousFeature continuousFeature = feature.toContinuousFeature();

	DefineFunction defineFunction = createBSplineFunction(bspline, encoder);

	Apply apply = PMMLUtil.createApply(defineFunction.getName())
		.addExpressions(continuousFeature.ref());

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("bspline", feature), apply);

	return Collections.singletonList(new ContinuousFeature(encoder, derivedField));
}
 

public DefineFunction encodeDefineFunction(){
	String analyzer = getAnalyzer();
	List<String> stopWords = getStopWords();
	Object[] nGramRange = getNGramRange();
	Boolean binary = getBinary();
	Object preprocessor = getPreprocessor();
	String stripAccents = getStripAccents();
	Splitter tokenizer = getTokenizer();

	switch(analyzer){
		case "word":
			break;
		default:
			throw new IllegalArgumentException(analyzer);
	}

	if(preprocessor != null){
		throw new IllegalArgumentException();
	} // End if

	if(stripAccents != null){
		throw new IllegalArgumentException(stripAccents);
	}

	ParameterField documentField = new ParameterField(FieldName.create("document"));

	ParameterField termField = new ParameterField(FieldName.create("term"));

	TextIndex textIndex = new TextIndex(documentField.getName(), new FieldRef(termField.getName()))
		.setTokenize(Boolean.TRUE)
		.setWordSeparatorCharacterRE(tokenizer.getSeparatorRE())
		.setLocalTermWeights(binary ? TextIndex.LocalTermWeights.BINARY : null);

	if((stopWords != null && stopWords.size() > 0) && !Arrays.equals(nGramRange, new Integer[]{1, 1})){
		Map<String, List<String>> data = new LinkedHashMap<>();
		data.put("string", Collections.singletonList("(^|\\s+)\\p{Punct}*(" + JOINER.join(stopWords) + ")\\p{Punct}*(\\s+|$)"));
		data.put("stem", Collections.singletonList(" "));
		data.put("regex", Collections.singletonList("true"));

		TextIndexNormalization textIndexNormalization = new TextIndexNormalization(null, PMMLUtil.createInlineTable(data))
			.setRecursive(Boolean.TRUE); // Handles consecutive matches. See http://stackoverflow.com/a/25085385

		textIndex.addTextIndexNormalizations(textIndexNormalization);
	}

	String name = functionName() + "@" + String.valueOf(CountVectorizer.SEQUENCE.getAndIncrement());

	DefineFunction defineFunction = new DefineFunction(name, OpType.CONTINUOUS, DataType.DOUBLE, null, textIndex)
		.addParameterFields(documentField, termField);

	return defineFunction;
}
 

private void setDefineFunction(DefineFunction defineFunction){
	this.defineFunction = Objects.requireNonNull(defineFunction);
}
 

public DefineFunction getDefineFunction(){
	return this.defineFunction;
}
 

@Test
public void inspectFunctions(){
	PMML pmml = createPMML();

	assertVersionRange(pmml, Version.PMML_3_0, Version.PMML_4_4);

	Apply apply = new Apply()
		.setFunction(PMMLFunctions.LOWERCASE);

	DefineFunction defineFunction = new DefineFunction("convert_case", OpType.CATEGORICAL, DataType.STRING, null, apply)
		.addParameterFields(new ParameterField(FieldName.create("string")));

	TransformationDictionary transformationDictionary = new TransformationDictionary()
		.addDefineFunctions(defineFunction);

	pmml.setTransformationDictionary(transformationDictionary);

	assertVersionRange(pmml, Version.PMML_4_1, Version.PMML_4_4);

	apply.setFunction(PMMLFunctions.UPPERCASE);

	assertVersionRange(pmml, Version.PMML_3_0, Version.PMML_4_4);

	apply.setFunction(null);

	assertVersionRange(pmml, Version.PMML_3_0, Version.PMML_3_0);
}