org.jpmml.converter.FeatureUtil Java Examples

@Override
public List<Feature> encodeFeatures(SparkMLEncoder encoder){
	RegexTokenizer transformer = getTransformer();

	if(!transformer.getGaps()){
		throw new IllegalArgumentException("Expected splitter mode, got token matching mode");
	} // End if

	if(transformer.getMinTokenLength() != 1){
		throw new IllegalArgumentException("Expected 1 as minimum token length, got " + transformer.getMinTokenLength() + " as minimum token length");
	}

	Feature feature = encoder.getOnlyFeature(transformer.getInputCol());

	Field<?> field = feature.getField();

	if(transformer.getToLowercase()){
		Apply apply = PMMLUtil.createApply(PMMLFunctions.LOWERCASE, feature.ref());

		field = encoder.createDerivedField(FeatureUtil.createName("lowercase", feature), OpType.CATEGORICAL, DataType.STRING, apply);
	}

	return Collections.singletonList(new DocumentFeature(encoder, field, transformer.getPattern()));
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String pattern = getPattern();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);
	if(!(DataType.STRING).equals(feature.getDataType())){
		throw new IllegalArgumentException();
	}

	Apply apply = PMMLUtil.createApply(PMMLFunctions.MATCHES)
		.addExpressions(feature.ref())
		.addExpressions(PMMLUtil.createConstant(pattern, DataType.STRING));

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("matches", feature), OpType.CATEGORICAL, DataType.BOOLEAN, apply);

	return Collections.singletonList(new BooleanFeature(encoder, derivedField));
}
 

static
public Feature encodeIndicatorFeature(Feature feature, Object missingValue, SkLearnEncoder encoder){
	Expression expression = feature.ref();

	if(missingValue != null){
		expression = PMMLUtil.createApply(PMMLFunctions.EQUAL, expression, PMMLUtil.createConstant(missingValue, feature.getDataType()));
	} else

	{
		expression = PMMLUtil.createApply(PMMLFunctions.ISMISSING, expression);
	}

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("missing_indicator", feature), OpType.CATEGORICAL, DataType.BOOLEAN, expression);

	return new BooleanFeature(encoder, derivedField);
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	Integer begin = getBegin();
	Integer end = getEnd();

	if((begin < 0) || (end < begin)){
		throw new IllegalArgumentException();
	}

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);
	if(!(DataType.STRING).equals(feature.getDataType())){
		throw new IllegalArgumentException();
	}

	Apply apply = PMMLUtil.createApply(PMMLFunctions.SUBSTRING)
		.addExpressions(feature.ref())
		.addExpressions(PMMLUtil.createConstant(begin + 1, DataType.INTEGER), PMMLUtil.createConstant((end - begin), DataType.INTEGER));

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("substring", feature), OpType.CATEGORICAL, DataType.STRING, apply);

	return Collections.singletonList(new StringFeature(encoder, derivedField));
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String pattern = getPattern();
	String replacement = getReplacement();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);
	if(!(DataType.STRING).equals(feature.getDataType())){
		throw new IllegalArgumentException();
	}

	Apply apply = PMMLUtil.createApply(PMMLFunctions.REPLACE)
		.addExpressions(feature.ref())
		.addExpressions(PMMLUtil.createConstant(pattern, DataType.STRING), PMMLUtil.createConstant(replacement, DataType.STRING));

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("replace", feature), OpType.CATEGORICAL, DataType.STRING, apply);

	return Collections.singletonList(new StringFeature(encoder, derivedField));
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String function = getFunction();

	if(features.size() <= 1){
		return features;
	}

	Apply apply = PMMLUtil.createApply(translateFunction(function));

	for(Feature feature : features){
		apply.addExpressions(feature.ref());
	}

	FieldName name = FeatureUtil.createName(function, features);

	DerivedField derivedField = encoder.createDerivedField(name, OpType.CONTINUOUS, DataType.DOUBLE, apply);

	return Collections.singletonList(new ContinuousFeature(encoder, derivedField));
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	UFunc func = getFunc();

	if(func == null){
		return features;
	}

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		ContinuousFeature continuousFeature = (features.get(i)).toContinuousFeature();

		DerivedField derivedField = encoder.ensureDerivedField(FeatureUtil.createName(func.getName(), continuousFeature), OpType.CONTINUOUS, DataType.DOUBLE, () -> UFuncUtil.encodeUFunc(func, Collections.singletonList(continuousFeature.ref())));

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

private void scaleFeatures(RExpEncoder encoder){
	RGenericVector mvr = getObject();

	RDoubleVector scale = mvr.getDoubleElement("scale", false);
	if(scale == null){
		return;
	}

	List<Feature> features = encoder.getFeatures();

	if(scale.size() != features.size()){
		throw new IllegalArgumentException();
	}

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);
		Double factor = scale.getValue(i);

		if(ValueUtil.isOne(factor)){
			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		Apply apply = PMMLUtil.createApply(PMMLFunctions.DIVIDE, continuousFeature.ref(), PMMLUtil.createConstant(factor));

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("scale", feature), OpType.CONTINUOUS, DataType.DOUBLE, apply);

		features.set(i, new ContinuousFeature(encoder, derivedField));
	}
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String function = getFunction();
	Boolean trimBlanks = getTrimBlanks();

	if(function == null && !trimBlanks){
		return features;
	}

	List<Feature> result = new ArrayList<>();

	for(Feature feature : features){
		Expression expression = feature.ref();

		if(function != null){
			expression = PMMLUtil.createApply(translateFunction(function), expression);
		} // End if

		if(trimBlanks){
			expression = PMMLUtil.createApply(PMMLFunctions.TRIMBLANKS, expression);
		}

		Field<?> field = encoder.toCategorical(feature.getName(), Collections.emptyList());

		// XXX: Should have been set by the previous transformer
		field.setDataType(DataType.STRING);

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("normalize", feature), OpType.CATEGORICAL, DataType.STRING, expression);

		feature = new StringFeature(encoder, derivedField);

		result.add(feature);
	}

	return result;
}
 

static
public Feature encodeIndexFeature(Feature feature, List<?> categories, List<? extends Number> indexCategories, Number mapMissingTo, Number defaultValue, DataType dataType, SkLearnEncoder encoder){
	ClassDictUtil.checkSize(categories, indexCategories);

	encoder.toCategorical(feature.getName(), categories);

	Supplier<MapValues> mapValuesSupplier = () -> {
		MapValues mapValues = PMMLUtil.createMapValues(feature.getName(), categories, indexCategories)
			.setMapMissingTo(mapMissingTo)
			.setDefaultValue(defaultValue);

		return mapValues;
	};

	DerivedField derivedField = encoder.ensureDerivedField(FeatureUtil.createName("encoder", feature), OpType.CATEGORICAL, dataType, mapValuesSupplier);

	Feature encodedFeature = new IndexFeature(encoder, derivedField, indexCategories);

	Feature result = new CategoricalFeature(encoder, feature, categories){

		@Override
		public ContinuousFeature toContinuousFeature(){
			return encodedFeature.toContinuousFeature();
		}
	};

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	Number threshold = getThreshold();

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		// "($name <= threshold) ? 0 : 1"
		Apply apply = PMMLUtil.createApply(PMMLFunctions.THRESHOLD, continuousFeature.ref(), PMMLUtil.createConstant(threshold));

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("binarizer", continuousFeature), apply);

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	List<?> classes = getClasses();

	Number negLabel = getNegLabel();
	Number posLabel = getPosLabel();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);

	List<Object> categories = new ArrayList<>();
	categories.addAll(classes);

	List<Number> labelCategories = new ArrayList<>();
	labelCategories.add(negLabel);
	labelCategories.add(posLabel);

	List<Feature> result = new ArrayList<>();

	classes = prepareClasses(classes);

	for(int i = 0; i < classes.size(); i++){
		Object value = classes.get(i);

		if(ValueUtil.isZero(negLabel) && ValueUtil.isOne(posLabel)){
			result.add(new BinaryFeature(encoder, feature, value));
		} else

		{
			// "($name == value) ? pos_label : neg_label"
			Apply apply = PMMLUtil.createApply(PMMLFunctions.IF)
				.addExpressions(PMMLUtil.createApply(PMMLFunctions.EQUAL, feature.ref(), PMMLUtil.createConstant(value, feature.getDataType())))
				.addExpressions(PMMLUtil.createConstant(posLabel), PMMLUtil.createConstant(negLabel));

			FieldName name = (classes.size() > 1 ? FeatureUtil.createName("label_binarizer", feature, i) : FeatureUtil.createName("label_binarizer", feature));

			DerivedField derivedField = encoder.createDerivedField(name, apply);

			result.add(new CategoricalFeature(encoder, derivedField, labelCategories));
		}
	}

	encoder.toCategorical(feature.getName(), categories);

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		ObjectFeature objectFeature = (ObjectFeature)features.get(i);

		FieldName name = FieldName.create("seconds_since_midnight(" + (FeatureUtil.getName(objectFeature)).getValue() + ")");

		DerivedField derivedField = encoder.ensureDerivedField(name, OpType.CONTINUOUS, DataType.INTEGER, () -> PMMLUtil.createApply(PMMLFunctions.DATESECONDSSINCEMIDNIGHT, objectFeature.ref()));

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	Boolean withMean = getWithMean();
	Boolean withStd = getWithStd();

	List<? extends Number> mean = (withMean ? getMean() : null);
	List<? extends Number> std = (withStd ? getStd() : null);

	if(mean == null && std == null){
		return features;
	}

	ClassDictUtil.checkSize(features, mean, std);

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);

		Number meanValue = (withMean ? mean.get(i) : 0d);
		Number stdValue = (withStd ? std.get(i) : 1d);

		if(ValueUtil.isZero(meanValue) && ValueUtil.isOne(stdValue)){
			result.add(feature);

			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		// "($name - mean) / std"
		Expression expression = continuousFeature.ref();

		if(!ValueUtil.isZero(meanValue)){
			expression = PMMLUtil.createApply(PMMLFunctions.SUBTRACT, expression, PMMLUtil.createConstant(meanValue));
		} // End if

		if(!ValueUtil.isOne(stdValue)){
			expression = PMMLUtil.createApply(PMMLFunctions.DIVIDE, expression, PMMLUtil.createConstant(stdValue));
		}

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("standard_scaler", continuousFeature), expression);

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	BSpline bspline = getBSpline();

	ClassDictUtil.checkSize(1, features);

	Feature feature = features.get(0);

	ContinuousFeature continuousFeature = feature.toContinuousFeature();

	DefineFunction defineFunction = createBSplineFunction(bspline, encoder);

	Apply apply = PMMLUtil.createApply(defineFunction.getName())
		.addExpressions(continuousFeature.ref());

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("bspline", feature), apply);

	return Collections.singletonList(new ContinuousFeature(encoder, derivedField));
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	List<? extends Number> min = getMin();
	List<? extends Number> scale = getScale();

	ClassDictUtil.checkSize(features, min, scale);

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);

		Number minValue = min.get(i);
		Number scaleValue = scale.get(i);

		if(ValueUtil.isOne(scaleValue) && ValueUtil.isZero(minValue)){
			result.add(feature);

			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		// "($name * scale) + min"
		Expression expression = continuousFeature.ref();

		if(!ValueUtil.isOne(scaleValue)){
			expression = PMMLUtil.createApply(PMMLFunctions.MULTIPLY, expression, PMMLUtil.createConstant(scaleValue));
		} // End if

		if(!ValueUtil.isZero(minValue)){
			expression = PMMLUtil.createApply(PMMLFunctions.ADD, expression, PMMLUtil.createConstant(minValue));
		}

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("mix_max_scaler", continuousFeature), expression);

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	Boolean withCentering = getWithCentering();
	Boolean withScaling = getWithScaling();

	List<? extends Number> center = (withCentering ? getCenter() : null);
	List<? extends Number> scale = (withScaling ? getScale() : null);

	if(center == null && scale == null){
		return features;
	}

	ClassDictUtil.checkSize(features, center, scale);

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);

		Number centerValue = (withCentering ? center.get(i) : 0d);
		Number scaleValue = (withScaling ? scale.get(i) : 1d);

		if(ValueUtil.isZero(centerValue) && ValueUtil.isOne(scaleValue)){
			result.add(feature);

			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		// "($name - center) / scale"
		Expression expression = continuousFeature.ref();

		if(!ValueUtil.isZero(centerValue)){
			expression = PMMLUtil.createApply(PMMLFunctions.SUBTRACT, expression, PMMLUtil.createConstant(centerValue));
		} // End if

		if(!ValueUtil.isOne(scaleValue)){
			expression = PMMLUtil.createApply(PMMLFunctions.DIVIDE, expression, PMMLUtil.createConstant(scaleValue));
		}

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("robust_scaler", continuousFeature), expression);

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

static
public Feature encodeFeature(Feature feature, Boolean addIndicator, Object missingValue, Object replacementValue, MissingValueTreatmentMethod missingValueTreatmentMethod, SkLearnEncoder encoder){
	Field<?> field = feature.getField();

	if(field instanceof DataField && !addIndicator){
		DataField dataField = (DataField)field;

		encoder.addDecorator(dataField, new MissingValueDecorator(missingValueTreatmentMethod, replacementValue));

		if(missingValue != null){
			PMMLUtil.addValues(dataField, Collections.singletonList(missingValue), Value.Property.MISSING);
		}

		return feature;
	} // End if

	if((field instanceof DataField) || (field instanceof DerivedField)){
		Expression expression = feature.ref();

		if(missingValue != null){
			expression = PMMLUtil.createApply(PMMLFunctions.EQUAL, expression, PMMLUtil.createConstant(missingValue, feature.getDataType()));
		} else

		{
			expression = PMMLUtil.createApply(PMMLFunctions.ISMISSING, expression);
		}

		expression = PMMLUtil.createApply(PMMLFunctions.IF)
			.addExpressions(expression)
			.addExpressions(PMMLUtil.createConstant(replacementValue, feature.getDataType()), feature.ref());

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("imputer", feature), field.getOpType(), field.getDataType(), expression);

		DataType dataType = derivedField.getDataType();
		switch(dataType){
			case INTEGER:
			case FLOAT:
			case DOUBLE:
				return new ContinuousFeature(encoder, derivedField);
			case STRING:
				return new StringFeature(encoder, derivedField);
			default:
				return new ObjectFeature(encoder, derivedField.getName(), derivedField.getDataType());
		}
	} else

	{
		throw new IllegalArgumentException();
	}
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	String separator = getSeparator();

	Apply apply = PMMLUtil.createApply(PMMLFunctions.CONCAT);

	List<Expression> expressions = apply.getExpressions();

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);

		if((i > 0) && !("").equals(separator)){
			expressions.add(PMMLUtil.createConstant(separator, DataType.STRING));
		}

		expressions.add(feature.ref());
	}

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("concat", features), OpType.CATEGORICAL, DataType.STRING, apply);

	return Collections.singletonList(new StringFeature(encoder, derivedField));
}
 

private void scaleFeatures(RExpEncoder encoder){
	RGenericVector svm = getObject();

	RDoubleVector sv = svm.getDoubleElement("SV");
	RBooleanVector scaled = svm.getBooleanElement("scaled");
	RGenericVector xScale = svm.getGenericElement("x.scale");

	RStringVector rowNames = sv.dimnames(0);
	RStringVector columnNames = sv.dimnames(1);

	List<Feature> features = encoder.getFeatures();

	if((scaled.size() != columnNames.size()) || (scaled.size() != features.size())){
		throw new IllegalArgumentException();
	}

	RDoubleVector xScaledCenter = xScale.getDoubleElement("scaled:center");
	RDoubleVector xScaledScale = xScale.getDoubleElement("scaled:scale");

	for(int i = 0; i < columnNames.size(); i++){
		String columnName = columnNames.getValue(i);

		if(!scaled.getValue(i)){
			continue;
		}

		Feature feature = features.get(i);

		Double center = xScaledCenter.getElement(columnName);
		Double scale = xScaledScale.getElement(columnName);

		if(ValueUtil.isZero(center) && ValueUtil.isOne(scale)){
			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		Expression expression = continuousFeature.ref();

		if(!ValueUtil.isZero(center)){
			expression = PMMLUtil.createApply(PMMLFunctions.SUBTRACT, expression, PMMLUtil.createConstant(center));
		} // End if

		if(!ValueUtil.isOne(scale)){
			expression = PMMLUtil.createApply(PMMLFunctions.DIVIDE, expression, PMMLUtil.createConstant(scale));
		}

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("scale", feature), OpType.CONTINUOUS, DataType.DOUBLE, expression);

		features.set(i, new ContinuousFeature(encoder, derivedField));
	}
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	List<? extends Number> scale = getScale();

	ClassDictUtil.checkSize(features, scale);

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		Feature feature = features.get(i);

		Number value = scale.get(i);
		if(ValueUtil.isOne(value)){
			result.add(feature);

			continue;
		}

		ContinuousFeature continuousFeature = feature.toContinuousFeature();

		// "$name / scale"
		Apply apply = PMMLUtil.createApply(PMMLFunctions.DIVIDE, continuousFeature.ref(), PMMLUtil.createConstant(value));

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("max_abs_scaler", continuousFeature), apply);

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	GregorianCalendar epoch = getEpoch();
	String function = getFunction();

	LocalDateTime epochDateTime = CalendarUtil.toLocalDateTime(epoch);
	if(epochDateTime.getMonthValue() != 1 || epochDateTime.getDayOfMonth() != 1){
		throw new IllegalArgumentException(String.valueOf(epochDateTime));
	}

	int year = epochDateTime.getYear();

	String dateFunction = function;

	if(dateFunction.startsWith("date")){
		dateFunction = dateFunction.substring("date".length(), dateFunction.length());
	}

	dateFunction = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, dateFunction);

	List<Feature> result = new ArrayList<>();

	for(int i = 0; i < features.size(); i++){
		ObjectFeature objectFeature = (ObjectFeature)features.get(i);

		FieldName name = FieldName.create(dateFunction + "(" + (FeatureUtil.getName(objectFeature)).getValue() + ", " + year + ")");

		DerivedField derivedField = encoder.ensureDerivedField(name, OpType.CONTINUOUS, DataType.INTEGER, () -> PMMLUtil.createApply(function, objectFeature.ref(), PMMLUtil.createConstant(year, DataType.INTEGER)));

		result.add(new ContinuousFeature(encoder, derivedField));
	}

	return result;
}
 

@Override
public List<Feature> encodeFeatures(SparkMLEncoder encoder){
	Tokenizer transformer = getTransformer();

	Feature feature = encoder.getOnlyFeature(transformer.getInputCol());

	Apply apply = PMMLUtil.createApply(PMMLFunctions.LOWERCASE, feature.ref());

	DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("lowercase", feature), OpType.CATEGORICAL, DataType.STRING, apply);

	return Collections.singletonList(new DocumentFeature(encoder, derivedField, "\\s+"));
}