org.dmg.pmml.Value Java Examples

static
private List<Object> parseCategories(DataField dataField){
	List<Object> result = new ArrayList<>();

	if(dataField.hasValues()){
		List<Value> pmmlValues = dataField.getValues();

		for(Value pmmlValue : pmmlValues){
			Object simpleValue = pmmlValue.getValue();
			if(simpleValue == null){
				throw new MissingAttributeException(pmmlValue, PMMLAttributes.VALUE_VALUE);
			}

			Value.Property property = pmmlValue.getProperty();
			switch(property){
				case VALID:
					result.add(simpleValue);
					break;
				default:
					break;
			}
		}
	}

	return result;
}
 

@Test
public void testBuildDataDictionary() {
  Map<Integer,Collection<String>> distinctValues = new HashMap<>();
  distinctValues.put(1, Arrays.asList("one", "two", "three", "four", "five"));
  CategoricalValueEncodings categoricalValueEncodings =
      new CategoricalValueEncodings(distinctValues);

  DataDictionary dictionary =
      AppPMMLUtils.buildDataDictionary(buildTestSchema(), categoricalValueEncodings);
  assertEquals(4, dictionary.getNumberOfFields().intValue());
  checkDataField(dictionary.getDataFields().get(0), "foo", null);
  checkDataField(dictionary.getDataFields().get(1), "bar", true);
  checkDataField(dictionary.getDataFields().get(2), "baz", null);
  checkDataField(dictionary.getDataFields().get(3), "bing", false);

  List<Value> dfValues = dictionary.getDataFields().get(1).getValues();
  assertEquals(5, dfValues.size());
  String[] categoricalValues = { "one", "two", "three", "four", "five" };
  for (int i = 0; i < categoricalValues.length; i++) {
    assertEquals(categoricalValues[i], dfValues.get(i).getValue());
  }
}
 

@Test
public void testBuildCategoricalEncoding() {
  List<DataField> dataFields = new ArrayList<>();
  dataFields.add(new DataField(FieldName.create("foo"), OpType.CONTINUOUS, DataType.DOUBLE));
  DataField barField =
      new DataField(FieldName.create("bar"), OpType.CATEGORICAL, DataType.STRING);
  barField.addValues(new Value("b"), new Value("a"));
  dataFields.add(barField);
  DataDictionary dictionary = new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
  CategoricalValueEncodings encodings = AppPMMLUtils.buildCategoricalValueEncodings(dictionary);
  assertEquals(2, encodings.getValueCount(1));
  assertEquals(0, encodings.getValueEncodingMap(1).get("b").intValue());
  assertEquals(1, encodings.getValueEncodingMap(1).get("a").intValue());
  assertEquals("b", encodings.getEncodingValueMap(1).get(0));
  assertEquals("a", encodings.getEncodingValueMap(1).get(1));
  assertEquals(Collections.singletonMap(1, 2), encodings.getCategoryCounts());
}
 

@Override
public VisitorAction visit(Value value){
	PMMLObject parent = getParent();

	Object simpleValue = value.getValue();
	if(simpleValue == null){
		throw new MissingAttributeException(value, PMMLAttributes.VALUE_VALUE);
	} // End if

	if(parent instanceof Field){
		Field<?> field = (Field<?>)parent;

		DataType dataType = field.getDataType();
		if(dataType != null){
			simpleValue = safeParseOrCast(dataType, simpleValue);

			value.setValue(simpleValue);
		}
	}

	return super.visit(value);
}
 

static
private Value[] createValues(String... values){
	List<Value> result = new ArrayList<>();

	for(String value : values){
		result.add(new Value(value));
	}

	return result.toArray(new Value[result.size()]);
}
 

static
private <F extends org.dmg.pmml.Field<F> & HasDiscreteDomain<F>> List<String> encodeDiscreteDomain(F field){

	if(field.hasValues()){
		List<Value> values = field.getValues();

		return values.stream()
			.filter(value -> (Value.Property.VALID).equals(value.getProperty()))
			.map(value -> TypeUtil.format(value.getValue()))
			.collect(Collectors.toList());
	}

	return Collections.emptyList();
}
 

static
private Value createValue(String value, Value.Property property){
	Value result = new Value(value)
		.setProperty(property);

	return result;
}
 

static
private void clearDomain(DataField dataField){
	List<Interval> intervals = dataField.getIntervals();
	intervals.clear();

	List<Value> values = dataField.getValues();
	values.clear();
}
 

static
private Value createValue(String value, Value.Property property){
	Value result = new Value(value)
		.setProperty(property);

	return result;
}
 

@Test
public void getValueMapping(){
	Value invalidValue = createValue("0", Value.Property.INVALID);
	Value validValueOne = createValue("1", Value.Property.VALID);
	Value validValueTwo = createValue("2", null);
	Value validValueThree = createValue("3", null);
	Value missingValue = createValue("N/A", Value.Property.MISSING);

	DataField dataField = new DataField(FieldName.create("x"), OpType.CATEGORICAL, DataType.STRING)
		.addValues(invalidValue, validValueOne, validValueTwo, validValueThree, missingValue);

	RichDataField richDataField = new RichDataField(dataField);

	Map<?, Integer> valueMap = richDataField.getMap();

	assertEquals(5, valueMap.size());

	assertEquals(FieldValue.STATUS_UNKNOWN_INVALID, valueMap.get("0"));
	assertEquals((Integer)1, valueMap.get("1"));
	assertEquals((Integer)2, valueMap.get("2"));
	assertEquals((Integer)3, valueMap.get("3"));
	assertEquals(FieldValue.STATUS_MISSING, valueMap.get("N/A"));

	dataField.setDataType(DataType.INTEGER);

	richDataField = new RichDataField(dataField);

	valueMap = richDataField.getMap();

	assertEquals(4, valueMap.size());

	assertEquals(FieldValue.STATUS_UNKNOWN_INVALID, valueMap.get(0));
	assertEquals((Integer)1, valueMap.get(1));
	assertEquals((Integer)2, valueMap.get(2));
	assertEquals((Integer)3, valueMap.get(3));
}
 

private Map<Object, Integer> parseValues(){
	DataType dataType = getDataType();

	Map<Object, Integer> result = new LinkedHashMap<>();

	int validIndex = 0;

	List<Value> pmmlValues = getValues();
	for(Value pmmlValue : pmmlValues){
		Object objectValue = pmmlValue.getValue();
		if(objectValue == null){
			throw new MissingAttributeException(pmmlValue, PMMLAttributes.VALUE_VALUE);
		}

		Value.Property property = pmmlValue.getProperty();
		switch(property){
			case VALID:
				{
					validIndex++;

					Object value = TypeUtil.parseOrCast(dataType, objectValue);

					result.put(value, validIndex);
				}
				break;
			case INVALID:
			case MISSING:
				throw new InvalidAttributeException(pmmlValue, property);
			default:
				throw new UnsupportedAttributeException(pmmlValue, property);
		}
	}

	return result;
}
 

static
private <F extends Field<F> & HasDiscreteDomain<F>> List<Object> parseValidValues(F field){
	List<Object> result = new ArrayList<>();

	DataType dataType = field.getDataType();
	if(dataType == null){
		throw new MissingAttributeException(MissingAttributeException.formatMessage(XPathUtil.formatElement(field.getClass()) + "@dataType"), field);
	} // End if

	if(field.hasValues()){
		List<Value> pmmlValues = field.getValues();

		for(Value pmmlValue : pmmlValues){
			Object simpleValue = pmmlValue.getValue();
			if(simpleValue == null){
				throw new MissingAttributeException(pmmlValue, PMMLAttributes.VALUE_VALUE);
			}

			Value.Property property = pmmlValue.getProperty();
			switch(property){
				case VALID:
					result.add(TypeUtil.parseOrCast(dataType, simpleValue));
					break;
				default:
					break;
			}
		}
	}

	return result;
}
 

private Map<Object, Integer> parseValues(){
	DataType dataType = getDataType();

	Map<Object, Integer> result = new LinkedHashMap<>();

	int validIndex = 0;

	List<Value> pmmlValues = getValues();
	for(Value pmmlValue : pmmlValues){
		Object objectValue = pmmlValue.getValue();
		if(objectValue == null){
			throw new MissingAttributeException(pmmlValue, PMMLAttributes.VALUE_VALUE);
		}

		Value.Property property = pmmlValue.getProperty();
		switch(property){
			case VALID:
				{
					validIndex++;

					Object value = TypeUtil.parseOrCast(dataType, objectValue);

					result.put(value, validIndex);
				}
				break;
			case INVALID:
			case MISSING:
				throw new InvalidAttributeException(pmmlValue, property);
			default:
				throw new UnsupportedAttributeException(pmmlValue, property);
		}
	}

	return result;
}
 

public static CategoricalValueEncodings buildCategoricalValueEncodings(
    DataDictionary dictionary) {
  Map<Integer,Collection<String>> indexToValues = new HashMap<>();
  List<DataField> dataFields = dictionary.getDataFields();
  for (int featureIndex = 0; featureIndex < dataFields.size(); featureIndex++) {
    DataField field = dataFields.get(featureIndex);
    Collection<Value> values = field.getValues();
    if (values != null && !values.isEmpty()) {
      Collection<String> categoricalValues =
          values.stream().map(v -> v.getValue().toString()).collect(Collectors.toList());
      indexToValues.put(featureIndex, categoricalValues);
    }
  }
  return new CategoricalValueEncodings(indexToValues);
}
 

public static DataDictionary buildDataDictionary(
    InputSchema schema,
    CategoricalValueEncodings categoricalValueEncodings) {
  List<String> featureNames = schema.getFeatureNames();

  List<DataField> dataFields = new ArrayList<>();
  for (int featureIndex = 0; featureIndex < featureNames.size(); featureIndex++) {
    String featureName = featureNames.get(featureIndex);
    OpType opType;
    DataType dataType;
    if (schema.isNumeric(featureName)) {
      opType = OpType.CONTINUOUS;
      dataType = DataType.DOUBLE;
    } else if (schema.isCategorical(featureName)) {
      opType = OpType.CATEGORICAL;
      dataType = DataType.STRING;
    } else {
      // Don't know
      opType = null;
      dataType = null;
    }
    DataField field = new DataField(FieldName.create(featureName), opType, dataType);
    if (schema.isCategorical(featureName)) {
      Objects.requireNonNull(categoricalValueEncodings);
      categoricalValueEncodings.getEncodingValueMap(featureIndex).entrySet().stream().
          sorted(Comparator.comparing(Map.Entry::getKey)).
          map(Map.Entry::getValue).
          forEach(value -> field.addValues(new Value(value)));
    }
    dataFields.add(field);
  }

  return new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
}
 

private void addValue(Value.Property property, String value){

		if(value == null){
			return;
		}

		List<String> values = this.valueMap.get(property);
		if(values == null){
			values = new ArrayList<>();

			this.valueMap.put(property, values);
		}

		values.add(value);
	}
 

@Override
public List<Feature> encodeFeatures(SparkMLEncoder encoder){
	ImputerModel transformer = getTransformer();

	Double missingValue = transformer.getMissingValue();
	String strategy = transformer.getStrategy();
	Dataset<Row> surrogateDF = transformer.surrogateDF();

	MissingValueTreatmentMethod missingValueTreatmentMethod = parseStrategy(strategy);

	List<Row> surrogateRows = surrogateDF.collectAsList();
	if(surrogateRows.size() != 1){
		throw new IllegalArgumentException();
	}

	Row surrogateRow = surrogateRows.get(0);

	InOutMode inputMode = getInputMode();

	List<Feature> result = new ArrayList<>();

	String[] inputCols = inputMode.getInputCols(transformer);
	for(String inputCol : inputCols){
		Feature feature = encoder.getOnlyFeature(inputCol);

		Field<?> field = feature.getField();

		if(field instanceof DataField){
			DataField dataField = (DataField)field;

			Object surrogate = surrogateRow.getAs(inputCol);

			encoder.addDecorator(dataField, new MissingValueDecorator(missingValueTreatmentMethod, surrogate));

			if(missingValue != null && !missingValue.isNaN()){
				PMMLUtil.addValues(dataField, Collections.singletonList(missingValue), Value.Property.MISSING);
			}
		} else

		{
			throw new IllegalArgumentException();
		}

		result.add(feature);
	}

	return result;
}
 

static
public Value getValidValue(DataField dataField, Object value){

	if(value == null){
		return null;
	} // End if

	if(dataField.hasValues()){
		DataType dataType = dataField.getDataType();
		if(dataType == null){
			throw new MissingAttributeException(dataField, PMMLAttributes.DATAFIELD_DATATYPE);
		}

		value = TypeUtil.parseOrCast(dataType, value);

		List<Value> pmmlValues = dataField.getValues();
		for(int i = 0, max = pmmlValues.size(); i < max; i++){
			Value pmmlValue = pmmlValues.get(i);

			Object simpleValue = pmmlValue.getValue();
			if(simpleValue == null){
				throw new MissingAttributeException(pmmlValue, PMMLAttributes.VALUE_VALUE);
			}

			Value.Property property = pmmlValue.getProperty();
			switch(property){
				case VALID:
					{
						boolean equals = TypeUtil.equals(dataType, value, simpleValue);

						if(equals){
							return pmmlValue;
						}
					}
					break;
				case INVALID:
				case MISSING:
					break;
				default:
					throw new UnsupportedAttributeException(pmmlValue, property);
			}
		}
	}

	return null;
}
 

private static PMML buildDummyClassificationModel(int numTrees) {
  PMML pmml = PMMLUtils.buildSkeletonPMML();

  List<DataField> dataFields = new ArrayList<>();
  DataField predictor =
      new DataField(FieldName.create("color"), OpType.CATEGORICAL, DataType.STRING);
  predictor.addValues(new Value("yellow"), new Value("red"));
  dataFields.add(predictor);
  DataField target =
      new DataField(FieldName.create("fruit"), OpType.CATEGORICAL, DataType.STRING);
  target.addValues(new Value("banana"), new Value("apple"));
  dataFields.add(target);
  DataDictionary dataDictionary =
      new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
  pmml.setDataDictionary(dataDictionary);

  List<MiningField> miningFields = new ArrayList<>();
  MiningField predictorMF = new MiningField(FieldName.create("color"))
      .setOpType(OpType.CATEGORICAL)
      .setUsageType(MiningField.UsageType.ACTIVE)
      .setImportance(0.5);
  miningFields.add(predictorMF);
  MiningField targetMF = new MiningField(FieldName.create("fruit"))
      .setOpType(OpType.CATEGORICAL)
      .setUsageType(MiningField.UsageType.PREDICTED);
  miningFields.add(targetMF);
  MiningSchema miningSchema = new MiningSchema(miningFields);

  double dummyCount = 2.0;
  Node rootNode =
    new ComplexNode().setId("r").setRecordCount(dummyCount).setPredicate(new True());

  double halfCount = dummyCount / 2;

  Node left = new ComplexNode().setId("r-").setRecordCount(halfCount).setPredicate(new True());
  left.addScoreDistributions(new ScoreDistribution("apple", halfCount));
  Node right = new ComplexNode().setId("r+").setRecordCount(halfCount)
      .setPredicate(new SimpleSetPredicate(FieldName.create("color"),
                                           SimpleSetPredicate.BooleanOperator.IS_NOT_IN,
                                           new Array(Array.Type.STRING, "red")));
  right.addScoreDistributions(new ScoreDistribution("banana", halfCount));

  rootNode.addNodes(right, left);

  TreeModel treeModel = new TreeModel(MiningFunction.CLASSIFICATION, miningSchema, rootNode)
      .setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT)
      .setMissingValueStrategy(TreeModel.MissingValueStrategy.DEFAULT_CHILD);

  if (numTrees > 1) {
    MiningModel miningModel = new MiningModel(MiningFunction.CLASSIFICATION, miningSchema);
    List<Segment> segments = new ArrayList<>();
    for (int i = 0; i < numTrees; i++) {
      segments.add(new Segment()
          .setId(Integer.toString(i))
          .setPredicate(new True())
          .setModel(treeModel)
          .setWeight(1.0));
    }
    miningModel.setSegmentation(
        new Segmentation(Segmentation.MultipleModelMethod.WEIGHTED_MAJORITY_VOTE, segments));
    pmml.addModels(miningModel);
  } else {
    pmml.addModels(treeModel);
  }

  return pmml;
}
 

@Override
public List<Feature> encodeFeatures(List<Feature> features, SkLearnEncoder encoder){
	MissingValueTreatmentMethod missingValueTreatment = DomainUtil.parseMissingValueTreatment(getMissingValueTreatment());
	Object missingValueReplacement = getMissingValueReplacement();
	List<?> missingValues = getMissingValues();

	if(missingValueReplacement != null){

		if(missingValueTreatment == null){
			missingValueTreatment = MissingValueTreatmentMethod.AS_VALUE;
		}
	}

	InvalidValueTreatmentMethod invalidValueTreatment = DomainUtil.parseInvalidValueTreatment(getInvalidValueTreatment());
	Object invalidValueReplacement = getInvalidValueReplacement();

	if(invalidValueReplacement != null){

		if(invalidValueTreatment == null){
			invalidValueTreatment = InvalidValueTreatmentMethod.AS_IS;
		}
	}

	for(Feature feature : features){
		WildcardFeature wildcardFeature = asWildcardFeature(feature);

		DataField dataField = wildcardFeature.getField();

		DataType dataType = dataField.getDataType();

		if(missingValueTreatment != null){
			Object pmmlMissingValueReplacement = (missingValueReplacement != null ? standardizeValue(dataType, missingValueReplacement) : null);

			encoder.addDecorator(dataField, new MissingValueDecorator(missingValueTreatment, pmmlMissingValueReplacement));
		} // End if

		if(missingValues != null){
			PMMLUtil.addValues(dataField, standardizeValues(dataType, missingValues), Value.Property.MISSING);
		} // End if

		if(invalidValueTreatment != null){
			Object pmmlInvalidValueReplacement = (invalidValueReplacement != null ? standardizeValue(dataType, invalidValueReplacement) : null);

			encoder.addDecorator(dataField, new InvalidValueDecorator(invalidValueTreatment, pmmlInvalidValueReplacement));
		}
	}

	return features;
}
 

static
public Feature encodeFeature(Feature feature, Boolean addIndicator, Object missingValue, Object replacementValue, MissingValueTreatmentMethod missingValueTreatmentMethod, SkLearnEncoder encoder){
	Field<?> field = feature.getField();

	if(field instanceof DataField && !addIndicator){
		DataField dataField = (DataField)field;

		encoder.addDecorator(dataField, new MissingValueDecorator(missingValueTreatmentMethod, replacementValue));

		if(missingValue != null){
			PMMLUtil.addValues(dataField, Collections.singletonList(missingValue), Value.Property.MISSING);
		}

		return feature;
	} // End if

	if((field instanceof DataField) || (field instanceof DerivedField)){
		Expression expression = feature.ref();

		if(missingValue != null){
			expression = PMMLUtil.createApply(PMMLFunctions.EQUAL, expression, PMMLUtil.createConstant(missingValue, feature.getDataType()));
		} else

		{
			expression = PMMLUtil.createApply(PMMLFunctions.ISMISSING, expression);
		}

		expression = PMMLUtil.createApply(PMMLFunctions.IF)
			.addExpressions(expression)
			.addExpressions(PMMLUtil.createConstant(replacementValue, feature.getDataType()), feature.ref());

		DerivedField derivedField = encoder.createDerivedField(FeatureUtil.createName("imputer", feature), field.getOpType(), field.getDataType(), expression);

		DataType dataType = derivedField.getDataType();
		switch(dataType){
			case INTEGER:
			case FLOAT:
			case DOUBLE:
				return new ContinuousFeature(encoder, derivedField);
			case STRING:
				return new StringFeature(encoder, derivedField);
			default:
				return new ObjectFeature(encoder, derivedField.getName(), derivedField.getDataType());
		}
	} else

	{
		throw new IllegalArgumentException();
	}
}
 

public void addMissingValue(String value){
	addValue(Value.Property.MISSING, value);
}
 

public void addInvalidValue(String value){
	addValue(Value.Property.INVALID, value);
}
 

@Test
public void parseRegressionModel(){
	Value falseValue = new Value("false");
	Value trueValue = new Value("true");
	Value invalidValue = new Value("N/A");

	DataField dataField = new DataField(FieldName.create("x1"), OpType.CATEGORICAL, DataType.STRING)
		.addValues(falseValue, trueValue, invalidValue);

	DataDictionary dataDictionary = new DataDictionary()
		.addDataFields(dataField);

	CategoricalPredictor falseTerm = new CategoricalPredictor(dataField.getName(), "false", -1d);
	CategoricalPredictor trueTerm = new CategoricalPredictor(dataField.getName(), "true", 1d);

	RegressionTable regressionTable = new RegressionTable()
		.addCategoricalPredictors(falseTerm, trueTerm);

	MiningField miningField = new MiningField(dataField.getName())
		.setMissingValueReplacement("false")
		.setInvalidValueReplacement("N/A");

	MiningSchema miningSchema = new MiningSchema()
		.addMiningFields(miningField);

	RegressionModel regressionModel = new RegressionModel(MiningFunction.REGRESSION, miningSchema, null)
		.addRegressionTables(regressionTable);

	PMML pmml = new PMML(Version.PMML_4_3.getVersion(), new Header(), dataDictionary)
		.addModels(regressionModel);

	List<DataField> dataFields = dataDictionary.getDataFields();

	ValueParser parser = new ValueParser(ValueParser.Mode.STRICT);
	parser.applyTo(pmml);

	dataField = dataFields.get(0);

	assertEquals("false", falseValue.getValue());
	assertEquals("true", trueValue.getValue());
	assertEquals("N/A", invalidValue.getValue());

	assertEquals("false", falseTerm.getValue());
	assertEquals("true", trueTerm.getValue());

	assertEquals("false", miningField.getMissingValueReplacement());
	assertEquals("N/A", miningField.getInvalidValueReplacement());

	dataField.setDataType(DataType.BOOLEAN);

	parser.applyTo(pmml);

	assertEquals(Boolean.FALSE, falseValue.getValue());
	assertEquals(Boolean.TRUE, trueValue.getValue());
	assertEquals("N/A", invalidValue.getValue());

	assertEquals(Boolean.FALSE, falseTerm.getValue());
	assertEquals(Boolean.TRUE, trueTerm.getValue());

	assertEquals(Boolean.FALSE, miningField.getMissingValueReplacement());
	assertEquals("N/A", miningField.getInvalidValueReplacement());
}
 

public void addValidValue(String value){
	addValue(Value.Property.VALID, value);
}
 

@Test
public void equals(){
	DataField left = new DataField()
		.setName(FieldName.create("x"))
		.setCyclic(null);

	DataField right = new DataField()
		.setName(FieldName.create("x"))
		.setCyclic(DataField.Cyclic.ZERO);

	// Initialize a live list instance
	right.getValues();

	assertTrue(ReflectionUtil.equals(left, right));

	Value leftValue = new Value()
		.setValue(0)
		.setProperty(null);

	Value rightValue = new Value()
		.setValue(0)
		.setProperty(Value.Property.VALID);

	right.addValues(rightValue);

	assertFalse(ReflectionUtil.equals(left, right));

	left.addValues(leftValue);

	assertTrue(ReflectionUtil.equals(left, right));

	// Double != Integer
	leftValue.setValue(((Number)rightValue.getValue()).doubleValue());

	assertFalse(ReflectionUtil.equals(left, right));

	leftValue.setValue(rightValue.getValue());

	assertTrue(ReflectionUtil.equals(left, right));

	Value missingValue = new Value()
		.setValue(-999)
		.setProperty(Value.Property.MISSING);

	right.addValues(missingValue);

	assertFalse(ReflectionUtil.equals(left, right));
}