org.dmg.pmml.tree.Node Java Examples

/**
 * @param trainPointData data to run down trees
 * @param model random decision forest model to count on
 * @return map of predictor index to the number of training examples that reached a
 *  node whose decision is based on that feature. The index is among predictors, not all
 *  features, since there are fewer predictors than features. That is, the index will
 *  match the one used in the {@link RandomForestModel}.
 */
private static IntLongHashMap predictorExampleCounts(JavaRDD<? extends LabeledPoint> trainPointData,
                                                     RandomForestModel model) {
  return trainPointData.mapPartitions(data -> {
      IntLongHashMap featureIndexCount = new IntLongHashMap();
      data.forEachRemaining(datum -> {
        double[] featureVector = datum.features().toArray();
        for (DecisionTreeModel tree : model.trees()) {
          org.apache.spark.mllib.tree.model.Node node = tree.topNode();
          // This logic cloned from Node.predict:
          while (!node.isLeaf()) {
            Split split = node.split().get();
            int featureIndex = split.feature();
            // Count feature
            featureIndexCount.addToValue(featureIndex, 1);
            node = nextNode(featureVector, node, split, featureIndex);
          }
        }
      });
      return Collections.singleton(featureIndexCount).iterator();
  }).reduce(RDFUpdate::merge);
}
 

/**
 * @param parent The parent Node of the Node that evaluated to the missing value.
 * @param node The Node that evaluated to the missing value.
 */
private Trail handleMissingValue(Trail trail, Node parent, Node node, EvaluationContext context){
	TreeModel treeModel = getModel();

	TreeModel.MissingValueStrategy missingValueStrategy = treeModel.getMissingValueStrategy();
	switch(missingValueStrategy){
		case NULL_PREDICTION:
			return trail.selectNull();
		case LAST_PREDICTION:
			return trail.selectLastPrediction();
		case DEFAULT_CHILD:
			return handleDefaultChild(trail, parent, context);
		case NONE:
			return null;
		default:
			throw new UnsupportedAttributeException(treeModel, missingValueStrategy);
	}
}
 

private void process(Node node){
	List<Node> path = new ArrayList<>();
	path.add(node);

	Deque<PMMLObject> parents = getParents();
	for(PMMLObject parent : parents){

		if(!(parent instanceof Node)){
			break;
		}

		path.add((Node)parent);
	}

	Collections.reverse(path);

	this.paths.put(node, path);
}
 

private <P extends Number> TreeModel encodeTreeModel(MiningFunction miningFunction, ScoreEncoder<P> scoreEncoder, List<? extends Number> leftDaughter, List<? extends Number> rightDaughter, List<P> nodepred, List<? extends Number> bestvar, List<Double> xbestsplit, Schema schema){
	RGenericVector randomForest = getObject();

	Node root = encodeNode(True.INSTANCE, 0, scoreEncoder, leftDaughter, rightDaughter, bestvar, xbestsplit, nodepred, new CategoryManager(), schema);

	TreeModel treeModel = new TreeModel(miningFunction, ModelUtil.createMiningSchema(schema.getLabel()), root)
		.setMissingValueStrategy(TreeModel.MissingValueStrategy.NULL_PREDICTION)
		.setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT);

	if(this.compact){
		Visitor visitor = new RandomForestCompactor();

		visitor.applyTo(treeModel);
	}

	return treeModel;
}
 

private Trail handleNoTrueChild(Trail trail){
	TreeModel treeModel = getModel();

	TreeModel.NoTrueChildStrategy noTrueChildStrategy = treeModel.getNoTrueChildStrategy();
	switch(noTrueChildStrategy){
		case RETURN_NULL_PREDICTION:
			return trail.selectNull();
		case RETURN_LAST_PREDICTION:
			Node lastPrediction = trail.getLastPrediction();

			// "Return the parent Node only if it specifies a score attribute"
			if(lastPrediction.hasScore()){
				return trail.selectLastPrediction();
			}
			return trail.selectNull();
		default:
			throw new UnsupportedAttributeException(treeModel, noTrueChildStrategy);
	}
}
 

private Boolean evaluateNode(Trail trail, Node node, EvaluationContext context){
	EmbeddedModel embeddedModel = node.getEmbeddedModel();
	if(embeddedModel != null){
		throw new UnsupportedElementException(embeddedModel);
	}

	Predicate predicate = PredicateUtil.ensurePredicate(node);

	// A compound predicate whose boolean operator is "surrogate" represents a special case
	if(predicate instanceof CompoundPredicate){
		CompoundPredicate compoundPredicate = (CompoundPredicate)predicate;

		PredicateUtil.CompoundPredicateResult result = PredicateUtil.evaluateCompoundPredicateInternal(compoundPredicate, context);
		if(result.isAlternative()){
			trail.addMissingLevel();
		}

		return result.getResult();
	} else

	{
		return PredicateUtil.evaluate(predicate, context);
	}
}
 

@Override
public VisitorAction visit(Node node){
	Object defaultChild = node.getDefaultChild();

	if(node.hasNodes()){
		List<Node> children = node.getNodes();

		for(int i = 0, max = children.size(); i < max; i++){
			Node child = children.get(i);

			Object id = child.getId();
			if(id != null && (id).equals(defaultChild)){
				node.setDefaultChild(child);

				break;
			}
		}
	}

	return super.visit(node);
}
 

private TreeModel encodeRegression(RGenericVector frame, RIntegerVector rowNames, RIntegerVector var, RIntegerVector n, int[][] splitInfo, RNumberVector<?> splits, RIntegerVector csplit, Schema schema){
	RNumberVector<?> yval = frame.getNumericElement("yval");

	ScoreEncoder scoreEncoder = new ScoreEncoder(){

		@Override
		public Node encode(Node node, int offset){
			Number score = yval.getValue(offset);
			Number recordCount = n.getValue(offset);

			node
				.setScore(score)
				.setRecordCount(recordCount);

			return node;
		}
	};

	Node root = encodeNode(True.INSTANCE, 1, rowNames, var, n, splitInfo, splits, csplit, scoreEncoder, schema);

	TreeModel treeModel = new TreeModel(MiningFunction.REGRESSION, ModelUtil.createMiningSchema(schema.getLabel()), root);

	return configureTreeModel(treeModel);
}
 

static
public <E extends Estimator & HasTree> TreeModel encodeTreeModel(E estimator, PredicateManager predicateManager, ScoreDistributionManager scoreDistributionManager, MiningFunction miningFunction, Schema schema){
	Tree tree = estimator.getTree();

	int[] leftChildren = tree.getChildrenLeft();
	int[] rightChildren = tree.getChildrenRight();
	int[] features = tree.getFeature();
	double[] thresholds = tree.getThreshold();
	double[] values = tree.getValues();

	Node root = encodeNode(True.INSTANCE, predicateManager, scoreDistributionManager, 0, leftChildren, rightChildren, features, thresholds, values, miningFunction, schema);

	TreeModel treeModel = new TreeModel(miningFunction, ModelUtil.createMiningSchema(schema.getLabel()), root)
		.setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT);

	ClassDictUtil.clearContent(tree);

	return treeModel;
}
 

@Test
public void intern(){
	ScoreDistribution left = new ScoreDistribution("event", 0.33d);
	ScoreDistribution right = new ScoreDistribution("event", 0.33d);

	Node leftChild = createNode(left);
	Node rightChild = createNode(right);

	Node root = new ComplexNode(True.INSTANCE)
		.addNodes(leftChild, rightChild);

	TreeModel treeModel = new TreeModel()
		.setNode(root);

	for(int i = 0; i < 2; i++){
		assertNotSame((leftChild.getScoreDistributions()).get(i), (rightChild.getScoreDistributions()).get(i));
	}

	ScoreDistributionInterner interner = new ScoreDistributionInterner();
	interner.applyTo(treeModel);

	for(int i = 0; i < 2; i++){
		assertSame((leftChild.getScoreDistributions()).get(i), (rightChild.getScoreDistributions()).get(i));
	}
}
 

private MiningModel encodeRegression(RGenericVector ranger, Schema schema){
	RGenericVector forest = ranger.getGenericElement("forest");

	ScoreEncoder scoreEncoder = new ScoreEncoder(){

		@Override
		public Node encode(Node node, Number splitValue, RNumberVector<?> terminalClassCount){
			node.setScore(splitValue);

			return node;
		}
	};

	List<TreeModel> treeModels = encodeForest(forest, MiningFunction.REGRESSION, scoreEncoder, schema);

	MiningModel miningModel = new MiningModel(MiningFunction.REGRESSION, ModelUtil.createMiningSchema(schema.getLabel()))
		.setSegmentation(MiningModelUtil.createSegmentation(Segmentation.MultipleModelMethod.AVERAGE, treeModels));

	return miningModel;
}
 

@Test
public void getProbability(){
	Node node = new LeafNode("ham", null);

	BiMap<String, Node> entityRegistry = ImmutableBiMap.of("1", node);

	NodeScoreDistribution<Double> classification = new NodeScoreDistribution<Double>(new ValueMap<Object, Double>(), node){

		@Override
		public BiMap<String, Node> getEntityRegistry(){
			return entityRegistry;
		}

		@Override
		public List<Node> getDecisionPath(){
			throw new UnsupportedOperationException();
		}
	};

	classification.put("ham", new DoubleValue(0.75d));
	classification.put("spam", new DoubleValue(0.25d));

	assertEquals(ImmutableSet.of("ham", "spam"), classification.getCategories());

	assertEquals((Double)0.75d, classification.getProbability("ham"));
	assertEquals((Double)0.25d, classification.getProbability("spam"));
}
 

@Override
public void exitNode(Node node){
	Predicate predicate = node.getPredicate();

	if(predicate instanceof True){
		Node parentNode = getParentNode();

		if(parentNode == null){
			return;
		}

		initScore(parentNode, node);
		replaceChildWithGrandchildren(parentNode, node);
	}
}
 

private static PMML buildDummyModel() {
  Node node = new CountingLeafNode().setRecordCount(123.0);
  TreeModel treeModel = new TreeModel(MiningFunction.CLASSIFICATION, null, node);
  PMML pmml = PMMLUtils.buildSkeletonPMML();
  pmml.addModels(treeModel);
  return pmml;
}
 

private TreeModel encodeTreeModel(RGenericVector trees, int index, Schema schema){
	RIntegerVector nrnodes = trees.getIntegerElement("nrnodes");
	RIntegerVector ntree = trees.getIntegerElement("ntree");
	RIntegerVector nodeStatus = trees.getIntegerElement("nodeStatus");
	RIntegerVector leftDaughter = trees.getIntegerElement("lDaughter");
	RIntegerVector rightDaughter = trees.getIntegerElement("rDaughter");
	RIntegerVector splitAtt = trees.getIntegerElement("splitAtt");
	RDoubleVector splitPoint = trees.getDoubleElement("splitPoint");
	RIntegerVector nSam = trees.getIntegerElement("nSam");

	int rows = nrnodes.asScalar();
	int columns = ntree.asScalar();

	Node root = encodeNode(
		True.INSTANCE,
		0,
		0,
		FortranMatrixUtil.getColumn(nodeStatus.getValues(), rows, columns, index),
		FortranMatrixUtil.getColumn(nSam.getValues(), rows, columns, index),
		FortranMatrixUtil.getColumn(leftDaughter.getValues(), rows, columns, index),
		FortranMatrixUtil.getColumn(rightDaughter.getValues(), rows, columns, index),
		FortranMatrixUtil.getColumn(splitAtt.getValues(), rows, columns, index),
		FortranMatrixUtil.getColumn(splitPoint.getValues(), rows, columns, index),
		schema
	);

	TreeModel treeModel = new TreeModel(MiningFunction.REGRESSION, ModelUtil.createMiningSchema(schema.getLabel()), root)
		.setSplitCharacteristic(TreeModel.SplitCharacteristic.BINARY_SPLIT);

	return treeModel;
}
 

@Override
public void exitNode(Node node){
	Predicate predicate = node.getPredicate();

	if(predicate instanceof True){
		Node parentNode = getParentNode();

		if(parentNode == null){
			return;
		}

		if((MiningFunction.REGRESSION).equals(this.miningFunction)){
			parentNode.setScore(null);

			initScore(parentNode, node);
			replaceChildWithGrandchildren(parentNode, node);
		} else

		if((MiningFunction.CLASSIFICATION).equals(this.miningFunction)){

			// Replace intermediate nodes, but not terminal nodes
			if(node.hasNodes()){
				replaceChildWithGrandchildren(parentNode, node);
			}
		} else

		{
			throw new IllegalArgumentException();
		}
	}
}
 

private void setNode(Node node){

		if(node == null){
			throw new IllegalArgumentException();
		}

		this.node = node;
	}
 

private <V extends Number> NodeScore<V> createNodeScore(ValueFactory<V> valueFactory, TargetField targetField, Node node){
	Object score = node.getScore();

	Value<V> value;

	if(score instanceof Number){
		value = valueFactory.newValue((Number)score);
	} else

	{
		value = valueFactory.newValue((String)score);
	}

	value = TargetUtil.evaluateRegressionInternal(targetField, value);

	NodeScore<V> result = new NodeScore<V>(value, node){

		@Override
		public BiMap<String, Node> getEntityRegistry(){
			return TreeModelEvaluator.this.getEntityRegistry();
		}

		@Override
		public List<Node> getDecisionPath(){
			return TreeModelEvaluator.this.getPath(getNode());
		}
	};

	return result;
}
 

private Trail handleTrue(Trail trail, Node node, EvaluationContext context){

		// A "true" leaf node
		if(!node.hasNodes()){
			return trail.selectNode(node);
		}

		trail.push(node);

		List<Node> children = node.getNodes();
		for(int i = 0, max = children.size(); i < max; i++){
			Node child = children.get(i);

			Boolean status = evaluateNode(trail, child, context);

			if(status == null){
				Trail destination = handleMissingValue(trail, node, child, context);

				if(destination != null){
					return destination;
				}
			} else

			if(status.booleanValue()){
				return handleTrue(trail, child, context);
			}
		}

		// A "true" non-leaf node
		return handleNoTrueChild(trail);
	}
 

/**
 * @param trainPointData data to run down trees
 * @param model random decision forest model to count on
 * @return maps of node IDs to the count of training examples that reached that node, one
 *  per tree in the model
 * @see #predictorExampleCounts(JavaRDD,RandomForestModel)
 */
private static List<IntLongHashMap> treeNodeExampleCounts(JavaRDD<? extends LabeledPoint> trainPointData,
                                                          RandomForestModel model) {
  return trainPointData.mapPartitions(data -> {
      DecisionTreeModel[] trees = model.trees();
      List<IntLongHashMap> treeNodeIDCounts = IntStream.range(0, trees.length).
          mapToObj(i -> new IntLongHashMap()).collect(Collectors.toList());
      data.forEachRemaining(datum -> {
        double[] featureVector = datum.features().toArray();
        for (int i = 0; i < trees.length; i++) {
          DecisionTreeModel tree = trees[i];
          IntLongHashMap nodeIDCount = treeNodeIDCounts.get(i);
          org.apache.spark.mllib.tree.model.Node node = tree.topNode();
          // This logic cloned from Node.predict:
          while (!node.isLeaf()) {
            // Count node ID
            nodeIDCount.addToValue(node.id(), 1);
            Split split = node.split().get();
            int featureIndex = split.feature();
            node = nextNode(featureVector, node, split, featureIndex);
          }
          nodeIDCount.addToValue(node.id(), 1);
        }
      });
      return Collections.singleton(treeNodeIDCounts).iterator();
    }
  ).reduce((a, b) -> {
      Preconditions.checkArgument(a.size() == b.size());
      for (int i = 0; i < a.size(); i++) {
        merge(a.get(i), b.get(i));
      }
      return a;
    });
}
 

private void setNode(Node node){

		if(node == null){
			throw new IllegalArgumentException();
		}

		this.node = node;
	}
 

@Override
protected void computeResult(DataType dataType){
	Node node = getNode();

	Object result = TypeUtil.parseOrCast(dataType, node.getScore());

	setResult(result);
}
 

private static void checkNode(Node node) {
  assertNotNull(node.getId());
  if (!node.hasScoreDistributions()) {
    // Non-leaf
    List<Node> children = node.getNodes();
    assertEquals(2, children.size());
    Node rightChild = children.get(0);
    Node leftChild = children.get(1);
    assertInstanceOf(leftChild.getPredicate(), True.class);
    assertEquals(node.getRecordCount().intValue(),
                 leftChild.getRecordCount().intValue() + rightChild.getRecordCount().intValue());
    assertEquals(node.getId() + "+", rightChild.getId());
    assertEquals(node.getId() + "-", leftChild.getId());
    checkNode(rightChild);
    checkNode(leftChild);
  } else {
    // Leaf
    List<ScoreDistribution> scoreDists = node.getScoreDistributions();
    int numDists = scoreDists.size();
    assertRange(numDists, 1, 2);
    ScoreDistribution first = scoreDists.get(0);
    if (numDists == 1) {
      assertEquals(1.0, first.getConfidence().doubleValue());
    } else {
      assertGreater(first.getConfidence().doubleValue(), 0.0);
      assertLess(first.getConfidence().doubleValue(), 1.0);
      ScoreDistribution second = scoreDists.get(1);
      assertGreater(second.getConfidence().doubleValue(), 0.0);
      assertLess(second.getConfidence().doubleValue(), 1.0);
    }
  }
}
 

public static PMML buildDummyModel() {
  Node node = new CountingLeafNode().setRecordCount(123.0);
  TreeModel treeModel = new TreeModel(MiningFunction.CLASSIFICATION, null, node);
  PMML pmml = PMMLUtils.buildSkeletonPMML();
  pmml.addModels(treeModel);
  return pmml;
}
 

@Override
public VisitorAction visit(Node node){

	if(node.hasNodes()){
		filterAll(node.getNodes());
	}

	return super.visit(node);
}
 

@Override
public VisitorAction visit(Node node){

	if(!node.hasNodes()){
		process(node);
	}

	return super.visit(node);
}
 

@Override
protected <V extends Number> Map<FieldName, ?> evaluateRegression(ValueFactory<V> valueFactory, EvaluationContext context){
	TargetField targetField = getTargetField();

	Trail trail = new Trail();

	Node node = evaluateTree(trail, context);
	if(node == null){
		return TargetUtil.evaluateRegressionDefault(valueFactory, targetField);
	}

	NodeScore<V> result = createNodeScore(valueFactory, targetField, node);

	return TargetUtil.evaluateRegression(targetField, result);
}
 

@Override
public VisitorAction visit(Node node){

	if(node.hasScore()){
		Object score = node.getScore();

		if(score instanceof String){
			score = parseScore(score);

			node.setScore(score);
		}
	}

	return super.visit(node);
}
 

@Override
public BiMap<String, Node> getEntityRegistry(){

	if(this.entityRegistry == null){
		this.entityRegistry = getValue(TreeModelEvaluator.entityCache);
	}

	return this.entityRegistry;
}
 

public TreeModelEvaluator(PMML pmml, TreeModel treeModel){
	super(pmml, treeModel);

	Node root = treeModel.getNode();
	if(root == null){
		throw new MissingElementException(treeModel, PMMLElements.TREEMODEL_NODE);
	}
}