org.dmg.pmml.PMML Java Examples

@Test
public void getValue(){
	PMML pmml = new CustomPMML();
	pmml.setVersion(Version.PMML_4_4.getVersion());

	assertEquals("4.4", pmml.getVersion());
	assertEquals("4.4", pmml.getBaseVersion());

	Field versionField = PMMLAttributes.PMML_VERSION;
	Field baseVersionField = PMMLAttributes.PMML_BASEVERSION;

	assertEquals("4.4", ReflectionUtil.getFieldValue(versionField, pmml));
	assertEquals((String)null, ReflectionUtil.getFieldValue(baseVersionField, pmml));

	Method versionGetterMethod = ReflectionUtil.getGetterMethod(versionField);
	Method baseVersionGetterMethod = ReflectionUtil.getGetterMethod(baseVersionField);

	assertEquals("4.4", ReflectionUtil.getGetterMethodValue(versionGetterMethod, pmml));
	assertEquals("4.4", ReflectionUtil.getGetterMethodValue(baseVersionGetterMethod, pmml));
}
 

static
private Constructor<?> findConstructor(Class<?> serviceProviderClass) throws NoSuchMethodException {
	Constructor<?>[] constructors = serviceProviderClass.getConstructors();

	for(Constructor<?> constructor : constructors){
		Parameter[] parameters = constructor.getParameters();

		if(parameters.length != 2){
			continue;
		}

		Parameter pmmlParameter = parameters[0];
		Parameter modelParameter = parameters[1];

		if((PMML.class).isAssignableFrom(pmmlParameter.getType()) && (Model.class).isAssignableFrom(modelParameter.getType())){
			return constructor;
		}
	}

	throw new NoSuchMethodException();
}
 

static
public PMML readPMML(File file, boolean acceptServiceJar) throws Exception {

	if(acceptServiceJar){

		if(isServiceJar(file, PMML.class)){
			URL url = (file.toURI()).toURL();

			return PMMLUtil.load(url);
		}
	}

	try(InputStream is = new FileInputStream(file)){
		return PMMLUtil.unmarshal(is);
	}
}
 

@Override
public Pair<String,String> generate(int id, RandomGenerator random) {
  if (id % 10 == 0) {
    PMML pmml = PMMLUtils.buildSkeletonPMML();
    AppPMMLUtils.addExtension(pmml, "features", 2);
    AppPMMLUtils.addExtension(pmml, "implicit", true);
    AppPMMLUtils.addExtensionContent(pmml, "XIDs", X.keySet());
    AppPMMLUtils.addExtensionContent(pmml, "YIDs", Y.keySet());
    return new Pair<>("MODEL", PMMLUtils.toString(pmml));
  } else {
    int xOrYID = id % 10;
    String xOrYIDString = ALSUtilsTest.idToStringID(id);
    String message;
    boolean isX = xOrYID >= 6;
    if (isX) {
      message = TextUtils.joinJSON(Arrays.asList(
          "X", xOrYIDString, X.get(xOrYIDString), A.get(xOrYIDString)));
    } else {
      message = TextUtils.joinJSON(Arrays.asList(
          "Y", xOrYIDString, Y.get(xOrYIDString), At.get(xOrYIDString)));
    }
    return new Pair<>("UP", message);
  }
}
 

@Test
public void mixedContent() throws Exception {
	PMML pmml = ResourceUtil.unmarshal(getClass());

	List<?> content = ExtensionUtil.getContent(pmml);

	assertEquals(5, content.size());

	assertEquals("First text value", content.get(0));
	assertEquals(Arrays.asList("First extension"), getDeepContent(content.get(1)));
	assertEquals("Second text value", content.get(2));
	assertEquals(Arrays.asList("Second extension"), getDeepContent(content.get(3)));
	assertEquals("Third text value", content.get(4));

	try {
		SerializationUtil.clone(pmml);

		fail();
	} catch(NotSerializableException nse){
		// Ignored
	}

	pmml.accept(new LocatorTransformer());

	SerializationUtil.clone(pmml);
}
 

@Override
public void writeTo(Model model, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, Object> httpHeaders, OutputStream entityStream) throws IOException {
	Evaluator evaluator = model.getEvaluator();

	HasPMML hasPMML = (HasPMML)evaluator;

	httpHeaders.putSingle(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_XML_TYPE.withCharset("UTF-8"));
	httpHeaders.putSingle(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=model.pmml.xml"); // XXX

	PMML pmml = hasPMML.getPMML();

	try {
		Result result = new StreamResult(entityStream);

		Marshaller marshaller = JAXBUtil.createMarshaller();

		marshaller.marshal(pmml, result);
	} catch(JAXBException je){
		throw new InternalServerErrorException(je);
	}
}
 

/**
 * Validates that the encoded PMML model received matches expected schema.
 *
 * @param pmml {@link PMML} encoding of KMeans Clustering
 * @param schema expected schema attributes of KMeans Clustering
 */
public static void validatePMMLVsSchema(PMML pmml, InputSchema schema) {
  List<Model> models = pmml.getModels();
  Preconditions.checkArgument(models.size() == 1,
      "Should have exactly one model, but had %s", models.size());

  Model model = models.get(0);
  Preconditions.checkArgument(model instanceof ClusteringModel);
  Preconditions.checkArgument(model.getMiningFunction() == MiningFunction.CLUSTERING);

  DataDictionary dictionary = pmml.getDataDictionary();
  Preconditions.checkArgument(
      schema.getFeatureNames().equals(AppPMMLUtils.getFeatureNames(dictionary)),
      "Feature names in schema don't match names in PMML");

  MiningSchema miningSchema = model.getMiningSchema();
  Preconditions.checkArgument(schema.getFeatureNames().equals(
      AppPMMLUtils.getFeatureNames(miningSchema)));

}
 

private void declareGlobalFields(PMML pmml, boolean transformations){
	List<Field<?>> scope = this.scopes.get(pmml);

	if(scope != null){
		scope.clear();
	}

	DataDictionary dataDictionary = pmml.getDataDictionary();
	if(dataDictionary != null && dataDictionary.hasDataFields()){
		declareFields(pmml, dataDictionary.getDataFields());
	}

	TransformationDictionary transformationDictionary = pmml.getTransformationDictionary();
	if(transformations && (transformationDictionary != null && transformationDictionary.hasDerivedFields())){
		declareFields(pmml, transformationDictionary.getDerivedFields());
	}
}
 

public NaiveBayesModelEvaluator(PMML pmml, NaiveBayesModel naiveBayesModel){
	super(pmml, naiveBayesModel);

	BayesInputs bayesInputs = naiveBayesModel.getBayesInputs();
	if(bayesInputs == null){
		throw new MissingElementException(naiveBayesModel, PMMLElements.NAIVEBAYESMODEL_BAYESINPUTS);
	} // End if

	if(!bayesInputs.hasBayesInputs() && !bayesInputs.hasExtensions()){
		throw new MissingElementException(bayesInputs, PMMLElements.BAYESINPUTS_BAYESINPUTS);
	}

	BayesOutput bayesOutput = naiveBayesModel.getBayesOutput();
	if(bayesOutput == null){
		throw new MissingElementException(naiveBayesModel, PMMLElements.NAIVEBAYESMODEL_BAYESOUTPUT);
	}

	TargetValueCounts targetValueCounts = bayesOutput.getTargetValueCounts();
	if(targetValueCounts == null){
		throw new MissingElementException(bayesOutput, PMMLElements.BAYESOUTPUT_TARGETVALUECOUNTS);
	} // End if

	if(!targetValueCounts.hasTargetValueCounts()){
		throw new MissingElementException(targetValueCounts, PMMLElements.TARGETVALUECOUNTS_TARGETVALUECOUNTS);
	}
}
 

static
public Model findModel(PMML pmml, Predicate<Model> predicate, String predicateXPath){

	if(!pmml.hasModels()){
		throw new MissingElementException(MissingElementException.formatMessage(XPathUtil.formatElement(pmml.getClass()) + "/" + predicateXPath), pmml);
	}

	List<Model> models = pmml.getModels();

	Optional<Model> result = models.stream()
		.filter(predicate)
		.findAny();

	if(!result.isPresent()){
		throw new MissingElementException(MissingElementException.formatMessage(XPathUtil.formatElement(pmml.getClass()) + "/" + predicateXPath), pmml);
	}

	return result.get();
}
 

@Test
public void testPreviousPMMLVersion() throws Exception {
  String pmml42 =
      "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n" +
      "<PMML xmlns=\"http://www.dmg.org/PMML-4_2\" version=\"4.2.1\">\n" +
      "    <Header>\n" +
      "        <Application name=\"Oryx\"/>\n" +
      "    </Header>\n" +
      "    <TreeModel functionName=\"classification\">\n" +
      "        <Node recordCount=\"123.0\"/>\n" +
      "    </TreeModel>\n" +
      "</PMML>\n";
  PMML model = PMMLUtils.fromString(pmml42);
  // Actually transforms to latest version:
  assertEquals(PMMLUtils.VERSION, model.getVersion());
}
 

/**
 * @param pmml {@link PMML} model to write
 * @param out stream to write the model to
 * @throws IOException if an error occurs while writing the model to storage
 */
public static void write(PMML pmml, OutputStream out) throws IOException {
  try {
    PMMLUtil.marshal(pmml, out);
  } catch (JAXBException e) {
    throw new IOException(e);
  }
}
 

public SupportVectorMachineModelEvaluator(PMML pmml, SupportVectorMachineModel supportVectorMachineModel){
	super(pmml, supportVectorMachineModel);

	boolean maxWins = supportVectorMachineModel.isMaxWins();
	if(maxWins){
		throw new UnsupportedAttributeException(supportVectorMachineModel, PMMLAttributes.SUPPORTVECTORMACHINEMODEL_MAXWINS, maxWins);
	}

	SupportVectorMachineModel.Representation representation = supportVectorMachineModel.getRepresentation();
	switch(representation){
		case SUPPORT_VECTORS:
			break;
		default:
			throw new UnsupportedAttributeException(supportVectorMachineModel, representation);
	}

	VectorDictionary vectorDictionary = supportVectorMachineModel.getVectorDictionary();
	if(vectorDictionary == null){
		throw new MissingElementException(supportVectorMachineModel, PMMLElements.SUPPORTVECTORMACHINEMODEL_VECTORDICTIONARY);
	}

	VectorFields vectorFields = vectorDictionary.getVectorFields();
	if(vectorFields == null){
		throw new MissingElementException(vectorDictionary, PMMLElements.VECTORDICTIONARY_VECTORFIELDS);
	} // End if

	if(!supportVectorMachineModel.hasSupportVectorMachines()){
		throw new MissingElementException(supportVectorMachineModel, PMMLElements.SUPPORTVECTORMACHINEMODEL_SUPPORTVECTORMACHINES);
	}
}
 

/**
 * @param pmml PMML representation of Clusters
 * @return List of {@link ClusterInfo}
 */
public static List<ClusterInfo> read(PMML pmml) {
  Model model = pmml.getModels().get(0);
  Preconditions.checkArgument(model instanceof ClusteringModel);
  ClusteringModel clusteringModel = (ClusteringModel) model;

  return clusteringModel.getClusters().stream().map(cluster ->
    new ClusterInfo(Integer.parseInt(cluster.getId()),
                    VectorMath.parseVector(
                        TextUtils.parseDelimited(cluster.getArray().getValue().toString(), ' ')),
                    cluster.getSize())
  ).collect(Collectors.toList());
}
 

private static PMML buildDummyModel() {
  Node node = new CountingLeafNode().setRecordCount(123.0);
  TreeModel treeModel = new TreeModel(MiningFunction.CLASSIFICATION, null, node);
  PMML pmml = PMMLUtils.buildSkeletonPMML();
  pmml.addModels(treeModel);
  return pmml;
}
 

public ScorecardEvaluator(PMML pmml, Scorecard scorecard){
	super(pmml, scorecard);

	Characteristics characteristics = scorecard.getCharacteristics();
	if(characteristics == null){
		throw new MissingElementException(scorecard, PMMLElements.SCORECARD_CHARACTERISTICS);
	} // End if

	if(!characteristics.hasCharacteristics()){
		throw new MissingElementException(characteristics, PMMLElements.CHARACTERISTICS_CHARACTERISTICS);
	}
}
 

@Override
public PMML transform(PMML pmml) throws Exception {
	MessageDigest messageDigest = MessageDigest.getInstance(this.algorithm);

	FieldHasher fieldHasher = new FieldHasher(messageDigest);
	fieldHasher.applyTo(pmml);

	Map<FieldName, FieldName> mappings = fieldHasher.getMappings();

	FieldRenamer fieldRenamer = new FieldRenamer(mappings);
	fieldRenamer.applyTo(pmml);

	return pmml;
}
 

public void marshalPMML(PMML pmml, File file) throws JAXBException, IOException {
	Marshaller marshaller = createMarshaller();

	try(OutputStream os = new FileOutputStream(file)){
		Result result = new StreamResult(os);

		marshaller.marshal(pmml, result);
	}
}
 

@Override
public void publishAdditionalModelData(JavaSparkContext sparkContext,
                                       PMML pmml,
                                       JavaRDD<String> newData,
                                       JavaRDD<String> pastData,
                                       Path modelParentPath,
                                       TopicProducer<String, String> modelUpdateTopic) {
  // Send item updates first, before users. That way, user-based endpoints like /recommend
  // may take longer to not return 404, but when they do, the result will be more complete.
  log.info("Sending item / Y data as model updates");
  String yPathString = AppPMMLUtils.getExtensionValue(pmml, "Y");
  JavaPairRDD<String,float[]> productRDD = readFeaturesRDD(sparkContext, new Path(modelParentPath, yPathString));

  String updateBroker = modelUpdateTopic.getUpdateBroker();
  String topic = modelUpdateTopic.getTopic();

  // For now, there is no use in sending known users for each item
  productRDD.foreachPartition(new EnqueueFeatureVecsFn("Y", updateBroker, topic));

  log.info("Sending user / X data as model updates");
  String xPathString = AppPMMLUtils.getExtensionValue(pmml, "X");
  JavaPairRDD<String,float[]> userRDD = readFeaturesRDD(sparkContext, new Path(modelParentPath, xPathString));

  if (noKnownItems) {
    userRDD.foreachPartition(new EnqueueFeatureVecsFn("X", updateBroker, topic));
  } else {
    log.info("Sending known item data with model updates");
    JavaRDD<String[]> allData =
        (pastData == null ? newData : newData.union(pastData)).map(MLFunctions.PARSE_FN);
    JavaPairRDD<String,Collection<String>> knownItems = knownsRDD(allData, true);
    userRDD.join(knownItems).foreachPartition(
        new EnqueueFeatureVecsAndKnownItemsFn("X", updateBroker, topic));
  }
}
 

@Test
public void testExtensionValue() throws Exception {
  PMML model = buildDummyModel();
  assertNull(AppPMMLUtils.getExtensionValue(model, "foo"));
  AppPMMLUtils.addExtension(model, "foo", "bar");

  PMML reserializedModel = PMMLUtils.fromString(PMMLUtils.toString(model));
  assertEquals("bar", AppPMMLUtils.getExtensionValue(reserializedModel, "foo"));
}
 

@Override
public VisitorAction visit(TransformationDictionary transformationDictionary){
	PMML pmml = (PMML)getParent();

	if(transformationDictionary.hasDerivedFields()){
		declareGlobalFields(pmml, false);

		suppressFields(transformationDictionary);
	}

	return super.visit(transformationDictionary);
}
 

@Test
public void testReadPMMLFromMessage() throws Exception {
  PMML pmml = PMMLUtils.buildSkeletonPMML();
  String pmmlString = PMMLUtils.toString(pmml);
  assertEquals(PMMLUtils.VERSION, AppPMMLUtils.readPMMLFromUpdateKeyMessage(
      "MODEL", pmmlString, null).getVersion());

  Path pmmlPath = getTempDir().resolve("out.pmml");
  Files.write(pmmlPath, Collections.singleton(pmmlString));
  assertEquals(PMMLUtils.VERSION, AppPMMLUtils.readPMMLFromUpdateKeyMessage(
      "MODEL-REF", pmmlPath.toAbsolutePath().toString(), null).getVersion());

  assertNull(AppPMMLUtils.readPMMLFromUpdateKeyMessage("MODEL-REF", "no-such-path", null));
}
 

@Test
public void testClustering() {
  PMML pmml = buildDummyClusteringModel();
  Map<String,Object> overlayConfig = new HashMap<>();
  overlayConfig.put("oryx.input-schema.feature-names", "[\"x\",\"y\"]");
  overlayConfig.put("oryx.input-schema.num-features", 2);
  overlayConfig.put("oryx.input-schema.categorical-features", "[]");
  Config config = ConfigUtils.overlayOn(overlayConfig, ConfigUtils.getDefault());
  InputSchema schema = new InputSchema(config);
  KMeansPMMLUtils.validatePMMLVsSchema(pmml, schema);
}
 

@Override
public PMML transform(PMML pmml) throws Exception {

	if(this.summary){
		printSummary(pmml);
	}

	List<String> visitorClasses = this.visitorClasses;
	for(String visitorClass : visitorClasses){
		Class<?> clazz = Class.forName(visitorClass);

		long begin = System.currentTimeMillis();

		Visitor visitor = (Visitor)clazz.newInstance();
		visitor.applyTo(pmml);

		long end = System.currentTimeMillis();

		System.out.println("Applied " + clazz.getName() + " in " + (end - begin) + " ms.");

		if(this.summary){
			printSummary(pmml);
		}
	}

	return pmml;
}
 

/**
 * @see #validatePMML(PMML)
 */
@Override
public PMML getPMML() throws Exception {
	PMML pmml = super.getPMML();

	LocatorTransformer locatorTransformer = new LocatorTransformer();
	locatorTransformer.applyTo(pmml);

	validatePMML(pmml);

	return pmml;
}
 

@Test
public void testReadClassification() {
  PMML pmml = buildDummyClassificationModel();
  Pair<DecisionForest,CategoricalValueEncodings> forestAndEncodings = RDFPMMLUtils.read(pmml);
  DecisionForest forest = forestAndEncodings.getFirst();
  assertEquals(1, forest.getTrees().length);
  assertArrayEquals(new double[] { 1.0 }, forest.getWeights());
  assertArrayEquals(new double[] { 0.5, 0.0 }, forest.getFeatureImportances());
  CategoricalValueEncodings encodings = forestAndEncodings.getSecond();
  assertEquals(2, encodings.getValueCount(0));
  assertEquals(2, encodings.getValueCount(1));
}
 

@Test
public void testReadClassificationForest() {
  PMML pmml = buildDummyClassificationModel(3);
  Pair<DecisionForest,CategoricalValueEncodings> forestAndEncodings = RDFPMMLUtils.read(pmml);
  DecisionForest forest = forestAndEncodings.getFirst();
  assertEquals(3, forest.getTrees().length);
}
 

/**
 * @param sparkContext    active Spark Context
 * @param trainData       training data on which to build a model
 * @param hyperParameters ordered list of hyper parameter values to use in building model
 * @param candidatePath   directory where additional model files can be written
 * @return a {@link PMML} representation of a model trained on the given data
 */
@Override
public PMML buildModel(JavaSparkContext sparkContext,
                       JavaRDD<String> trainData,
                       List<?> hyperParameters,
                       Path candidatePath) {
  int numClusters = (Integer) hyperParameters.get(0);
  Preconditions.checkArgument(numClusters > 1);
  log.info("Building KMeans Model with {} clusters", numClusters);

  JavaRDD<Vector> trainingData = parsedToVectorRDD(trainData.map(MLFunctions.PARSE_FN));
  KMeansModel kMeansModel = KMeans.train(trainingData.rdd(), numClusters, maxIterations, initializationStrategy);

  return kMeansModelToPMML(kMeansModel, fetchClusterCountsFromModel(trainingData, kMeansModel));
}
 

static
public byte[] upgradeToLatest(byte[] bytes) throws IOException, JAXBException, SAXException {
	ByteArrayOutputStream result = new ByteArrayOutputStream();

	try(InputStream is = new ByteArrayInputStream(bytes)){
		PMML pmml = PMMLUtil.unmarshal(is);

		PMMLUtil.marshal(pmml, result);
	}

	return result.toByteArray();
}
 

@Test
public void copyState(){
	PMML pmml = new PMML(Version.PMML_4_4.getVersion(), new Header(), new DataDictionary());

	// Initialize a live list instance
	pmml.getModels();

	CustomPMML customPmml = new CustomPMML();

	ReflectionUtil.copyState(pmml, customPmml);

	assertSame(pmml.getVersion(), customPmml.getVersion());
	assertSame(pmml.getHeader(), customPmml.getHeader());
	assertSame(pmml.getDataDictionary(), customPmml.getDataDictionary());

	assertFalse(pmml.hasModels());
	assertFalse(customPmml.hasModels());

	pmml.addModels(new RegressionModel());

	assertTrue(pmml.hasModels());
	assertTrue(customPmml.hasModels());

	assertSame(pmml.getModels(), customPmml.getModels());

	try {
		ReflectionUtil.copyState(customPmml, pmml);

		fail();
	} catch(IllegalArgumentException iae){
		// Ignored
	}
}