org.dmg.pmml.clustering.Cluster Java Examples

@Override
public ClusteringModel encodeModel(Schema schema){
	KMeansModel model = getTransformer();

	List<Cluster> clusters = new ArrayList<>();

	Vector[] clusterCenters = model.clusterCenters();
	for(int i = 0; i < clusterCenters.length; i++){
		Cluster cluster = new Cluster(PMMLUtil.createRealArray(VectorUtil.toList(clusterCenters[i])))
			.setId(String.valueOf(i));

		clusters.add(cluster);
	}

	ComparisonMeasure comparisonMeasure = new ComparisonMeasure(ComparisonMeasure.Kind.DISTANCE, new SquaredEuclidean())
		.setCompareFunction(CompareFunction.ABS_DIFF);

	return new ClusteringModel(MiningFunction.CLUSTERING, ClusteringModel.ModelClass.CENTER_BASED, clusters.size(), ModelUtil.createMiningSchema(schema.getLabel()), comparisonMeasure, ClusteringModelUtil.createClusteringFields(schema.getFeatures()), clusters);
}
 

@Override
public ClusteringModel encodeModel(Schema schema){
	int[] shape = getClusterCentersShape();

	int numberOfClusters = shape[0];
	int numberOfFeatures = shape[1];

	List<? extends Number> clusterCenters = getClusterCenters();
	List<Integer> labels = getLabels();

	Multiset<Integer> labelCounts = HashMultiset.create();

	if(labels != null){
		labelCounts.addAll(labels);
	}

	List<Cluster> clusters = new ArrayList<>();

	for(int i = 0; i < numberOfClusters; i++){
		Cluster cluster = new Cluster(PMMLUtil.createRealArray(CMatrixUtil.getRow(clusterCenters, numberOfClusters, numberOfFeatures, i)))
			.setId(String.valueOf(i))
			.setSize((labelCounts.size () > 0 ? labelCounts.count(i) : null));

		clusters.add(cluster);
	}

	ComparisonMeasure comparisonMeasure = new ComparisonMeasure(ComparisonMeasure.Kind.DISTANCE, new SquaredEuclidean())
		.setCompareFunction(CompareFunction.ABS_DIFF);

	ClusteringModel clusteringModel = new ClusteringModel(MiningFunction.CLUSTERING, ClusteringModel.ModelClass.CENTER_BASED, numberOfClusters, ModelUtil.createMiningSchema(schema.getLabel()), comparisonMeasure, ClusteringModelUtil.createClusteringFields(schema.getFeatures()), clusters)
		.setOutput(ClusteringModelUtil.createOutput(FieldName.create("Cluster"), DataType.DOUBLE, clusters));

	return clusteringModel;
}
 

@Override
public Model encodeModel(Schema schema){
	RGenericVector kmeans = getObject();

	RDoubleVector centers = kmeans.getDoubleElement("centers");
	RIntegerVector size = kmeans.getIntegerElement("size");

	RIntegerVector centersDim = centers.dim();

	int rows = centersDim.getValue(0);
	int columns = centersDim.getValue(1);

	List<Cluster> clusters = new ArrayList<>();

	RStringVector rowNames = centers.dimnames(0);
	for(int i = 0; i < rowNames.size(); i++){
		Cluster cluster = new Cluster(PMMLUtil.createRealArray(FortranMatrixUtil.getRow(centers.getValues(), rows, columns, i)))
			.setId(String.valueOf(i + 1))
			.setName(rowNames.getValue(i))
			.setSize(size.getValue(i));

		clusters.add(cluster);
	}

	ComparisonMeasure comparisonMeasure = new ComparisonMeasure(ComparisonMeasure.Kind.DISTANCE, new SquaredEuclidean())
		.setCompareFunction(CompareFunction.ABS_DIFF);

	ClusteringModel clusteringModel = new ClusteringModel(MiningFunction.CLUSTERING, ClusteringModel.ModelClass.CENTER_BASED, rows, ModelUtil.createMiningSchema(schema.getLabel()), comparisonMeasure, ClusteringModelUtil.createClusteringFields(schema.getFeatures()), clusters)
		.setOutput(ClusteringModelUtil.createOutput(FieldName.create("cluster"), DataType.DOUBLE, clusters));

	return clusteringModel;
}
 

private ClusteringModel pmmlClusteringModel(KMeansModel model,
                                            Map<Integer,Long> clusterSizesMap) {
  Vector[] clusterCenters = model.clusterCenters();

  List<ClusteringField> clusteringFields = new ArrayList<>();
  for (int i = 0; i < inputSchema.getNumFeatures(); i++) {
    if (inputSchema.isActive(i)) {
      FieldName fieldName = FieldName.create(inputSchema.getFeatureNames().get(i));
      ClusteringField clusteringField =
          new ClusteringField(fieldName).setCenterField(ClusteringField.CenterField.TRUE);
      clusteringFields.add(clusteringField);
    }
  }

  List<Cluster> clusters = new ArrayList<>(clusterCenters.length);
  for (int i = 0; i < clusterCenters.length; i++) {
    clusters.add(new Cluster().setId(Integer.toString(i))
                     .setSize(clusterSizesMap.get(i).intValue())
                     .setArray(AppPMMLUtils.toArray(clusterCenters[i].toArray())));
  }

  return new ClusteringModel(
      MiningFunction.CLUSTERING,
      ClusteringModel.ModelClass.CENTER_BASED,
      clusters.size(),
      AppPMMLUtils.buildMiningSchema(inputSchema),
      new ComparisonMeasure(ComparisonMeasure.Kind.DISTANCE, new SquaredEuclidean()),
      clusteringFields,
      clusters);
}
 

@Override
public BiMap<String, Cluster> getEntityRegistry(){

	if(this.entityRegistry == null){
		this.entityRegistry = getValue(ClusteringModelEvaluator.entityCache);
	}

	return this.entityRegistry;
}
 

private <V extends Number> ClusterAffinityDistribution<V> evaluateDistance(ValueFactory<V> valueFactory, ComparisonMeasure comparisonMeasure, List<ClusteringField> clusteringFields, List<FieldValue> values){
	ClusteringModel clusteringModel = getModel();

	List<Cluster> clusters = clusteringModel.getClusters();

	Value<V> adjustment;

	MissingValueWeights missingValueWeights = clusteringModel.getMissingValueWeights();
	if(missingValueWeights != null){
		Array array = missingValueWeights.getArray();

		List<? extends Number> adjustmentValues = ArrayUtil.asNumberList(array);
		if(values.size() != adjustmentValues.size()){
			throw new InvalidElementException(missingValueWeights);
		}

		adjustment = MeasureUtil.calculateAdjustment(valueFactory, values, adjustmentValues);
	} else

	{
		adjustment = MeasureUtil.calculateAdjustment(valueFactory, values);
	}

	ClusterAffinityDistribution<V> result = createClusterAffinityDistribution(Classification.Type.DISTANCE, clusters);

	for(Cluster cluster : clusters){
		List<FieldValue> clusterValues = CacheUtil.getValue(cluster, ClusteringModelEvaluator.clusterValueCache);

		if(values.size() != clusterValues.size()){
			throw new InvalidElementException(cluster);
		}

		Value<V> distance = MeasureUtil.evaluateDistance(valueFactory, comparisonMeasure, clusteringFields, values, clusterValues, adjustment);

		result.put(cluster, distance);
	}

	return result;
}
 

private <V extends Number> ClusterAffinityDistribution<V> createClusterAffinityDistribution(Classification.Type type, List<Cluster> clusters){
	ClusterAffinityDistribution<V> result = new ClusterAffinityDistribution<V>(type, new ValueMap<String, V>(2 * clusters.size())){

		@Override
		public BiMap<String, Cluster> getEntityRegistry(){
			return ClusteringModelEvaluator.this.getEntityRegistry();
		}
	};

	return result;
}
 

@Override
public List<FieldValue> load(Cluster cluster){
	Array array = cluster.getArray();

	List<? extends Number> values = ArrayUtil.asNumberList(array);

	return ImmutableList.copyOf(Lists.transform(values, value -> FieldValueUtil.create(TypeInfos.CONTINUOUS_DOUBLE, value)));
}
 

public static PMML buildDummyClusteringModel() {
  PMML pmml = PMMLUtils.buildSkeletonPMML();

  List<DataField> dataFields = new ArrayList<>();
  dataFields.add(new DataField(FieldName.create("x"), OpType.CONTINUOUS, DataType.DOUBLE));
  dataFields.add(new DataField(FieldName.create("y"), OpType.CONTINUOUS, DataType.DOUBLE));
  DataDictionary dataDictionary =
      new DataDictionary(dataFields).setNumberOfFields(dataFields.size());
  pmml.setDataDictionary(dataDictionary);

  List<MiningField> miningFields = new ArrayList<>();
  MiningField xMF = new MiningField(FieldName.create("x"))
      .setOpType(OpType.CONTINUOUS).setUsageType(MiningField.UsageType.ACTIVE);
  miningFields.add(xMF);
  MiningField yMF = new MiningField(FieldName.create("y"))
      .setOpType(OpType.CONTINUOUS).setUsageType(MiningField.UsageType.ACTIVE);
  miningFields.add(yMF);
  MiningSchema miningSchema = new MiningSchema(miningFields);

  List<ClusteringField> clusteringFields = new ArrayList<>();
  clusteringFields.add(new ClusteringField(
      FieldName.create("x")).setCenterField(ClusteringField.CenterField.TRUE));
  clusteringFields.add(new ClusteringField(
      FieldName.create("y")).setCenterField(ClusteringField.CenterField.TRUE));

  List<Cluster> clusters = new ArrayList<>();
  clusters.add(new Cluster().setId("0").setSize(1).setArray(AppPMMLUtils.toArray(1.0, 0.0)));
  clusters.add(new Cluster().setId("1").setSize(2).setArray(AppPMMLUtils.toArray(2.0, -1.0)));
  clusters.add(new Cluster().setId("2").setSize(3).setArray(AppPMMLUtils.toArray(-1.0, 0.0)));

  pmml.addModels(new ClusteringModel(
      MiningFunction.CLUSTERING,
      ClusteringModel.ModelClass.CENTER_BASED,
      clusters.size(),
      miningSchema,
      new ComparisonMeasure(ComparisonMeasure.Kind.DISTANCE, new SquaredEuclidean()),
      clusteringFields,
      clusters));

  return pmml;
}
 

@Test
public void testKMeans() throws Exception {
  Path tempDir = getTempDir();
  Path dataDir = tempDir.resolve("data");
  Path modelDir = tempDir.resolve("model");

  Map<String,Object> overlayConfig = new HashMap<>();
  overlayConfig.put("oryx.batch.update-class", KMeansUpdate.class.getName());
  ConfigUtils.set(overlayConfig, "oryx.batch.storage.data-dir", dataDir);
  ConfigUtils.set(overlayConfig, "oryx.batch.storage.model-dir", modelDir);
  overlayConfig.put("oryx.batch.streaming.generation-interval-sec", GEN_INTERVAL_SEC);
  overlayConfig.put("oryx.kmeans.hyperparams.k", NUM_CLUSTERS);
  overlayConfig.put("oryx.kmeans.iterations", 5);
  overlayConfig.put("oryx.input-schema.num-features", NUM_FEATURES);
  overlayConfig.put("oryx.input-schema.categorical-features", "[]");
  overlayConfig.put("oryx.kmeans.evaluation-strategy", EVALUATION_STRATEGY);

  Config config = ConfigUtils.overlayOn(overlayConfig, getConfig());

  startMessaging();

  List<KeyMessage<String, String>> updates = startServerProduceConsumeTopics(
      config,
      new RandomKMeansDataGenerator(NUM_FEATURES),
      DATA_TO_WRITE,
      WRITE_INTERVAL_MSEC);

  List<Path> modelInstanceDirs = IOUtils.listFiles(modelDir, "*");

  int generations = modelInstanceDirs.size();
  checkIntervals(generations, DATA_TO_WRITE, WRITE_INTERVAL_MSEC, GEN_INTERVAL_SEC);

  for (Path modelInstanceDir : modelInstanceDirs) {
    Path modelFile = modelInstanceDir.resolve(MLUpdate.MODEL_FILE_NAME);
    assertNonEmpty(modelFile);
    PMMLUtils.read(modelFile); // Shouldn't throw exception
  }

  InputSchema schema = new InputSchema(config);

  for (KeyMessage<String,String> km : updates) {

    String type = km.getKey();
    String value = km.getMessage();

    assertContains(Arrays.asList("MODEL", "MODEL-REF"), type);
    PMML pmml = AppPMMLUtils.readPMMLFromUpdateKeyMessage(type, value, null);
    assertNotNull(pmml);
    checkHeader(pmml.getHeader());

    checkDataDictionary(schema, pmml.getDataDictionary());

    Model rootModel = pmml.getModels().get(0);

    ClusteringModel clusteringModel = (ClusteringModel) rootModel;

    // Check if Basic hyperparameters match
    assertEquals(NUM_CLUSTERS, clusteringModel.getNumberOfClusters().intValue());
    assertEquals(NUM_CLUSTERS, clusteringModel.getClusters().size());
    assertEquals(NUM_FEATURES, clusteringModel.getClusteringFields().size());
    assertEquals(ComparisonMeasure.Kind.DISTANCE,
                 clusteringModel.getComparisonMeasure().getKind());
    assertEquals(NUM_FEATURES, clusteringModel.getClusters().get(0).getArray().getN().intValue());
    for (Cluster cluster : clusteringModel.getClusters()) {
      assertGreater(cluster.getSize(), 0);
    }
  }
}
 

@Override
public String getDisplayValue(){
	Cluster cluster = getEntity();

	return cluster.getName();
}
 

private <V extends Number> ClusterAffinityDistribution<V> evaluateSimilarity(ValueFactory<V> valueFactory, ComparisonMeasure comparisonMeasure, List<ClusteringField> clusteringFields, List<FieldValue> values){
	ClusteringModel clusteringModel = getModel();

	List<Cluster> clusters = clusteringModel.getClusters();

	ClusterAffinityDistribution<V> result = createClusterAffinityDistribution(Classification.Type.SIMILARITY, clusters);

	BitSet flags = MeasureUtil.toBitSet(values);

	for(Cluster cluster : clusters){
		BitSet clusterFlags = CacheUtil.getValue(cluster, ClusteringModelEvaluator.clusterFlagCache);

		if(flags.size() != clusterFlags.size()){
			throw new InvalidElementException(cluster);
		}

		Value<V> similarity = MeasureUtil.evaluateSimilarity(valueFactory, comparisonMeasure, clusteringFields, flags, clusterFlags);

		result.put(cluster, similarity);
	}

	return result;
}
 

@Override
public BitSet load(Cluster cluster){
	List<FieldValue> values = CacheUtil.getValue(cluster, ClusteringModelEvaluator.clusterValueCache);

	return MeasureUtil.toBitSet(values);
}
 

@Override
public BiMap<String, Cluster> load(ClusteringModel clusteringModel){
	return EntityUtil.buildBiMap(clusteringModel.getClusters());
}
 

public void put(Cluster entity, Value<V> value){
	BiMap<String, Cluster> entityRegistry = getEntityRegistry();

	String id = EntityUtil.getId(entity, entityRegistry);

	put(entity, id, value);
}