org.apache.mahout.math.DenseVector Java Examples

/**
 * Initialize synthetic data using 4 clusters dC units from origin having 4 representative points dP from each center
 * @param dC a double cluster center offset
 * @param dP a double representative point offset
* @param measure the DistanceMeasure
 */
private void initData(double dC, double dP, DistanceMeasure measure) {
  clusters = new HashMap<Integer, Cluster>();
  clusters.put(1, new Canopy(new DenseVector(new double[] { -dC, -dC }), 1, measure));
  clusters.put(3, new Canopy(new DenseVector(new double[] { -dC, dC }), 3, measure));
  clusters.put(5, new Canopy(new DenseVector(new double[] { dC, dC }), 5, measure));
  clusters.put(7, new Canopy(new DenseVector(new double[] { dC, -dC }), 7, measure));
  representativePoints = new HashMap<Integer, List<VectorWritable>>();
  for (Cluster cluster : clusters.values()) {
    List<VectorWritable> points = new ArrayList<VectorWritable>();
    representativePoints.put(cluster.getId(), points);
    points.add(new VectorWritable(cluster.getCenter().clone()));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { dP, dP }))));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { dP, -dP }))));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { -dP, -dP }))));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { -dP, dP }))));
  }
}
 

/**
 * Initialize synthetic data using 4 clusters dC units from origin having 4 representative points dP from each center
 * @param dC a double cluster center offset
 * @param dP a double representative point offset
* @param measure TODO
 */
private void initData(double dC, double dP, DistanceMeasure measure) {
  clusters = new HashMap<Integer, Cluster>();
  clusters.put(1, new Canopy(new DenseVector(new double[] { -dC, -dC }), 1, measure));
  clusters.put(3, new Canopy(new DenseVector(new double[] { -dC, dC }), 3, measure));
  clusters.put(5, new Canopy(new DenseVector(new double[] { dC, dC }), 5, measure));
  clusters.put(7, new Canopy(new DenseVector(new double[] { dC, -dC }), 7, measure));
  representativePoints = new HashMap<Integer, List<VectorWritable>>();
  for (Cluster cluster : clusters.values()) {
    List<VectorWritable> points = new ArrayList<VectorWritable>();
    representativePoints.put(cluster.getId(), points);
    points.add(new VectorWritable(cluster.getCenter().clone()));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { dP, dP }))));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { dP, -dP }))));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { -dP, -dP }))));
    points.add(new VectorWritable(cluster.getCenter().plus(new DenseVector(new double[] { -dP, dP }))));
  }
}
 

@Test
public void testDirichlet() throws Exception {
  ModelDistribution<VectorWritable> modelDistribution =
      new GaussianClusterDistribution(new VectorWritable(new DenseVector(2)));
  DirichletDriver.runJob(getTestTempDirPath("testdata"),
                         getTestTempDirPath("output"),
                         modelDistribution,
                         15,
                         5,
                         1.0,
                         1,
                         true,
                         true,
                         0,
                         true);
  int numIterations = 2;
  Path output = getTestTempDirPath("output");
  CDbwDriver.runJob(new Path(output, "clusters-5"),
                    new Path(output, "clusteredPoints"),
                    output,
                    new EuclideanDistanceMeasure(),
                    numIterations,
                    1);
  checkRefPoints(numIterations);
}
 

@Test
public void testDirichlet() throws Exception {
  ModelDistribution<VectorWritable> modelDistribution =
      new GaussianClusterDistribution(new VectorWritable(new DenseVector(2)));
  DirichletDriver.runJob(getTestTempDirPath("testdata"),
                         getTestTempDirPath("output"),
                         modelDistribution,
                         15,
                         5,
                         1.0,
                         1,
                         true,
                         true,
                         0,
                         true);
  int numIterations = 2;
  Path output = getTestTempDirPath("output");
  CDbwDriver.runJob(new Path(output, "clusters-5"),
                    new Path(output, "clusteredPoints"),
                    output,
                    new EuclideanDistanceMeasure(),
                    numIterations,
                    1);
  checkRefPoints(numIterations);
}
 

@Override
public Vector extractFeatures(PredictionCandidate predictionCandidate) {
    MultiLabel prediction = predictionCandidate.multiLabel;
    double[] calibratedLabelProbs = predictionCandidate.labelProbs;
    double prod = 1;
    for (int l=0;l<calibratedLabelProbs.length;l++){
        if (prediction.matchClass(l)){
            prod *= calibratedLabelProbs[l];
        } else {
            prod *= 1-calibratedLabelProbs[l];
        }
    }
    Vector vector = new DenseVector(1);
    vector.set(0,prod);
    return vector;
}
 

public static void write(File inputFile, Path outputPath)
    throws IOException {
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.get(conf);

  SequenceFile.Writer writer =
      SequenceFile.createWriter(fs, conf, outputPath, NullWritable.class,
          VectorWritable.class,
          SequenceFile.CompressionType.BLOCK,
          new DefaultCodec());
  try {
    for (String line : FileUtils.readLines(inputFile)) {
      String parts[] = StringUtils.split(line);

      writer.append(NullWritable.get(),
          new VectorWritable(new DenseVector(
              new double[]{
                  Double.valueOf(parts[0]),
                  Double.valueOf(parts[1])
              }
          )));
    }
  } finally {
    writer.close();
  }
}
 

private void updateCenters(int k){

        Vector center = new DenseVector(dataSet.getNumFeatures());
        double count = 0;
        for (int i=0;i<dataSet.getNumDataPoints();i++){
            if (assignments[i]==k){
                Vector instance = dataSet.getRow(i);
                for (int j=0;j<instance.size();j++){
                    center.set(j, center.get(j)+instance.get(j));
                }
                count += 1;
            }
        }
        center = center.divide(count);
        centers[k] = center;
        System.out.println("update the centroid of cluster "+(k+1)+" based on "+(int)count+" instances in the cluster");
    }
 

private void updateTransformProb(int dataPoint, int comIndex){
    MultiLabel labels = dataSet.getMultiLabels()[dataPoint];
    MultiLabel candidate = combinations.get(comIndex);
    Vector toMinus = new DenseVector(dataSet.getNumClasses());
    for (int i=0;i<dataSet.getNumClasses();i++){
        toMinus.set(i,0.5);
    }
    double prod = 1;
    for (int l = 0; l < dataSet.getNumClasses(); l++) {
        if (labels.matchClass(l)) {
            prod *= this.lrTransforms.get(l).predictClassProb(candidate.toVector(dataSet.getNumClasses()).minus(toMinus), 1);
        } else {
            prod *= this.lrTransforms.get(l).predictClassProb(candidate.toVector(dataSet.getNumClasses()).minus(toMinus), 0);
        }
    }
    transformProbs[dataPoint][comIndex] = prod;
}
 

public MLLogisticLoss(MLLogisticRegression mlLogisticRegression,
                    MultiLabelClfDataSet dataSet, double gaussianPriorVariance) {
    int numDataPoints = dataSet.getNumDataPoints();
    int numAssignments = mlLogisticRegression.getAssignments().size();
    int numClasses = dataSet.getNumClasses();
    this.mlLogisticRegression = mlLogisticRegression;
    numParameters = mlLogisticRegression.getWeights().totalSize();
    this.dataSet = dataSet;
    this.gaussianPriorVariance = gaussianPriorVariance;
    this.empiricalCounts = new DenseVector(numParameters);
    this.predictedCounts = new DenseVector(numParameters);
    this.classScoreMatrix = new double[numDataPoints][numClasses];
    this.classProbMatrix = new double[dataSet.getNumDataPoints()][dataSet.getNumClasses()];

    this.assignmentProbMatrix = new double[numDataPoints][numAssignments];
    this.assignmentScoreMatrix = new double[numDataPoints][numAssignments];
    this.updateEmpricalCounts();
    this.isValueCacheValid=false;
    this.isGradientCacheValid=false;
}
 

public Weights(int numClasses, int numFeatures, boolean random) {
    if (random) {
        this.numClasses = numClasses;
        this.numFeatures = numFeatures;
        this.weightVector = new DenseVector((numFeatures + 1)*numClasses);
        Random randomGenerator = new Random(0L);
        for (int i=0; i<weightVector.size(); i++) {
            double p = randomGenerator.nextDouble()-0.5;
            weightVector.set(i,p);
        }
    } else {
        this.numClasses = numClasses;
        this.numFeatures = numFeatures;
        this.weightVector = new DenseVector((numFeatures + 1)*numClasses);
    }

}
 

public ElasticNetLogisticTrainer build(){
    ElasticNetLogisticTrainer trainer = new ElasticNetLogisticTrainer();
    trainer.logisticRegression = logisticRegression;
    trainer.dataSet = dataSet;
    trainer.targets = targets;
    trainer.weights = weights;
    trainer.sumWeights = sumWeights;
    trainer.numClasses = numClasses;
    trainer.regularization = this.regularization;
    trainer.l1Ratio = this.l1Ratio;
    trainer.epsilon = this.epsilon;
    trainer.lineSearch = this.lineSearch;
    trainer.numParameters = logisticRegression.getWeights().totalSize();
    trainer.empiricalCounts = new DenseVector(trainer.numParameters);
    trainer.predictedCounts = new DenseVector(trainer.numParameters);
    trainer.probabilityMatrix = new double[numClasses][dataSet.getNumDataPoints()];
    trainer.updateEmpricalCounts();
    trainer.updateClassProbMatrix();
    trainer.updatePredictedCounts();
    trainer.terminator = new Terminator();
    trainer.maxNumLinearRegUpdates = maxNumLinearRegUpdates;
    return trainer;
}
 

private static void test1(){
    Vector vector = new DenseVector(10);
    vector.set(0,0.1);
    vector.set(1,0.2);
    vector.set(2,0.15);
    vector.set(3,0.4);
    vector.set(4,0.7);
    vector.set(8,0.9);
    int[] labels = new int[10];
    labels[0] = 0 ;
    labels[1] = 1;
    labels[2] = 1;
    labels[3] = 1;
    labels[9] = 1;
    FusedKolmogorovFilter filter = new FusedKolmogorovFilter();
    filter.setNumBins(10);
    List<List<Double>> inputsEachClass = filter.generateInputsEachClass(vector, labels, 2);
    System.out.println(inputsEachClass);
    List<EmpiricalCDF> empiricalCDFs = filter.generateCDFs(vector,inputsEachClass);
    System.out.println(empiricalCDFs);
    System.out.println(filter.maxDistance(empiricalCDFs));
}
 

private static void test2(){
    Vector vector = new DenseVector(10);
    vector.set(0,0.1);
    vector.set(1,0.2);
    vector.set(2,0.15);
    vector.set(3,0.4);
    vector.set(4,0.7);
    vector.set(8,0.9);
    vector.set(9,0.8);
    int[] labels = new int[10];
    labels[0] = 0 ;
    labels[1] = 1;
    labels[2] = 2;
    labels[3] = 1;
    labels[9] = 2;
    FusedKolmogorovFilter filter = new FusedKolmogorovFilter();
    filter.setNumBins(10);
    List<List<Double>> inputsEachClass = filter.generateInputsEachClass(vector, labels, 3);
    System.out.println(inputsEachClass);
    List<EmpiricalCDF> empiricalCDFs = filter.generateCDFs(vector,inputsEachClass);
    System.out.println(empiricalCDFs);
    System.out.println(filter.maxDistance(empiricalCDFs));
    System.out.println(EmpiricalCDF.distance(empiricalCDFs.get(0),empiricalCDFs.get(1)));
    System.out.println(EmpiricalCDF.distance(empiricalCDFs.get(0),empiricalCDFs.get(2)));
    System.out.println(EmpiricalCDF.distance(empiricalCDFs.get(1),empiricalCDFs.get(2)));
}
 

static void test7(){
    RegTreeConfig regTreeConfig = new RegTreeConfig().setNumSplitIntervals(4);
    Vector vector = new DenseVector(4);
    vector.set(0,0);
    vector.set(1,1);
    vector.set(2,2);
    vector.set(3,3);
    double[] probs = {0,0.5,0.2,0.6};
    double[] labels = {1,2,3,4};
    Splitter.GlobalStats globalStats = new Splitter.GlobalStats(labels,probs);
    List<Interval> intervals = IntervalSplitter.generateIntervals(regTreeConfig, vector, probs, labels,globalStats);
    System.out.println(intervals);
    System.out.println(IntervalSplitter.compress(intervals));

}
 

/**
 * Encodes a feature
 * @param handle
 * @param row
 * @param classify
 * @return
 */
private Vector encodeFeatures(DataHandleInternal handle, int row, boolean classify) {

    // Prepare
    DenseVector vector = new DenseVector(config.getVectorLength());
    interceptEncoder.addToVector("1", vector);
    
    // Special case where there are no features
    if (specification.featureIndices.length == 0) {
        wordEncoder.addToVector("Feature:1", 1, vector);
        return vector;
    }
    
    // For each attribute
    int count = 0;
    for (int index : specification.featureIndices) {
        
        // Obtain data
        ClassificationFeatureMetadata metadata = specification.featureMetadata[count];
        String value = null;
        if (classify && metadata.isNumericMicroaggregation()) {
            value = inputHandle.getValue(row, index, true);
        } else {
            value = handle.getValue(row, index, true);
        }
        Double numeric = metadata.getNumericValue(value);
        if (Double.isNaN(numeric)) {    
            wordEncoder.addToVector("Attribute-" + index + ":" + value, 1, vector);
        } else {
            wordEncoder.addToVector("Attribute-" + index, numeric, vector);
        }
        count++;
    }
    
    // Return
    return vector;
}
 

@Test
public void testEmptyCluster() {
  DistanceMeasure measure = new EuclideanDistanceMeasure();
  initData(1, 0.25, measure);
  Canopy cluster = new Canopy(new DenseVector(new double[] { 10, 10 }), 19, measure);
  clusters.put(cluster.getId(), cluster);
  List<VectorWritable> points = new ArrayList<VectorWritable>();
  representativePoints.put(cluster.getId(), points);
  CDbwEvaluator evaluator = new CDbwEvaluator(representativePoints, clusters, measure);
  assertEquals("inter cluster density", 0.0, evaluator.interClusterDensity(), EPSILON);
  assertEquals("separation", 1.5, evaluator.separation(), EPSILON);
  assertEquals("intra cluster density", 0.7155417527999326, evaluator.intraClusterDensity(), EPSILON);
  assertEquals("CDbw", 1.073312629199899, evaluator.getCDbw(), EPSILON);
}
 

private void readObject(java.io.ObjectInputStream in)
        throws IOException, ClassNotFoundException{
    numClasses = in.readInt();
    numFeatures = in.readInt();
    int[] indices = (int[])in.readObject();
    double[] values = (double[])in.readObject();
    weightVector = new DenseVector((numFeatures + 1)*numClasses);
    for (int i=0;i<indices.length;i++){
        weightVector.set(indices[i],values[i]);
    }
}
 

private Vector penaltyGradient(){
    Vector weightsVector = this.logisticRegression.getWeights().getAllWeights();
    Vector penalty = new DenseVector(weightsVector.size());

    penalty = penalty.plus(weightsVector.divide(priorGaussianVariance));

    for (int j:logisticRegression.getWeights().getAllBiasPositions()){
        penalty.set(j,0);
    }
    return penalty;
}
 

static void test10(){
    RegTreeConfig regTreeConfig = new RegTreeConfig().setNumSplitIntervals(2);
    Vector vector = new DenseVector(4);
    vector.set(0,Double.NaN);
    vector.set(1,1);
    vector.set(2,2);
    vector.set(3,3);
    double[] probs = {1,0.5,1,0.6};
    double[] labels = {1,2,3,4};
    Splitter.GlobalStats globalStats = new Splitter.GlobalStats(labels,probs);
    List<Interval> intervals = IntervalSplitter.generateIntervals(regTreeConfig, vector, probs, labels,globalStats);
    System.out.println(intervals);
    System.out.println(IntervalSplitter.compress(intervals));

}
 

@Override
public double[] predictClassProbs(Vector vector) {
    double[] scores = scoreEstimator.predictClassScores(vector);
    double[] probs = new double[scores.length];
    for (int k=0;k<scores.length;k++){
        Vector scoreFeatureVector = new DenseVector(1);
        scoreFeatureVector.set(0,scores[k]);
        probs[k] = logisticRegression.predictClassProb(scoreFeatureVector,1);
    }
    return probs;
}
 

static void test6(){
    RegTreeConfig regTreeConfig = new RegTreeConfig().setNumSplitIntervals(4);
    Vector vector = new DenseVector(4);
    vector.set(0,0);
    vector.set(1,1);
    vector.set(2,2);
    vector.set(3,3);
    double[] probs = {1,0.5,0.2,0.6};
    double[] labels = {1,2,3,4};
    Splitter.GlobalStats globalStats = new Splitter.GlobalStats(labels,probs);
    System.out.println(IntervalSplitter.generateIntervals(regTreeConfig, vector, probs, labels,globalStats));

}
 

public IMLLogisticLoss(IMLLogisticRegression mlLogisticRegression,
                      MultiLabelClfDataSet dataSet, double gaussianPriorVariance) {
    this.logisticRegression = mlLogisticRegression;
    numParameters = mlLogisticRegression.getWeights().totalSize();
    this.dataSet = dataSet;
    this.gaussianPriorVariance = gaussianPriorVariance;
    this.empiricalCounts = new DenseVector(numParameters);
    this.predictedCounts = new DenseVector(numParameters);
    this.classProbMatrix = new double[dataSet.getNumDataPoints()][dataSet.getNumClasses()];
    this.updateEmpricalCounts();
    this.isValueCacheValid=false;
    this.isGradientCacheValid=false;
}
 

public KLLoss (CMLCRF cmlcrf, MultiLabelClfDataSet dataSet, double[][] targetDistribution, double gaussianPriorVariance) {
    this.cmlcrf = cmlcrf;
    this.supportedCombinations = cmlcrf.getSupportCombinations();
    this.numSupport = cmlcrf.getNumSupports();
    this.dataSet = dataSet;
    this.numData = dataSet.getNumDataPoints();
    this.numClasses = dataSet.getNumClasses();
    this.targetDistribution = targetDistribution;
    this.gaussianPriorVariance = gaussianPriorVariance;
    this.numParameters = cmlcrf.getWeights().totalSize();
    this.numWeightsForFeatures = cmlcrf.getWeights().getNumWeightsForFeatures();
    this.numWeightsForLabelPairs = cmlcrf.getWeights().getNumWeightsForLabels();
    this.classScoreMatrix = new double[numData][numClasses];
    this.classProbMatrix = new double[numData][numClasses];
    this.combScoreMatrix = new double[numData][numSupport];
    this.combProbMatrix = new double[numData][numSupport];
    this.isGradientCacheValid = false;
    this.isValueCacheValid = false;
    this.empiricalCounts = new double[numParameters];
    this.gradient = new DenseVector(numParameters);
    this.combProbSums = new double[numSupport];
    this.initTargetMarginals();
    this.mapParameters();
    this.initComContainsLabel();
    this.mapPairToCombination();
    this.initEmpiricalCounts();


}
 

private void readObject(java.io.ObjectInputStream in)
        throws IOException, ClassNotFoundException{
    numClasses = in.readInt();
    numFeatures = in.readInt();
    numWeightsForFeatures = in.readInt();
    numWeightsForLabels = in.readInt();
    serializableWeights = (double[])in.readObject();
    weightVector = new DenseVector(numWeightsForFeatures + numWeightsForLabels);
    for (int i=0;i<serializableWeights.length;i++){
        weightVector.set(i,serializableWeights[i]);
    }
}
 

protected static Vector ratingVector(final PreferenceArray prefs) {
    final double[] ratings = new double[prefs.length()];
    for (int n = 0; n < prefs.length(); n++) {
        ratings[n] = prefs.get(n).getValue();
    }
    return new DenseVector(ratings, true);
}
 

public CRFLoss (CMLCRF cmlcrf, MultiLabelClfDataSet dataSet, double gaussianPriorVariance) {
    this.cmlcrf = cmlcrf;
    this.supportedCombinations = cmlcrf.getSupportCombinations();
    this.numSupport = cmlcrf.getNumSupports();
    this.dataSet = dataSet;
    this.numData = dataSet.getNumDataPoints();
    this.numClasses = dataSet.getNumClasses();
    this.gaussianPriorVariance = gaussianPriorVariance;
    this.numParameters = cmlcrf.getWeights().totalSize();
    this.numWeightsForFeatures = cmlcrf.getWeights().getNumWeightsForFeatures();
    this.numWeightsForLabelPairs = cmlcrf.getWeights().getNumWeightsForLabels();
    this.classScoreMatrix = new double[numData][numClasses];
    this.classProbMatrix = new double[numData][numClasses];
    this.combScoreMatrix = new double[numData][numSupport];
    this.combProbMatrix = new double[numData][numSupport];
    this.isGradientCacheValid = false;
    this.isValueCacheValid = false;
    this.empiricalCounts = new double[numParameters];
    this.initCache();
    this.updateEmpiricalCounts();
    this.gradient = new DenseVector(numParameters);
    this.labelPairToCombination = new ArrayList<>();
    for (int i=0;i< numWeightsForLabelPairs;i++){
        labelPairToCombination.add(new ArrayList<>());
    }
    this.mapPairToCombination();
    this.combProbSums = new double[numSupport];

    Map<MultiLabel,Integer> map = new HashMap<>();
    for (int s=0;s< numSupport;s++){
        map.put(supportedCombinations.get(s),s);
    }
    this.labelComIndices = new int[dataSet.getNumDataPoints()];
    for (int i=0;i<dataSet.getNumDataPoints();i++){
        labelComIndices[i] = map.get(dataSet.getMultiLabels()[i]);
    }
}
 

public static Weights getMean(CBM bmm, int label){
    int numClusters = bmm.getNumComponents();
    int length = ((LogisticRegression)bmm.getBinaryClassifiers()[0][0]).getWeights().getAllWeights().size();
    int numFeatures = ((LogisticRegression)bmm.getBinaryClassifiers()[0][0]).getNumFeatures();
    Vector mean = new DenseVector(length);
    for (int k=0;k<numClusters;k++){
        mean = mean.plus(((LogisticRegression)bmm.getBinaryClassifiers()[k][label]).getWeights().getAllWeights());
    }

    mean = mean.divide(numClusters);
    return new Weights(2,numFeatures,mean);
}
 

private static void test4() {
    double[] prior = {0.3, 0.7};
    LogisticRegression logisticRegression = new LogisticRegression(2, 10, prior);
    Vector vector = new DenseVector(10);
    for (int d=0;d<10;d++){
        vector.set(d, Math.random());
    }
    System.out.println(Arrays.toString(logisticRegression.predictClassProbs(vector)));
}
 

public static void main(String[] args) {
    double[] d = {1,2,5};
    Vector v = new DenseVector(d);
    System.out.println(Vectors.concatenate(v,4.5));

    test2();
    test3();
}
 

public Vector getParameters() {
    int numData = this.updatedEmbeddingMatrix.getNumDataPoints();
    int numFeatures = this.updatedEmbeddingMatrix.getNumFeatures();
    int vecSize = numData * numFeatures;
    Vector pVec = new DenseVector(vecSize);
    for (int i = 0; i < numData; i++) {
        for (int j = 0; j < numFeatures; j++) {
            pVec.set(i * numFeatures + j, this.updatedEmbeddingMatrix.getRow(i).get(j));
        }
    }
    return pVec;
}