Java Code Examples for weka.core.Instances#insertAttributeAt()

/**
 * AddZtoD - Add attribute space Z[N][H] (N rows of H columns) to Instances D, which should have N rows also.
 * @param	D 	dataset (of N instances)
 * @param	Z	attribute space (of N rows, H columns)
 * @param	L	column to add Z from in D
 */
private static Instances addZtoD(Instances D, double Z[][], int L) {

	int H = Z[0].length;
	int N = D.numInstances();

	// add attributes
	for(int a = 0; a < H; a++) {
		D.insertAttributeAt(new Attribute("A"+a),L+a);
	}

	// add values Z[0]...Z[N] to D
	for(int a = 0; a < H; a++) {
		for(int i = 0; i < N; i++) {
			D.instance(i).setValue(L+a,Z[i][a]);
		}
	}

	D.setClassIndex(L);
	return D;
}
 

/**
 * InsertZintoD - Insert data Z[][] to Instances D (e.g., as labels).
 * NOTE: Assumes binary labels!
 * @see #addZtoD(Instances, double[][], int)
 */
private static Instances insertZintoD(Instances D, double Z[][]) {

	int L = Z[0].length;

	// add attributes
	for(int j = 0; j < L; j++) {
		D.insertAttributeAt(new Attribute("c"+j,Arrays.asList(new String[]{"0","1"})),j);
	}

	// add values Z[0]...Z[N] to D
	// (note that if D.numInstances() < Z.length, only some are added)
	for(int j = 0; j < L; j++) {
		for(int i = 0; i < D.numInstances(); i++) {
			D.instance(i).setValue(j,Z[i][j] > 0.5 ? 1.0 : 0.0);
		}
	}

	D.setClassIndex(L);
	return D;
}
 

@Override
public void buildClassifier(Instances data) throws Exception {
    //creating the 2class version of the insts
    numericClassInsts = new Instances(data);
    numericClassInsts.setClassIndex(0); //temporary
    numericClassInsts.deleteAttributeAt(numericClassInsts.numAttributes()-1);
    Attribute newClassAtt = new Attribute("newClassVal"); //numeric class
    numericClassInsts.insertAttributeAt(newClassAtt, numericClassInsts.numAttributes());
    numericClassInsts.setClassIndex(numericClassInsts.numAttributes()-1); //temporary

    //and building the regressors
    regressors = new LinearRegression[data.numClasses()];
    double[] trueClassVals = data.attributeToDoubleArray(data.classIndex());
    for (int c = 0; c < data.numClasses(); c++) {

        for (int i = 0; i < numericClassInsts.numInstances(); i++) {
            //if this inst is of the class we're currently handling (c), set new class val to 1 else 0
            double cval = trueClassVals[i] == c ? 1 : 0; 
            numericClassInsts.instance(i).setClassValue(cval);
        }    

        regressors[c] = new LinearRegression();
        regressors[c].buildClassifier(numericClassInsts);
    }
}
 

@Override
public void buildClassifier(Instances data) throws Exception {
    //creating the 2class version of the insts
    numericClassInsts = new Instances(data);
    numericClassInsts.setClassIndex(0); //temporary
    numericClassInsts.deleteAttributeAt(numericClassInsts.numAttributes()-1);
    Attribute newClassAtt = new Attribute("newClassVal"); //numeric class
    numericClassInsts.insertAttributeAt(newClassAtt, numericClassInsts.numAttributes());
    numericClassInsts.setClassIndex(numericClassInsts.numAttributes()-1); //temporary

    //and building the regressors
    regressors = new M5P[data.numClasses()];
    double[] trueClassVals = data.attributeToDoubleArray(data.classIndex());
    for (int c = 0; c < data.numClasses(); c++) {

        for (int i = 0; i < numericClassInsts.numInstances(); i++) {
            //if this inst is of the class we're currently handling (c), set new class val to 1 else 0
            double cval = trueClassVals[i] == c ? 1 : 0; 
            numericClassInsts.instance(i).setClassValue(cval);
        }    

        regressors[c] = new M5P();
        regressors[c].buildClassifier(numericClassInsts);
    }
}
 

public static Instances concatenate(Instances[] train) {
    // make a super arff for finding params that need stdev etc
    Instances temp = new Instances(train[0], 0);
    for (int i = 1; i < train.length; i++) {
        for (int j = 0; j < train[i].numAttributes() - 1; j++) {
            temp.insertAttributeAt(train[i].attribute(j), temp.numAttributes() - 1);
        }
    }

    int dataset, attFromData;

    for (int insId = 0; insId < train[0].numInstances(); insId++) {
        DenseInstance dense = new DenseInstance(temp.numAttributes());
        for (int attId = 0; attId < temp.numAttributes() - 1; attId++) {

            dataset = attId / (train[0].numAttributes() - 1);
            attFromData = attId % (train[0].numAttributes() - 1);
            dense.setValue(attId, train[dataset].instance(insId).value(attFromData));

        }
        dense.setValue(temp.numAttributes() - 1, train[0].instance(insId).classValue());
        temp.add(dense);
    }
    return temp;
}
 

public Instances generateMore(int number, int existedNum,
		Instances header) {
	ArrayList<Attribute> localAtts = new ArrayList<Attribute>();
	Enumeration<Attribute> enu = header.enumerateAttributes();
	while (enu.hasMoreElements()) {
		localAtts.add(enu.nextElement());
	}
	Instances samplePoints = LHSInitializer.getMultiDimContinuous(
			localAtts, number + existedNum, false);
	samplePoints.insertAttributeAt(header.classAttribute(),
			samplePoints.numAttributes());
	samplePoints.setClassIndex(samplePoints.numAttributes() - 1);
	return samplePoints;
}
 

/**
 * mulan2meka - Move label attributes from the End to the Beginning of attribute space (MULAN format to MEKA format). 
 * Note: can use e.g.: java weka.filters.unsupervised.attribute.Reorder -i thyroid.arff -R 30-last,1-29"
 * See also: F.reorderLabels(D,s)
 */
public static final Instances mulan2meka(Instances D, int L) {
	int d = D.numAttributes();
	for(int j = 0; j < L; j++) {
		D.insertAttributeAt(D.attribute(d-1).copy(D.attribute(d-1).name()+"-"),0);
		for(int i = 0; i < D.numInstances(); i++) {
			D.instance(i).setValue(0,D.instance(i).value(d));
		}
		D.deleteAttributeAt(d);
	}
	return D;
}
 

/**
 * meka2mulan - Move L label attributes from the beginning to end of attribute space of an Instances. 
 * Necessary because MULAN assumes label attributes are at the end, not the beginning.
 * (the extra time for this process is not counted in the running-time analysis of published work).
 */
public static final Instances meka2mulan(Instances D, int L) {
	for(int j = 0; j < L; j++) {
		//D.insertAttributeAt(new Attribute(D.attribute(0).name()+"-"),D.numAttributes());
		D.insertAttributeAt(D.attribute(0).copy(D.attribute(0).name()+"-"),D.numAttributes());
		for(int i = 0; i < D.numInstances(); i++) {
			D.instance(i).setValue(D.numAttributes()-1,D.instance(i).value(0));
		}
		D.deleteAttributeAt(0);
	}
	return D;
}
 

/**
 * add a new feature in the dataset containing the predicted values by the classifier
 * @param c the classifier
 * @param i the dataset
 * @throws Exception
 */
public static void add_predictions(Classifier c, Instances i) throws Exception{
	
	double[] computed = new double[i.numInstances()];
	for(int m = 0;m<computed.length;m++){
		computed[m] = c.classifyInstance(i.instance(m)); 
	}
	Attribute a = new Attribute("interpolate");
	int num = i.numAttributes();
	i.insertAttributeAt(a, num);
	for(int m = 0;m<computed.length;m++){
		i.instance(m).setValue(num, computed[m]);
	}
}
 

@Override
public Instances collectPerfs(Instances samplePoints, String perfAttName) {
	Instances retVal = null;
	
	if(samplePoints.attribute(perfAttName) == null){
		Attribute performance = new Attribute(perfAttName);
		samplePoints.insertAttributeAt(performance, samplePoints.numAttributes());
	}
	
	File perfFolder = new File(perfsfilepath);
	int tot=0;
	if(perfFolder.exists()){
		//let's get all the name set for the sample points
		Iterator<Instance> itr = samplePoints.iterator();
		TreeSet<String> insNameSet = new TreeSet<String>();
		HashMap<String, Integer> mapping = new HashMap<String, Integer>();
		int pos=0;
		while(itr.hasNext()){
			String mdstr = getMD5(itr.next());
			insNameSet.add(mdstr);
			mapping.put(mdstr, new Integer(pos++));
		}
		
		//now we collect
		File[] perfFiles = perfFolder.listFiles(new PerfsFileFilter(insNameSet));
		tot = perfFiles.length;
		if(tot > 0) isInterrupt = true;
		for(int i=0;i<tot;i++){
			Instance ins = samplePoints.get(mapping.get(perfFiles[i].getName()));
			double[] results = getPerf(perfFiles[i].getAbsolutePath());
			if(results!=null){
				ins.setValue(samplePoints.numAttributes()-1, results[0]);
			}
		}
	}
	retVal = samplePoints;
	retVal.setClassIndex(retVal.numAttributes()-1);
	System.out.println("Total number of collected performances is : "+tot);
	return retVal;
}
 

public static void testCOMT2() throws Exception{
	BestConf bestconf = new BestConf();
	Instances trainingSet = DataIOFile.loadDataFromArffFile("data/trainingBestConf0.arff");
	trainingSet.setClassIndex(trainingSet.numAttributes()-1);
	
	Instances samplePoints = LHSInitializer.getMultiDimContinuous(bestconf.getAttributes(), InitialSampleSetSize, false);
	samplePoints.insertAttributeAt(trainingSet.classAttribute(), samplePoints.numAttributes());
	samplePoints.setClassIndex(samplePoints.numAttributes()-1);
	
	COMT2 comt = new COMT2(samplePoints, COMT2Iteration);
	
	comt.buildClassifier(trainingSet);
	
	Evaluation eval = new Evaluation(trainingSet);
	eval.evaluateModel(comt, trainingSet);
	System.err.println(eval.toSummaryString());
	
	Instance best = comt.getInstanceWithPossibleMaxY(samplePoints.firstInstance());
	Instances bestInstances = new Instances(trainingSet,2);
	bestInstances.add(best);
	DataIOFile.saveDataToXrffFile("data/trainingBestConf_COMT2.arff", bestInstances);
	
	//now we output the training set with the class value updated as the predicted value
	Instances output = new Instances(trainingSet, trainingSet.numInstances());
	Enumeration<Instance> enu = trainingSet.enumerateInstances();
	while(enu.hasMoreElements()){
		Instance ins = enu.nextElement();
		double[] values = ins.toDoubleArray();
		values[values.length-1] = comt.classifyInstance(ins);
		output.add(ins.copy(values));
	}
	DataIOFile.saveDataToXrffFile("data/trainingBestConf0_predict.xrff", output);
}
 

/**
 * Sets the format of the input instances.
 *
 * @param instanceInfo an Instances object containing the input instance
 * structure (any instances contained in the object are ignored - only the
 * structure is required).
 * @return true if the outputFormat may be collected immediately
 * @throws Exception if a problem occurs setting the input format
 */
public boolean setInputFormat(Instances instanceInfo) throws Exception {

  super.setInputFormat(instanceInfo);
  
  m_CopyCols.setUpper(instanceInfo.numAttributes() - 1);

  // Create the output buffer
  Instances outputFormat = new Instances(instanceInfo, 0); 
  m_SelectedAttributes = m_CopyCols.getSelection();
  for (int i = 0; i < m_SelectedAttributes.length; i++) {
    int current = m_SelectedAttributes[i];
    // Create a copy of the attribute with a different name
    Attribute origAttribute = instanceInfo.attribute(current);
    outputFormat.insertAttributeAt((Attribute)origAttribute.copy("Copy of " + origAttribute.name()),
		     outputFormat.numAttributes());

  }

  // adapt locators
  int[] newIndices = new int[instanceInfo.numAttributes() + m_SelectedAttributes.length];
  for (int i = 0; i < instanceInfo.numAttributes(); i++)
    newIndices[i] = i;
  for (int i = 0; i < m_SelectedAttributes.length; i++)
    newIndices[instanceInfo.numAttributes() + i] = m_SelectedAttributes[i];
  initInputLocators(instanceInfo, newIndices);

  setOutputFormat(outputFormat);
  
  return true;
}
 

/**
 * Sets the format of the input instances.
 *
 * @param instanceInfo an Instances object containing the input 
 * instance structure (any instances contained in the object are 
 * ignored - only the structure is required).
 * @return true if the outputFormat may be collected immediately
 * @throws Exception if the input format can't be set 
 * successfully
 */
public boolean setInputFormat(Instances instanceInfo) 
  throws Exception {

  if (instanceInfo.attribute(1).type()!=Attribute.RELATIONAL) {
    throw new Exception("Can only handle relational-valued attribute!");
  }  
  super.setInputFormat(instanceInfo);   

  m_NumBags = instanceInfo.numInstances();
  m_NumInstances = 0;
  for (int i=0; i<m_NumBags; i++)
    m_NumInstances += instanceInfo.instance(i).relationalValue(1).numInstances();

  Attribute classAttribute = (Attribute) instanceInfo.classAttribute().copy();
  Attribute bagIndex = (Attribute) instanceInfo.attribute(0).copy();

  /* create a new output format (propositional instance format) */
  Instances newData = instanceInfo.attribute(1).relation().stringFreeStructure();
  newData.insertAttributeAt(bagIndex, 0);
  newData.insertAttributeAt(classAttribute, newData.numAttributes());
  newData.setClassIndex(newData.numAttributes() - 1);

  super.setOutputFormat(newData.stringFreeStructure());

  m_BagStringAtts = new StringLocator(instanceInfo.attribute(1).relation().stringFreeStructure());
  m_BagRelAtts    = new RelationalLocator(instanceInfo.attribute(1).relation().stringFreeStructure());

  return true;
}
 

/**
    * Converts training data to numeric version. The class variable is replaced by a pseudo-class 
    * used by LogitBoost.
    * 
    * @param data the data to convert
    * @return the converted data
    * @throws Exception if something goes wrong
    */
   protected Instances getNumericData(Instances data) throws Exception{
Instances numericData = new Instances(data);

int classIndex = numericData.classIndex();
numericData.setClassIndex(-1);
numericData.deleteAttributeAt(classIndex);
numericData.insertAttributeAt(new Attribute("'pseudo class'"), classIndex);
numericData.setClassIndex(classIndex);
return numericData;
   }
 

/** Inserts new attributes in current dataset or instance 
 *
 * @exception Exception if something goes wrong
 */
protected Instances insertNewAttr(Instances data) throws Exception{
  
  int i;
  for (i=0; i<data.classAttribute().numValues(); i++)
    {
      data.insertAttributeAt( new Attribute("N"+ i), i); 
    }
  return data;
}
 

@Override
protected Instances determineOutputFormat(Instances inputFormat)
        throws Exception {
    
    //Check all attributes are real valued, otherwise throw exception
    for (int i = 0; i < inputFormat.numAttributes(); i++) {
        if (inputFormat.classIndex() != i) {
            if (!inputFormat.attribute(i).isNumeric()) {
                throw new Exception("Non numeric attribute not allowed for BoP conversion");
            }
        }
    }

    ArrayList<Attribute> attributes = new ArrayList<>();
    for (String word : dictionary) 
        attributes.add(new Attribute(word));
    
    Instances result = new Instances("BagOfPatterns_" + inputFormat.relationName(), attributes, inputFormat.numInstances());
    
    if (inputFormat.classIndex() >= 0) {	//Classification set, set class 
        //Get the class values as a fast vector			
        Attribute target = inputFormat.attribute(inputFormat.classIndex());

        ArrayList<String> vals = new ArrayList<>(target.numValues());
        for (int i = 0; i < target.numValues(); i++) {
            vals.add(target.value(i));
        }
        
        result.insertAttributeAt(new Attribute(inputFormat.attribute(inputFormat.classIndex()).name(), vals), result.numAttributes());
        result.setClassIndex(result.numAttributes() - 1);
    }
 
    return result;
}
 

public Instances runExp(Instances samplePoints, String perfAttName){
	Instances retVal = null;
	if(samplePoints.attribute(perfAttName) == null){
		Attribute performance = new Attribute(perfAttName);
		samplePoints.insertAttributeAt(performance, samplePoints.numAttributes());
	}
	int pos = samplePoints.numInstances();
	int count = 0;
	for (int i = 0; i < pos; i++) {
		Instance ins = samplePoints.get(i);
		HashMap hm = new HashMap();
		int tot = 0;
		for (int j = 0; j < ins.numAttributes(); j++) {
			hm.put(ins.attribute(j).name(), ins.value(ins.attribute(j)));
		}

		boolean testRet;
		if (Double.isNaN(ins.value(ins.attribute(ins.numAttributes() - 1)))) {
			testRet = this.startTest(hm, i, isInterrupt);
			double y = 0;
			if (!testRet) {// the setting does not work, we skip it
				y = -1;
				count++;
				if (count >= targetTestErrorNum) {
					System.out.println("There must be somthing wrong with the system. Please check and restart.....");
					System.exit(1);
				}
			} else {
				y = getPerformanceByType(performanceType);
				count = 0;
			}

			ins.setValue(samplePoints.numAttributes() - 1, y);
			writePerfstoFile(ins);
		} else {
			continue;
		}
	}
	retVal = samplePoints;
	retVal.setClassIndex(retVal.numAttributes()-1);
	
	return retVal;
}
 

public Instances convertInstances(Instances D, int L) throws Exception {

		//Gather combinations
		HashMap<String,Integer> distinctCombinations = MLUtils.classCombinationCounts(D);
		if(getDebug())
			System.out.println("Found "+distinctCombinations.size()+" unique combinations");

		//Prune combinations
		MLUtils.pruneCountHashMap(distinctCombinations,m_P);
		if(getDebug())
			System.out.println("Pruned to "+distinctCombinations.size()+" with P="+m_P);

		// Remove all class attributes
		Instances D_ = MLUtils.deleteAttributesAt(new Instances(D),MLUtils.gen_indices(L));
		// Add a new class attribute
		D_.insertAttributeAt(new Attribute("CLASS", new ArrayList(distinctCombinations.keySet())),0); // create the class attribute
		D_.setClassIndex(0);

		//Add class values
		for (int i = 0; i < D.numInstances(); i++) {
			String y = MLUtils.encodeValue(MLUtils.toIntArray(D.instance(i),L));
			// add it
			if(distinctCombinations.containsKey(y)) 	//if its class value exists
				D_.instance(i).setClassValue(y);
			// decomp
			else if(m_N > 0) { 
				String d_subsets[] = SuperLabelUtils.getTopNSubsets(y, distinctCombinations, m_N);
				for (String s : d_subsets) {
					int w = distinctCombinations.get(s);
					Instance copy = (Instance)(D_.instance(i)).copy();
					copy.setClassValue(s);
					copy.setWeight(1.0 / d_subsets.length);
					D_.add(copy);
				}
			}
		}

		// remove with missing class
		D_.deleteWithMissingClass();

		// keep the header of new dataset for classification
		m_InstancesTemplate = new Instances(D_, 0);

		if (getDebug())
			System.out.println(""+D_);

		return D_;
	}
 

/**
 * Builds a ripple-down manner rule learner.
 *
 * @param instances the training data
 * @throws Exception if classifier can't be built successfully
 */
public void buildClassifier(Instances instances) throws Exception {

  // can classifier handle the data?
  getCapabilities().testWithFail(instances);

  // remove instances with missing class
  Instances data = new Instances(instances);
  data.deleteWithMissingClass();
  
  int numCl = data.numClasses();
  m_Root = new Ridor_node();
  m_Class = instances.classAttribute();     // The original class label
	
  int index = data.classIndex();
  m_Cover = data.sumOfWeights();

  m_Random = new Random(m_Seed);
	
  /* Create a binary attribute */
  FastVector binary_values = new FastVector(2);
  binary_values.addElement("otherClasses");
  binary_values.addElement("defClass");
  Attribute attr = new Attribute ("newClass", binary_values);
  data.insertAttributeAt(attr, index);	
  data.setClassIndex(index);                 // The new class label

  /* Partition the data into bags according to their original class values */
  Instances[] dataByClass = new Instances[numCl];
  for(int i=0; i < numCl; i++)
    dataByClass[i] = new Instances(data, data.numInstances()); // Empty bags
  for(int i=0; i < data.numInstances(); i++){ // Partitioning
    Instance inst = data.instance(i);
    inst.setClassValue(0);           // Set new class vaue to be 0
    dataByClass[(int)inst.value(index+1)].add(inst); 
  }	
	
  for(int i=0; i < numCl; i++)    
    dataByClass[i].deleteAttributeAt(index+1);   // Delete original class
	
  m_Root.findRules(dataByClass, 0);
  
}
 

/** 
    * performs a boosting iteration, returning a new model for the committee
    * 
    * @param data the data to boost on
    * @return the new model
    * @throws Exception if anything goes wrong
    */
   protected Classifier[] boost(Instances data) throws Exception {
     
     Classifier[] newModel = AbstractClassifier.makeCopies(m_Classifier, m_NumClasses);
     
     // Create a copy of the data with the class transformed into numeric
     Instances boostData = new Instances(data);
     boostData.deleteWithMissingClass();
     int numInstances = boostData.numInstances();
     
     // Temporarily unset the class index
     int classIndex = data.classIndex();
     boostData.setClassIndex(-1);
     boostData.deleteAttributeAt(classIndex);
     boostData.insertAttributeAt(new Attribute("'pseudo class'"), classIndex);
     boostData.setClassIndex(classIndex);
     double [][] trainFs = new double [numInstances][m_NumClasses];
     double [][] trainYs = new double [numInstances][m_NumClasses];
     for (int j = 0; j < m_NumClasses; j++) {
for (int i = 0, k = 0; i < numInstances; i++, k++) {
  while (data.instance(k).classIsMissing()) k++;
  trainYs[i][j] = (data.instance(k).classValue() == j) ? 1 : 0;
}
     }
     
     // Evaluate / increment trainFs from the classifiers
     for (int x = 0; x < m_models.size(); x++) {
for (int i = 0; i < numInstances; i++) {
  double [] pred = new double [m_NumClasses];
  double predSum = 0;
  Classifier[] model = (Classifier[]) m_models.elementAt(x);
  for (int j = 0; j < m_NumClasses; j++) {
    pred[j] = model[j].classifyInstance(boostData.instance(i));
    predSum += pred[j];
  }
  predSum /= m_NumClasses;
  for (int j = 0; j < m_NumClasses; j++) {
    trainFs[i][j] += (pred[j] - predSum) * (m_NumClasses-1) 
      / m_NumClasses;
  }
}
     }

     for (int j = 0; j < m_NumClasses; j++) {

// Set instance pseudoclass and weights
for (int i = 0; i < numInstances; i++) {
  double p = RtoP(trainFs[i], j);
  Instance current = boostData.instance(i);
  double z, actual = trainYs[i][j];
  if (actual == 1) {
    z = 1.0 / p;
    if (z > Z_MAX) { // threshold
      z = Z_MAX;
    }
  } else if (actual == 0) {
    z = -1.0 / (1.0 - p);
    if (z < -Z_MAX) { // threshold
      z = -Z_MAX;
    }
  } else {
    z = (actual - p) / (p * (1 - p));
  }

  double w = (actual - p) / z;
  current.setValue(classIndex, z);
  current.setWeight(numInstances * w);
}

Instances trainData = boostData;
if (m_UseResampling) {
  double[] weights = new double[boostData.numInstances()];
  for (int kk = 0; kk < weights.length; kk++) {
    weights[kk] = boostData.instance(kk).weight();
  }
  trainData = boostData.resampleWithWeights(m_RandomInstance, 
					    weights);
}

// Build the classifier
newModel[j].buildClassifier(trainData);
     }      
     
     return newModel;
   }