Java Code Examples for weka.filters.unsupervised.attribute.Remove#setAttributeIndicesArray()

private Instances applyFiltersToDataset(final Instances dataset) throws Exception {
	Remove removeFilter = new Remove();
	removeFilter.setAttributeIndicesArray(this.labelIndices.stream().mapToInt(x -> x).toArray());
	removeFilter.setInvertSelection(false);
	removeFilter.setInputFormat(dataset);
	Instances filteredDataset = Filter.useFilter(dataset, removeFilter);

	Add addTarget = new Add();
	addTarget.setAttributeIndex("last");
	addTarget.setNominalLabels("true,false");
	addTarget.setAttributeName("a>b");
	addTarget.setInputFormat(filteredDataset);
	filteredDataset = Filter.useFilter(filteredDataset, addTarget);
	filteredDataset.setClassIndex(filteredDataset.numAttributes() - 1);
	return filteredDataset;
}
 

/**
  * Build a linear model for this node using those attributes
  * specified in indices.
  *
  * @param indices an array of attribute indices to include in the linear
  * model
  * @throws Exception if something goes wrong
  */
 private void buildLinearModel(int [] indices) throws Exception {
   // copy the training instances and remove all but the tested
   // attributes
   Instances reducedInst = new Instances(m_instances);
   Remove attributeFilter = new Remove();
   
   attributeFilter.setInvertSelection(true);
   attributeFilter.setAttributeIndicesArray(indices);
   attributeFilter.setInputFormat(reducedInst);

   reducedInst = Filter.useFilter(reducedInst, attributeFilter);
   
   // build a linear regression for the training data using the
   // tested attributes
   LinearRegression temp = new LinearRegression();
   temp.buildClassifier(reducedInst);

   double [] lmCoeffs = temp.coefficients();
   double [] coeffs = new double [m_instances.numAttributes()];

   for (int i = 0; i < lmCoeffs.length - 1; i++) {
     if (indices[i] != m_classIndex) {
coeffs[indices[i]] = lmCoeffs[i];
     }
   }
   m_nodeModel = new PreConstructedLinearModel(coeffs, lmCoeffs[lmCoeffs.length - 1]);
   m_nodeModel.buildClassifier(m_instances);
 }
 

/**
 * Apply trained model to a candidate record-pair. Therefore a new
 * FeatureDataSet is created, which is afterwards classified as match or
 * non-match
 * 
 * @param record1
 *            the first record (must not be null)
 * @param record2
 *            the second record (must not be null)
 * @param schemaCorrespondences
 *            the schema correspondences between the first and the second
 *            records
 * @return A correspondence holding the input parameters plus the
 *         classification´s result, which is either match (1.0) or
 *         non-match(0.0).
 */

@Override
public Correspondence<RecordType, SchemaElementType> apply(RecordType record1, RecordType record2,
		Processable<Correspondence<SchemaElementType, Matchable>> schemaCorrespondences) {

	if (this.classifier == null) {
		logger.error("Please initialise a classifier!");
		return null;
	} else {
		FeatureVectorDataSet matchSet = this.initialiseFeatures(record1, record2, schemaCorrespondences);
		Record matchRecord = generateFeatures(record1, record2, schemaCorrespondences, matchSet);

		// transform entry for classification.
		matchSet.add(matchRecord);
		Instances matchInstances = this.transformToWeka(matchSet, this.matchSet);

		// reduce dimensions if feature subset selection was applied before.
		if ((this.backwardSelection || this.forwardSelection) && this.fs != null)
			try {
				Remove removeFilter = new Remove();
				removeFilter.setAttributeIndicesArray(this.fs.selectedAttributes());
				removeFilter.setInvertSelection(true);
				removeFilter.setInputFormat(matchInstances);
				matchInstances = Filter.useFilter(matchInstances, removeFilter);
			} catch (Exception e1) {
				e1.printStackTrace();
			}
		// Apply matching rule
		try {
			double[] distribution = this.classifier.distributionForInstance(matchInstances.firstInstance());
			int positiveClassIndex = matchInstances.attribute(matchInstances.classIndex()).indexOfValue("1");
			double matchConfidence = distribution[positiveClassIndex];
			if (this.isDebugReportActive()) {
				fillSimilarity(record1, record2, matchConfidence);
			}
			return new Correspondence<RecordType, SchemaElementType>(record1, record2, matchConfidence,
					schemaCorrespondences);

			
		} catch (Exception e) {
			e.printStackTrace();
			logger.error(String.format("Classifier Exception for Record '%s': %s",
					matchRecord == null ? "null" : matchRecord.toString(), e.getMessage()));
		}
		return null;
	}
}
 

@Override
public void buildClassifier(Instances data) throws Exception {
	testCapabilities(data);

	int c = data.classIndex();

	if(getDebug()) System.out.print("-: Creating "+c+" models ("+m_Classifier.getClass().getName()+"): ");
	m_MultiClassifiers = AbstractClassifier.makeCopies(m_Classifier,c);

	Instances sub_data = null;

	for(int i = 0; i < c; i++) {

		int indices[][] = new int[c][c - 1];
		for(int j = 0, k = 0; j < c; j++) {
			if(j != i) {
				indices[i][k++] = j;
			}
		}

		//Select only class attribute 'i'
		Remove FilterRemove = new Remove();
		FilterRemove.setAttributeIndicesArray(indices[i]);
		FilterRemove.setInputFormat(data);
		FilterRemove.setInvertSelection(true);
		sub_data = Filter.useFilter(data, FilterRemove);
		sub_data.setClassIndex(0);
		/* BEGIN downsample for this link */
		sub_data.randomize(m_Random);
		int numToRemove = sub_data.numInstances() - (int)Math.round(sub_data.numInstances() * m_DownSampleRatio);
		for(int m = 0, removed = 0; m < sub_data.numInstances(); m++) {
			if (sub_data.instance(m).classValue() <= 0.0) {
				sub_data.instance(m).setClassMissing();
				if (++removed >= numToRemove)
					break;
			}
		}
		sub_data.deleteWithMissingClass();
		/* END downsample for this link */


		//Build the classifier for that class
		m_MultiClassifiers[i].buildClassifier(sub_data);
		if(getDebug()) System.out.print(" " + (i+1));

	}

	if(getDebug()) System.out.println(" :-");

	m_InstancesTemplate = new Instances(sub_data, 0);

}