Java Code Examples for meka.core.Result#getStats()

/**
 * EvaluateModel - Build model 'h' on 'D_train', test it on 'D_test', threshold it according to 'top', verbosity 'vop'.
 * @param	h		a multi-dim. classifier
 * @param	D_train	training data
 * @param	D_test 	test data
 * @param	top    	Threshold OPtion (pertains to multi-label data only)
 * @param	vop    	Verbosity OPtion (which measures do we want to calculate/output)
 * @return	Result	raw prediction data with evaluation statistics included.
 */
public static Result evaluateModel(MultiXClassifier h, Instances D_train, Instances D_test, String top, String vop) throws Exception {
	Result r = evaluateModel(h,D_train,D_test);
	if (h instanceof MultiTargetClassifier || isMT(D_test)) {
		r.setInfo("Type","MT");
	}
	else if (h instanceof MultiLabelClassifier) {
		r.setInfo("Type","ML");
		r.setInfo("Threshold",MLEvalUtils.getThreshold(r.predictions,D_train,top)); // <-- only relevant to ML (for now), but we'll put it in here in any case
	}
	r.setInfo("Verbosity",vop);
	r.output = Result.getStats(r, vop);
	return r;
}
 

/**
 * EvaluateModel - Assume 'h' is already built, test it on 'D_test', threshold it according to 'top', verbosity 'vop'.
 * @param	h		a multi-dim. classifier
 * @param	D_test 	test data
 * @param	tal    	Threshold VALUES (not option)
 * @param	vop    	Verbosity OPtion (which measures do we want to calculate/output)
 * @return	Result	raw prediction data with evaluation statistics included.
 */
public static Result evaluateModel(MultiXClassifier h, Instances D_test, String tal, String vop) throws Exception {
	Result r = testClassifier(h,D_test);
	if (h instanceof MultiTargetClassifier || isMT(D_test)) {
		r.setInfo("Type","MT");
	}
	else if (h instanceof MultiLabelClassifier) {
		r.setInfo("Type","ML");
	}
	r.setInfo("Threshold",tal);
	r.setInfo("Verbosity",vop);
	r.output = Result.getStats(r, vop);
	return r;
}
 

/**
 * CVModel - Split D into train/test folds, and then train and evaluate on each one.
 * @param	h		 a multi-output classifier
 * @param	D      	 test data Instances
 * @param	numFolds number of folds of CV
 * @param	top    	 Threshold OPtion (pertains to multi-label data only)
 * @param	vop    	Verbosity OPtion (which measures do we want to calculate/output)
 * @return	Result	raw prediction data with evaluation statistics included.
 */
public static Result cvModel(MultiLabelClassifier h, Instances D, int numFolds, String top, String vop) throws Exception {
	Result r_[] = new Result[numFolds];
	for(int i = 0; i < numFolds; i++) {
		Instances D_train = D.trainCV(numFolds,i);
		Instances D_test = D.testCV(numFolds,i);
		if (h.getDebug()) System.out.println(":- Fold ["+i+"/"+numFolds+"] -: "+MLUtils.getDatasetName(D)+"\tL="+D.classIndex()+"\tD(t:T)=("+D_train.numInstances()+":"+D_test.numInstances()+")\tLC(t:T)="+Utils.roundDouble(MLUtils.labelCardinality(D_train,D.classIndex()),2)+":"+Utils.roundDouble(MLUtils.labelCardinality(D_test,D.classIndex()),2)+")");
		r_[i] = evaluateModel(h, D_train, D_test); // <-- should not run stats yet!
	}
	Result r = MLEvalUtils.combinePredictions(r_);
	if (h instanceof MultiTargetClassifier || isMT(D)) {
		r.setInfo("Type","MT-CV");
	}
	else if (h instanceof MultiLabelClassifier) {
		r.setInfo("Type","ML-CV");
		try {
			r.setInfo("Threshold",String.valueOf(Double.parseDouble(top)));
		} catch(Exception e) {
			System.err.println("[WARNING] Automatic threshold calibration not currently enabled for cross-fold validation, setting threshold = 0.5.\n");
			r.setInfo("Threshold",String.valueOf(0.5));
		}
	}
	r.setInfo("Verbosity",vop);
	r.output = Result.getStats(r, vop);
	// Need to reset this because of CV
	r.setValue("Number of training instances",D.numInstances());
	r.setValue("Number of test instances",D.numInstances());
	return r;
}
 

public static Result evaluateModelM(MultiXClassifier h, Instances D_train, Instances D_test, String top, String vop) throws Exception {
	// Train
	long before = System.currentTimeMillis();
			/*if (h instanceof SemisupervisedClassifier) { // *NEW* for semi-supervised 
				((SemisupervisedClassifier)h).setUnlabelledData(MLUtils.setLabelsMissing(new Instances(D_test)));
			}*/
	h.buildClassifier(D_train);
	long after = System.currentTimeMillis();

	//System.out.println(":- Classifier -: "+h.getClass().getName()+": "+Arrays.toString(h.getOptions()));

	// Test
	long before_test = System.currentTimeMillis();
	Result result = testClassifierM(h,D_test);
	long after_test = System.currentTimeMillis();

	result.setValue("N_train",D_train.numInstances());
	result.setValue("N_test",D_test.numInstances());
	result.setValue("LCard_train",MLUtils.labelCardinality(D_train));
	result.setValue("LCard_test",MLUtils.labelCardinality(D_test));

	result.setValue("Build_time",(after - before)/1000.0);
	result.setValue("Test_time",(after_test - before_test)/1000.0);
	result.setValue("Total_time",(after_test - before)/1000.0);

	result.setInfo("Classifier_name",h.getClass().getName());
	result.setInfo("Classifier_ops",Arrays.toString(h.getOptions()));
	result.setInfo("Classifier_info",h.toString());
	result.setInfo("Dataset_name",MLUtils.getDatasetName(D_train));
	//result.setInfo("Maxfreq_set",MLUtils.mostCommonCombination(D_train,result.L));

	if (h instanceof MultiTargetClassifier || isMT(D_test)) {
		result.setInfo("Type","MT");
	}
	else if (h instanceof MultiLabelClassifier) {
		result.setInfo("Type","ML");
	}
	result.setInfo("Threshold",MLEvalUtils.getThreshold(result.predictions,D_train,top)); // <-- only relevant to ML (for now), but we'll put it in here in any case
	result.setInfo("Verbosity",vop);
	result.output = Result.getStats(result, vop);
	return result;
}