Python sklearn.metrics.matthews_corrcoef() Examples

def multi_class_classification(data_X,data_Y):
    '''
    calculate multi-class classification and return related evaluation metrics
    '''

    svc = svm.SVC(C=1, kernel='linear')
    # X_train, X_test, y_train, y_test = train_test_split( data_X, data_Y, test_size=0.4, random_state=0) 
    clf = svc.fit(data_X, data_Y) #svm
    # array = svc.coef_
    # print array
    predicted = cross_val_predict(clf, data_X, data_Y, cv=2)
    print "accuracy",metrics.accuracy_score(data_Y, predicted)
    print "f1 score macro",metrics.f1_score(data_Y, predicted, average='macro') 
    print "f1 score micro",metrics.f1_score(data_Y, predicted, average='micro') 
    print "precision score",metrics.precision_score(data_Y, predicted, average='macro') 
    print "recall score",metrics.recall_score(data_Y, predicted, average='macro') 
    print "hamming_loss",metrics.hamming_loss(data_Y, predicted)
    print "classification_report", metrics.classification_report(data_Y, predicted)
    print "jaccard_similarity_score", metrics.jaccard_similarity_score(data_Y, predicted)
    # print "log_loss", metrics.log_loss(data_Y, predicted)
    print "zero_one_loss", metrics.zero_one_loss(data_Y, predicted)
    # print "AUC&ROC",metrics.roc_auc_score(data_Y, predicted)
    # print "matthews_corrcoef", metrics.matthews_corrcoef(data_Y, predicted) 

def evaluation_analysis(true_label,predicted): 
    '''
    return all metrics results
    '''
    print "accuracy",metrics.accuracy_score(true_label, predicted)
    print "f1 score macro",metrics.f1_score(true_label, predicted, average='macro')     
    print "f1 score micro",metrics.f1_score(true_label, predicted, average='micro') 
    print "precision score",metrics.precision_score(true_label, predicted, average='macro') 
    print "recall score",metrics.recall_score(true_label, predicted, average='macro') 
    print "hamming_loss",metrics.hamming_loss(true_label, predicted)
    print "classification_report", metrics.classification_report(true_label, predicted)
    print "jaccard_similarity_score", metrics.jaccard_similarity_score(true_label, predicted)
    print "log_loss", metrics.log_loss(true_label, predicted)
    print "zero_one_loss", metrics.zero_one_loss(true_label, predicted)
    print "AUC&ROC",metrics.roc_auc_score(true_label, predicted)
    print "matthews_corrcoef", metrics.matthews_corrcoef(true_label, predicted) 

def compute_metrics(task_name, pred_srs, label_srs):
    assert len(pred_srs) == len(label_srs)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(label_srs, pred_srs)}
    elif task_name == "sst":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "mrpc":
        return acc_and_f1(pred_srs, label_srs)
    elif task_name == "stsb":
        return pearson_and_spearman(pred_srs, label_srs)
    elif task_name == "qqp":
        return acc_and_f1(pred_srs, label_srs)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    else:
        raise KeyError(task_name) 

def score_eval_func(self, y_true, y_pred, mode='accuracy'):
        prob_pred = two_class_encoding(y_pred)
        label_pred = np.argmax(prob_pred, axis=1)
        if mode == 'accuracy':
            score_pred = accuracy_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'precision':
            score_pred = precision_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'recall':
            score_pred = recall_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'f1':
            score_pred = f1_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'MCC':
            score_pred = matthews_corrcoef(y_true=y_true, y_pred=label_pred)
        else:
            raise ValueError('Score function mode unrecognized! Must from one in the list '
                             '[\'accuracy\', \'precision\',\'recall\',\'f1\',\'MCC\']')

        return score_pred 

def validation_step(self, inputs, model: tf.keras.Model, metrics=None):
    if self.metric_type == 'accuracy':
      return super(SentencePredictionTask,
                   self).validation_step(inputs, model, metrics)
    features, labels = inputs
    outputs = self.inference_step(features, model)
    loss = self.build_losses(
        labels=labels, model_outputs=outputs, aux_losses=model.losses)
    logs = {self.loss: loss}
    if self.metric_type == 'matthews_corrcoef':
      logs.update({
          'sentence_prediction':
              tf.expand_dims(
                  tf.math.argmax(outputs['sentence_prediction'], axis=1),
                  axis=0),
          'labels':
              labels,
      })
    if self.metric_type == 'pearson_spearman_corr':
      logs.update({
          'sentence_prediction': outputs['sentence_prediction'],
          'labels': labels,
      })
    return logs 

def get_metric_fn(task_name):
    if task_name == "cola":
        return lambda p, l: {"mcc": matthews_corrcoef(p, l)}
    if task_name == "sst-2":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "mrpc":
        return acc_and_f1
    if task_name == "sts-b":
        return pearson_and_spearman
    if task_name == "qqp":
        return acc_and_f1
    if task_name == "mnli":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "mnli-mm":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "qnli":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "rte":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "wnli":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    raise KeyError(task_name) 

def compute_metrics(task_name, preds, labels):
    assert len(preds) == len(labels)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif task_name == "sst-2":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mrpc":
        return acc_and_f1(preds, labels)
    elif task_name == "sts-b":
        return pearson_and_spearman(preds, labels)
    elif task_name == "qqp":
        return acc_and_f1(preds, labels)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(preds, labels)}
    else:
        raise KeyError(task_name) 

def compute_metrics(task_name, preds, labels):
    assert len(preds) == len(labels)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif task_name == "sst-2":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mrpc":
        return acc_and_f1(preds, labels)
    elif task_name == "sts-b":
        return pearson_and_spearman(preds, labels)
    elif task_name == "qqp":
        return acc_and_f1(preds, labels)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(preds, labels)}
    else:
        raise KeyError(task_name) 

def test_confusion_matrix_binary():
    # Test confusion matrix - binary classification case
    y_true, y_pred, _ = make_prediction(binary=True)

    def test(y_true, y_pred):
        cm = confusion_matrix(y_true, y_pred)
        assert_array_equal(cm, [[22, 3], [8, 17]])

        tp, fp, fn, tn = cm.flatten()
        num = (tp * tn - fp * fn)
        den = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))

        true_mcc = 0 if den == 0 else num / den
        mcc = matthews_corrcoef(y_true, y_pred)
        assert_array_almost_equal(mcc, true_mcc, decimal=2)
        assert_array_almost_equal(mcc, 0.57, decimal=2)

    test(y_true, y_pred)
    test([str(y) for y in y_true],
         [str(y) for y in y_pred]) 

def test_confusion_matrix_binary():
    # Test confusion matrix - binary classification case
    y_true, y_pred, _ = make_prediction(binary=True)

    def test(y_true, y_pred):
        cm = confusion_matrix(y_true, y_pred)
        assert_array_equal(cm, [[22, 3], [8, 17]])

        tp, fp, fn, tn = cm.flatten()
        num = (tp * tn - fp * fn)
        den = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))

        true_mcc = 0 if den == 0 else num / den
        mcc = matthews_corrcoef(y_true, y_pred)
        assert_array_almost_equal(mcc, true_mcc, decimal=2)
        assert_array_almost_equal(mcc, 0.57, decimal=2)

    test(y_true, y_pred)
    test([str(y) for y in y_true],
         [str(y) for y in y_pred]) 

def matthews_correlation_coefficient_scorer(
    golds: ndarray,
    probs: Optional[ndarray],
    preds: ndarray,
    uids: Optional[List[str]] = None,
) -> Dict[str, float]:
    """Matthews correlation coefficient (MCC).

    Args:
      golds: Ground truth values.
      probs: Predicted probabilities.
      preds: Predicted values.
      uids: Unique ids, defaults to None.

    Returns:
      Matthews correlation coefficient score.
    """
    # Convert probabilistic label to hard label
    if len(golds.shape) == 2:
        golds = prob_to_pred(golds)

    return {"matthews_corrcoef": matthews_corrcoef(golds, preds)} 

def get_scores(scores,y,label):
    '''
    Returns a dictionary of metrics for a given classification of the data (given by Cross_val_predict).
    scores: list
        Classifier predictions on data
    y: list
        True Class labels
    label: string
        Name of the classifier used
    '''

    roc_auc = metrics.roc_auc_score(y, scores,average=None)
    print("roc_auc (No-Av): %0.4f " % (roc_auc))
    roc_auc = metrics.roc_auc_score(y, scores,average='weighted')
    print("roc_auc (weighted-Av): %0.4f " % (roc_auc))
    f1_pos = metrics.f1_score(y, scores,average='binary')
    print("POS f1: %0.4f  " % (f1_pos))
    av_PR = metrics.average_precision_score(y, scores) # corresponds to the area under the precision-recall curve
    print("Av_precision (Prec-Recall AUC): %0.3f " % (av_PR))
    accuracy = metrics.accuracy_score(y, scores)
    print("Accuracy: %0.3f " % (accuracy))
    precision,recall,fscore,support = metrics.precision_recall_fscore_support(y, scores,average='binary')
    print("Precision: %0.3f " % (precision))
    print("Recall: %0.3f " % (recall))
    # print("fscore(fBeta): %0.4f  [%s]" % (fscore, label))
    mcc = metrics.matthews_corrcoef(y, scores)
    print("MCC: %0.3f " % (mcc))

    results_dict = {'roc_auc(macro)':roc_auc,'f1_Pos':f1_pos,'accuracy':accuracy,
    'precision':precision,'recall':recall,
    # 'fscore-fBeta':fscore,
    'average Precision':av_PR,'mcc':mcc
    }
    results_dict = {k:round(float(v),4) for k, v in results_dict.items()}


    return results_dict



# E:\Dropbox\Dropbox\Protein Cleavage Prediction\data\NeuroPred\V4\features-11_8_KR.csv 

def test_matthews_corrcoef(self):
        msg = 'multilabel-indicator is not supported'
        with pytest.raises(ValueError, match=msg):
            self.df.metrics.matthews_corrcoef() 

def test_matthews_corrcoef_overflow():
    # https://github.com/scikit-learn/scikit-learn/issues/9622
    rng = np.random.RandomState(20170906)

    def mcc_safe(y_true, y_pred):
        conf_matrix = confusion_matrix(y_true, y_pred)
        true_pos = conf_matrix[1, 1]
        false_pos = conf_matrix[1, 0]
        false_neg = conf_matrix[0, 1]
        n_points = len(y_true)
        pos_rate = (true_pos + false_neg) / n_points
        activity = (true_pos + false_pos) / n_points
        mcc_numerator = true_pos / n_points - pos_rate * activity
        mcc_denominator = activity * pos_rate * (1 - activity) * (1 - pos_rate)
        return mcc_numerator / np.sqrt(mcc_denominator)

    def random_ys(n_points):    # binary
        x_true = rng.random_sample(n_points)
        x_pred = x_true + 0.2 * (rng.random_sample(n_points) - 0.5)
        y_true = (x_true > 0.5)
        y_pred = (x_pred > 0.5)
        return y_true, y_pred

    for n_points in [100, 10000, 1000000]:
        arr = np.repeat([0., 1.], n_points)  # binary
        assert_almost_equal(matthews_corrcoef(arr, arr), 1.0)
        arr = np.repeat([0., 1., 2.], n_points)  # multiclass
        assert_almost_equal(matthews_corrcoef(arr, arr), 1.0)

        y_true, y_pred = random_ys(n_points)
        assert_almost_equal(matthews_corrcoef(y_true, y_true), 1.0)
        assert_almost_equal(matthews_corrcoef(y_true, y_pred),
                            mcc_safe(y_true, y_pred)) 

def compute_mcc(labels, preds):
    # Compute ROC curve and ROC area for each class
    mcc = matthews_corrcoef(labels.flatten(), preds.flatten())
    return mcc 

def compute_mcc(labels, preds):
    # Compute ROC curve and ROC area for each class
    mcc = matthews_corrcoef(labels.flatten(), preds.flatten())
    return mcc 

def compute_mcc(labels, preds):
    # Compute ROC curve and ROC area for each class
    mcc = matthews_corrcoef(labels.flatten(), preds.flatten())
    return mcc 

def compute_mcc(labels, preds):
    # Compute ROC curve and ROC area for each class
    mcc = matthews_corrcoef(labels.flatten(), preds.flatten())
    return mcc 

def compute_mcc(labels, preds):
    # Compute ROC curve and ROC area for each class
    mcc = matthews_corrcoef(labels.flatten(), preds.flatten())
    return mcc 

def compute_mcc(predicts, labels):
    return 100.0 * matthews_corrcoef(labels, predicts) 

def test_matthews_corrcoef_nan():
    assert_equal(matthews_corrcoef([0], [1]), 0.0)
    assert_equal(matthews_corrcoef([0, 0], [0, 1]), 0.0) 

def reduce_aggregated_logs(self, aggregated_logs):
    if self.metric_type == 'matthews_corrcoef':
      preds = np.concatenate(aggregated_logs['sentence_prediction'], axis=0)
      labels = np.concatenate(aggregated_logs['labels'], axis=0)
      return {
          self.metric_type: sklearn_metrics.matthews_corrcoef(preds, labels)
      }
    if self.metric_type == 'pearson_spearman_corr':
      preds = np.concatenate(aggregated_logs['sentence_prediction'], axis=0)
      labels = np.concatenate(aggregated_logs['labels'], axis=0)
      pearson_corr = stats.pearsonr(preds, labels)[0]
      spearman_corr = stats.spearmanr(preds, labels)[0]
      corr_metric = (pearson_corr + spearman_corr) / 2
      return {self.metric_type: corr_metric} 

def test_matthews_corrcoef():
    rng = np.random.RandomState(0)
    y_true = ["a" if i == 0 else "b" for i in rng.randint(0, 2, size=20)]

    # corrcoef of same vectors must be 1
    assert_almost_equal(matthews_corrcoef(y_true, y_true), 1.0)

    # corrcoef, when the two vectors are opposites of each other, should be -1
    y_true_inv = ["b" if i == "a" else "a" for i in y_true]
    assert_almost_equal(matthews_corrcoef(y_true, y_true_inv), -1)

    y_true_inv2 = label_binarize(y_true, ["a", "b"])
    y_true_inv2 = np.where(y_true_inv2, 'a', 'b')
    assert_almost_equal(matthews_corrcoef(y_true, y_true_inv2), -1)

    # For the zero vector case, the corrcoef cannot be calculated and should
    # result in a RuntimeWarning
    mcc = assert_warns_message(RuntimeWarning, 'invalid value encountered',
                               matthews_corrcoef, [0, 0, 0, 0], [0, 0, 0, 0])

    # But will output 0
    assert_almost_equal(mcc, 0.)

    # And also for any other vector with 0 variance
    mcc = assert_warns_message(RuntimeWarning, 'invalid value encountered',
                               matthews_corrcoef, y_true, ['a'] * len(y_true))

    # But will output 0
    assert_almost_equal(mcc, 0.)

    # These two vectors have 0 correlation and hence mcc should be 0
    y_1 = [1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1]
    y_2 = [1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1]
    assert_almost_equal(matthews_corrcoef(y_1, y_2), 0.)

    # Check that sample weight is able to selectively exclude
    mask = [1] * 10 + [0] * 10
    # Now the first half of the vector elements are alone given a weight of 1
    # and hence the mcc will not be a perfect 0 as in the previous case
    assert_raises(AssertionError, assert_almost_equal,
                  matthews_corrcoef(y_1, y_2, sample_weight=mask), 0.) 

def test_matthews_corrcoef(self):
        if not pdml.compat._SKLEARN_ge_019:
            with pytest.raises(ValueError, match='multiclass is not supported'):
                self.df.metrics.matthews_corrcoef() 

def metric_mcc(logits, labels):
  predicts = np.argmax(logits, axis=1)
  return matthews_corrcoef(labels, predicts) 

def test_matthews_corrcoef_against_jurman():
    # Check that the multiclass matthews_corrcoef agrees with the definition
    # presented in Jurman, Riccadonna, Furlanello, (2012). A Comparison of MCC
    # and CEN Error Measures in MultiClass Prediction
    rng = np.random.RandomState(0)
    y_true = rng.randint(0, 2, size=20)
    y_pred = rng.randint(0, 2, size=20)
    sample_weight = rng.rand(20)

    C = confusion_matrix(y_true, y_pred, sample_weight=sample_weight)
    N = len(C)
    cov_ytyp = sum([
        C[k, k] * C[m, l] - C[l, k] * C[k, m]
        for k in range(N) for m in range(N) for l in range(N)
    ])
    cov_ytyt = sum([
        C[:, k].sum() *
        np.sum([C[g, f] for f in range(N) for g in range(N) if f != k])
        for k in range(N)
    ])
    cov_ypyp = np.sum([
        C[k, :].sum() *
        np.sum([C[f, g] for f in range(N) for g in range(N) if f != k])
        for k in range(N)
    ])
    mcc_jurman = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)
    mcc_ours = matthews_corrcoef(y_true, y_pred, sample_weight)

    assert_almost_equal(mcc_ours, mcc_jurman, 10) 

def test_matthews_corrcoef_against_numpy_corrcoef():
    rng = np.random.RandomState(0)
    y_true = rng.randint(0, 2, size=20)
    y_pred = rng.randint(0, 2, size=20)

    assert_almost_equal(matthews_corrcoef(y_true, y_pred),
                        np.corrcoef(y_true, y_pred)[0, 1], 10) 

def compute_mcc(predicts, labels):
    return 100.0 * matthews_corrcoef(labels, predicts) 

def test_matthews_corrcoef_against_jurman():
    # Check that the multiclass matthews_corrcoef agrees with the definition
    # presented in Jurman, Riccadonna, Furlanello, (2012). A Comparison of MCC
    # and CEN Error Measures in MultiClass Prediction
    rng = np.random.RandomState(0)
    y_true = rng.randint(0, 2, size=20)
    y_pred = rng.randint(0, 2, size=20)
    sample_weight = rng.rand(20)

    C = confusion_matrix(y_true, y_pred, sample_weight=sample_weight)
    N = len(C)
    cov_ytyp = sum([
        C[k, k] * C[m, l] - C[l, k] * C[k, m]
        for k in range(N) for m in range(N) for l in range(N)
    ])
    cov_ytyt = sum([
        C[:, k].sum() *
        np.sum([C[g, f] for f in range(N) for g in range(N) if f != k])
        for k in range(N)
    ])
    cov_ypyp = np.sum([
        C[k, :].sum() *
        np.sum([C[f, g] for f in range(N) for g in range(N) if f != k])
        for k in range(N)
    ])
    mcc_jurman = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)
    mcc_ours = matthews_corrcoef(y_true, y_pred, sample_weight)

    assert_almost_equal(mcc_ours, mcc_jurman, 10) 

def matthews_corr(preds, labels):
    return {
        "matthews_corr": matthews_corrcoef(labels, preds),
    }