Allow multi-scoring and return a list at the same time in sklearn cross_validate

Viewed 26

I want to return multiple-scoring and, at the same time, the probabilities scores of each prediction to build the ROC curve. The problem is that I can't return something different to a number because the error "scoring must return a number" raised. I let my code below:

from sklearn.datasets import make_classification
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.metrics import confusion_matrix
from sklearn.metrics import make_scorer
from sklearn.model_selection import cross_validate
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import MinMaxScaler

pipeline_list = []
pipeline_list.append(('scaler',MinMaxScaler()))
pipeline_list.append(('model',LogisticRegression(max_iter=100000)))

pipeline = Pipeline(pipeline_list)

X, y = make_classification(n_samples=100, n_features=20, n_informative=15, n_redundant=5, random_state=None)

kf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10, random_state=None)
def confusion_matrix_tn(y_true, y_pred):
    cm = confusion_matrix(y_true, y_pred)
    return cm[0, 0]

def confusion_matrix_fp(y_true, y_pred):
    cm = confusion_matrix(y_true, y_pred)
    return cm[0, 1]

def confusion_matrix_fn(y_true, y_pred):
    cm = confusion_matrix(y_true, y_pred)
    return cm[1, 0]

def confusion_matrix_tp(y_true, y_pred):
    cm = confusion_matrix(y_true, y_pred)
    return cm[1, 1]


scoring = {'acc': 'accuracy',
            'prec_micro': 'precision_micro',
            'rec_micro': 'recall_micro',
            'auc':'roc_auc',
            'f1_score':'f1_micro',
            'true_neg':make_scorer(confusion_matrix_tn),
            'false_pos':make_scorer(confusion_matrix_fp, greater_is_better=False),
            'false_neg':make_scorer(confusion_matrix_fn, greater_is_better=False),
            'true_pos':make_scorer(confusion_matrix_tp)
           }

scores = cross_validate(pipeline, X, y, scoring=scoring,
                     cv=kf, return_train_score=False,return_estimator=True,verbose=1,n_jobs=-1)
0 Answers
Related