I want to return multiple-scoring and, at the same time, the probabilities scores of each prediction to build the ROC curve. The problem is that I can't return something different to a number because the error "scoring must return a number" raised. I let my code below:
from sklearn.datasets import make_classification
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.metrics import confusion_matrix
from sklearn.metrics import make_scorer
from sklearn.model_selection import cross_validate
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import MinMaxScaler
pipeline_list = []
pipeline_list.append(('scaler',MinMaxScaler()))
pipeline_list.append(('model',LogisticRegression(max_iter=100000)))
pipeline = Pipeline(pipeline_list)
X, y = make_classification(n_samples=100, n_features=20, n_informative=15, n_redundant=5, random_state=None)
kf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10, random_state=None)
def confusion_matrix_tn(y_true, y_pred):
cm = confusion_matrix(y_true, y_pred)
return cm[0, 0]
def confusion_matrix_fp(y_true, y_pred):
cm = confusion_matrix(y_true, y_pred)
return cm[0, 1]
def confusion_matrix_fn(y_true, y_pred):
cm = confusion_matrix(y_true, y_pred)
return cm[1, 0]
def confusion_matrix_tp(y_true, y_pred):
cm = confusion_matrix(y_true, y_pred)
return cm[1, 1]
scoring = {'acc': 'accuracy',
'prec_micro': 'precision_micro',
'rec_micro': 'recall_micro',
'auc':'roc_auc',
'f1_score':'f1_micro',
'true_neg':make_scorer(confusion_matrix_tn),
'false_pos':make_scorer(confusion_matrix_fp, greater_is_better=False),
'false_neg':make_scorer(confusion_matrix_fn, greater_is_better=False),
'true_pos':make_scorer(confusion_matrix_tp)
}
scores = cross_validate(pipeline, X, y, scoring=scoring,
cv=kf, return_train_score=False,return_estimator=True,verbose=1,n_jobs=-1)