How to draw ROC and PR curves for a classification model trained with KFold CV

user_17118152 0
Viewed 230

How should I draw the ROC and PR curves for this NN model which I am training with 10 fold cross-validation?

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
from tensorflow.keras.losses import sparse_categorical_crossentropy
from tensorflow.keras.optimizers import Adam
from sklearn.model_selection import KFold
from numpy import loadtxt
import numpy as np
import pandas as pd

from google.colab import files
uploaded = files.upload()

dataset = loadtxt('mod_dfn.csv', delimiter=',')

X = dataset[:,0:25]
y = dataset[:,25]

kfold = KFold(n_splits=10, shuffle=True)

fold_no = 1
for train, test in kfold.split(X, y):

  model = Sequential()
  model.add(Dense(12, input_dim=25, activation='relu'))
  model.add(Dense(8, activation='relu'))
  model.add(Dense(1, activation='sigmoid'))

  model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

  print('------------------------------------------------------------------------')
  print(f'Training for fold {fold_no} ...')

  history = model.fit(X[train], y[train], batch_size=10, epochs=150, verbose=0)

  scores = model.evaluate(X[test], y[test], verbose=0)
  print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%')
  acc_per_fold.append(scores[1] * 100)
  loss_per_fold.append(scores[0])

  fold_no = fold_no + 1
machine-learning neural-network
1 Answers

You can use RocCurveDisplay and PrecisionRecallDisplay for this purpose.

Try adding these lines just before last line of your code fold_no = fold_no + 1, and see if it works for you.

pred = model.predict(X[test])
# ROC curve
fpr, tpr, thresholds = metrics.roc_curve(y[test], pred)
roc_auc = metrics.auc(fpr, tpr)
roc_display = metrics.RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc)
roc_display.plot()
roc_display.figure_.savefig(f'./out/ROC_curve_for_fold#{fold_no}.jpeg')

# PR Curve
precision, recall, _ = metrics.precision_recall_curve(y[test], pred)
pr_display = metrics.PrecisionRecallDisplay(precision=precision, recall=recall)
pr_display.plot()
pr_display.figure_.savefig(f'./out/PR_curve_for_fold#{fold_no}.jpeg')
user_29615230
Related