I've run a Grid Search for a Random Forest Classifier with the scoring set to precision. The best score is 0.9639, great! But what does that tell me? Because, when I run the RF Classifier with the best parameters, I get a precision score of .85.
Thanks for helping!
from sklearn.model_selection import GridSearchCV
params_rf = { 'criterion': ['gini','entropy'],
'min_samples_leaf': [1,3,5,10,20,25,30,35,40,45,50]
}
rf = RandomForestClassifier(n_estimators=2500,
random_state=SEED,
max_features='sqrt',
bootstrap=True,
oob_score=True,
n_jobs=-1,
class_weight={1:3},
warm_start=True,
refit=True,
return_train_score=True)
prec = make_scorer(precision_score)
grid_rf = GridSearchCV(estimator = rf,
param_grid=params_rf,scoring=prec,cv=10,n_jobs=-1,
verbose=True)
grid_rf.fit(X_resampled,y_resampled)
y_pred = grid_rf.predict(X_test)
best_hyperparams = grid_rf.best_params_
best_score = grid_rf.best_score_
best_estimator = grid_rf.best_estimator_
print('Best hyperparameters:\n', best_hyperparams)
print('Best score:\n', best_score.round(4))
print('Best estimator:\n', best_estimator)
Fitting 10 folds for each of 22 candidates, totalling 220
fits
Best hyperparameters:
{'criterion': 'entropy', 'min_samples_leaf': 1}
Best score:
0.9639
Best estimator:
RandomForestClassifier(class_weight={1: 3},
criterion='entropy',
n_estimators=2500, n_jobs=-1, oob_score=True,
random_state=121864, warm_start=True)
# Run a Random Forest Classifier (A)
rf_A = RandomForestClassifier(class_weight={1: 3},
criterion='entropy',
n_estimators=2500, n_jobs=-1, oob_score=True,
random_state=121864,
warm_start=True,min_samples_leaf=1
)
rf_A.fit(X_resampled,y_resampled)
y_pred_A=(rf_A.predict(X_test))
importances_rf_A = pd.Series(rf_A.feature_importances_, index
= Features.columns)
# Create a heatmap of the confusion matrix
matrix =confusion_matrix(y_test,y_pred_A).round(2)
text = np.array([['True Positive', 'False Negative'],
['False Positive', 'True Negative']])
# combining text with values
formatted_text = (np.asarray(["{0}\n{1:.0f}".format(
text, matrix) for text, matrix in zip(text.flatten(),
matrix.flatten())])).reshape(2,2)
# drawing heatmap
fig, ax = plt.subplots(figsize=(7,2))
sns.set(font_scale=1.3)
ax = sns.heatmap(matrix, annot=formatted_text, fmt="",
linewidth=1,cbar=False)
ax.set_title('Confusion Matrix', size = 18)
plt.show()
target_names = ['Fully Paid', 'Not Fully Paid']
print(classification_report(y_test,y_pred_A,
target_names=target_names,zero_division=1))
# Create a horizontal bar chart of the feature importances
# Sort importances_dt
sorted_importances_rf_A = importances_rf_A.sort_values()
sorted_importances_descend_A =
importances_rf_A.sort_values(ascending=False)
# Make a horizontal bar plot
#fig, ax = plt.subplots()
plt.figure(figsize=(20, 15))
sorted_importances_rf_A.plot(kind='barh',grid=True)
#ax.set_xticks(,minor=True)
plt.show()
print(sorted_importances_descend_A.cumsum())
Ruuning the RF classifier with the best hyper parameters:
precision recall f1-score support
Fully Paid 0.87 0.65 0.75 1609
Not Fully Paid 0.22 0.51 0.31 307
accuracy 0.63 1916
macro avg 0.55 0.58 0.53 1916
weighted avg 0.77 0.63 0.68 1916