How can I save my model results from this function so I can refer to them later?

user15404111 0
Viewed 20

I have a function to train and test regression models. I am using it to test 10 different models. How can I save the 4 evaluation metrics that I calculate so they may be referred to later? Functions:

def relative_squared_error(true, pred):
    true_mean = np.mean(true)
    squared_error_num = np.sum(np.square(true - pred))
    squared_error_den = np.sum(np.square(true - true_mean))
    rse_loss = squared_error_num / squared_error_den
    return rse_loss


def train_and_test(model, name=""):
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    mse = cross_val_score(model, X_test, y_test, scoring='neg_mean_squared_error', cv=10, n_jobs=-1)
    rmse = cross_val_score(model, X_test, y_test, scoring='neg_root_mean_squared_error', cv=10, n_jobs=-1)
    mae = cross_val_score(model, X_test, y_test, scoring='neg_mean_absolute_error', cv=10, n_jobs=-1)
    rse = relative_squared_error(y_test, y_pred)
    
    print('Mean MSE: %.4f' % abs(np.mean(mse)))
    print('Mean RMSE: %.4f' % abs(np.mean(rmse)))
    print('Mean MAE: %.4f' % abs(np.mean(mae)))
    print('RSE Loss: %.4f' % rse)

Example using the function:

%%time
#Linear Regression

from sklearn.linear_model import LinearRegression

lin_reg = Pipeline([('transformer', pipeline),
                    ('regressor', LinearRegression())
                   ])

train_and_test(lin_reg, 'Linear Regression')
regression scikit-learn python
1 Answers

One of the easiest way would be to save the metrics in the dictionary.

import json

def train_and_test(model, name=""):
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    mse = cross_val_score(model, X_test, y_test, scoring='neg_mean_squared_error', cv=10, n_jobs=-1)
    rmse = cross_val_score(model, X_test, y_test, scoring='neg_root_mean_squared_error', cv=10, n_jobs=-1)
    mae = cross_val_score(model, X_test, y_test, scoring='neg_mean_absolute_error', cv=10, n_jobs=-1)
    rse = relative_squared_error(y_test, y_pred)
    
    print('Mean MSE: %.4f' % abs(np.mean(mse)))
    print('Mean RMSE: %.4f' % abs(np.mean(rmse)))
    print('Mean MAE: %.4f' % abs(np.mean(mae)))
    print('RSE Loss: %.4f' % rise)

    metric_dictionary = {'Mean MSE': abs(np.mean(mse)), 'Mean RMSE': abs(np.mean(rmse)), 'Mean MAE': abs(np.mean(mae)), 'RSE': rise}
    
    return metric_dictionary

scores_dictionary = {}

lin_reg = Pipeline([('transformer', pipeline),
                    ('regressor', LinearRegression())
                   ])

scores_dictionary['Linear Regression'] = train_and_test(lin_reg, 'Linear Regression')

rf_reg = Pipeline([('transformer', pipeline),
                    ('regressor', RandomForest())
                   ])

scores_dictionary['Random Forest'] = train_and_test(rf_reg, 'Random Forest')
Andrey Lukyanenko0
Related