I'm trying to train a LightGBM model on the Kaggle Iowa housing dataset and I wrote a small script to randomly try different parameters within a given range. I'm not sure what's wrong with my code, but the script returns the same score with different parameters, which shouldn't be happening. I tried the same script with Catboost and it works as expected, so I'm guessing the issue is with LGBM.
The code:
import numpy as np
import pandas as pd
from lightgbm import LGBMRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
from random import choice, randrange, uniform
complete_train = pd.read_csv(
"train.csv",
encoding = "UTF-8",
index_col = "Id")
complete_test = pd.read_csv(
"test.csv",
encoding = "UTF-8",
index_col = "Id")
def encode_impute(*datasets):
for dataset in datasets:
for column in dataset.columns:
dataset[
column].fillna(
-999,
inplace = True)
if dataset[
column].dtype == "object":
dataset[
column] = dataset[
column].astype("category", copy = False)
encode_impute(
complete_train,
complete_test)
X = complete_train.drop(
columns = "SalePrice")
y = complete_train[
"SalePrice"]
X_train, X_valid, y_train, y_valid = train_test_split(X, y)
def objective():
while True:
params = {
"boosting_type": choice(["gbdt", "goss", "dart", "rf"]),
"num_leaves": randrange(10000),
"learning_rate": uniform(0.01, 1),
"subsample_for_bin": randrange(100000000),
"min_data_in_leaf": randrange(100000000),
"reg_alpha": uniform(0, 1),
"reg_lambda": uniform(0, 1),
"feature_fraction": uniform(0, 1),
"bagging_fraction": uniform(0, 1),
"bagging_freq": randrange(1, 100)}
params["bagging_fraction"] = 1.0 if params[
"boosting_type"] == "goss" else params[
"bagging_fraction"]
model = LGBMRegressor().set_params(**params)
model.fit(X_train, y_train)
predictions = model.predict(X_valid)
error_rate = mean_absolute_error(
y_valid, predictions)
print(f"Score = {error_rate} with parameters: {params}","\n" *5)
objective()
Example of the output I'm getting:
Score = 55967.70375930444 with parameters: {'boosting_type': 'gbdt', 'num_leaves': 6455, 'learning_rate': 0.2479700848039991, 'subsample_for_bin': 83737077, 'min_data_in_leaf': 51951103, 'reg_alpha': 0.1856001984332697, 'reg_lambda': 0.7849262049058852, 'feature_fraction': 0.10550627738309537, 'bagging_fraction': 0.2613298736131875, 'bagging_freq': 96}
Score = 55967.70375930444 with parameters: {'boosting_type': 'dart', 'num_leaves': 9678, 'learning_rate': 0.28670432435369037, 'subsample_for_bin': 24246091, 'min_data_in_leaf': 559094, 'reg_alpha': 0.07261459695501371, 'reg_lambda': 0.8834743560240725, 'feature_fraction': 0.5361519020265366, 'bagging_fraction': 0.9120030047714073, 'bagging_freq': 10}
Score = 55967.70375930444 with parameters: {'boosting_type': 'goss', 'num_leaves': 4898, 'learning_rate': 0.09237499846487345, 'subsample_for_bin': 32620066, 'min_data_in_leaf': 71317820, 'reg_alpha': 0.9818297737748625, 'reg_lambda': 0.11638265354331834, 'feature_fraction': 0.4230342728468828, 'bagging_fraction': 1.0, 'bagging_freq': 64}