I have a dataframe 'df' and I created a train, validation, and test dataset from this. From the test data, I want to add a column with the predicted spending amount ('Spending') from the Decision Tree Regressor model. I got the following error message though. Is there some way to resolve this?
Dataframe: df
sequence_number US source_a source_c source_b source_d source_e source_m source_o source_h ... source_x source_w Freq last_update_days_ago 1st_update_days_ago Web order Gender=male Address_is_res Purchase Spending
0 1 1 0 0 1 0 0 0 0 0 ... 0 0 2 3662 3662 1 0 1 1 128
1 2 1 0 0 0 0 1 0 0 0 ... 0 0 0 2900 2900 1 1 0 0 0
2 3 1 0 0 0 0 0 0 0 0 ... 0 0 2 3883 3914 0 0 0 1 127
3 4 1 0 1 0 0 0 0 0 0 ... 0 0 1 829 829 0 1 0 0 0
4 5 1 0 1 0 0 0 0 0 0 ... 0 0 1 869 869 0 0 0 0 0
5 rows × 25 columns
X = df.drop(columns=['Purchase'])
y = df['Purchase']
# Split the data in 40:35:25 for train:valid:test dataset
X_train, X_rem, y_train, y_rem = train_test_split(X, y, train_size = 0.4)
X_valid, X_test, y_valid, y_test = train_test_split(X_rem, y_rem, test_size = 500/1200)
logit = LogisticRegression(penalty = "l2", C = 1e42, solver = 'liblinear')
logit.fit(X_train, y_train)
# User grid search to find optimized tree
param_grid = {
'max_depth': [5, 10, 15, 20, 25],
'min_impurity_decrease': [0, 0.001, 0.005, 0.01],
'min_samples_split': [10, 20, 30, 40, 50],
}
gridSearch = GridSearchCV(DecisionTreeRegressor(), param_grid, cv = 5, n_jobs = -1)
gridSearch.fit(X_train, y_train)
print("Initial parameters: ", gridSearch.best_params_)
param_grid = {
'max_depth': [3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
'min_impurity_decrease': [0, 0.001, 0.002, 0.003, 0.005, 0.006, 0.007, 0.008],
'min_samples_split': [14, 15, 16, 18, 20, ],
}
gridSearch = GridSearchCV(DecisionTreeRegressor(), param_grid, cv = 5, n_jobs = -1)
gridSearch.fit(X_train, y_train)
print("Improved parameters: ", gridSearch.best_params_)
regTree = gridSearch.best_estimator_
score_analysis = X_test.assign(Predicted_Score = logit.predict(X_test),
Predicted_Spending = regTree.predict(X_test['Spending']))
score_analysis.head()
Error message:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/var/folders/wv/42dn23fd1cb0czpvqdnb6zw00000gn/T/ipykernel_24044/2039087822.py in <module>
1 # Regression trees model chosen
2 score_analysis = X_test.assign(Predicted_Score = logit.predict(X_test),
----> 3 Predicted_Spending = regTree.predict(X_test['Spending']))
4 score_analysis.head()
~/opt/miniconda3/lib/python3.9/site-packages/sklearn/tree/_classes.py in predict(self, X, check_input)
465 """
466 check_is_fitted(self)
--> 467 X = self._validate_X_predict(X, check_input)
468 proba = self.tree_.predict(X)
469 n_samples = X.shape[0]
~/opt/miniconda3/lib/python3.9/site-packages/sklearn/tree/_classes.py in _validate_X_predict(self, X, check_input)
431 """Validate the training data on predict (probabilities)."""
432 if check_input:
--> 433 X = self._validate_data(X, dtype=DTYPE, accept_sparse="csr", reset=False)
434 if issparse(X) and (
435 X.indices.dtype != np.intc or X.indptr.dtype != np.intc
~/opt/miniconda3/lib/python3.9/site-packages/sklearn/base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
564 raise ValueError("Validation should be done on X, y or both.")
565 elif not no_val_X and no_val_y:
--> 566 X = check_array(X, **check_params)
567 out = X
568 elif no_val_X and not no_val_y:
~/opt/miniconda3/lib/python3.9/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
767 # If input is 1D raise error
768 if array.ndim == 1:
--> 769 raise ValueError(
770 "Expected 2D array, got 1D array instead:\narray={}.\n"
771 "Reshape your data either using array.reshape(-1, 1) if "
ValueError: Expected 2D array, got 1D array instead:
array=[ 0. 160. 199. 0. 0. 0. 0. 0. 0. 130. 0. 322.
0. 459. 95. 55. 0. 161. 274. 32. 0. 0. 1446. 10.
160. 0. 0. 0. 489. 163. 179. 56. 0. 0. 0. 163.
0. 103. 0. 0. 0. 0. 0. 326. 228. 265. 0. 598.
0. 0. 0. 0. 0. 54. 247. 490. 184. 164. 0. 0.
71. 0. 0. 0. 150. 0. 90. 0. 0. 0. 129. 150.
163. 98. 182. 324. 128. 138. 163. 0. 83. 114. 1416. 0.
188. 170. 0. 0. 0. 0. 160. 0. 158. 0. 577. 335.
0. 456. 0. 0. 406. 0. 159. 167. 0. 253. 226. 109.
44. 0. 162. 0. 0. 0. 85. 0. 0. 0. 282. 0.
243. 0. 50. 0. 0. 350. 0. 134. 100. 0. 0. 0.
69. 0. 38. 331. 0. 0. 164. 44. 130. 0. 292. 1255.
0. 54. 0. 138. 161. 0. 0. 0. 220. 0. 232. 0.
91. 0. 0. 30. 138. 54. 451. 819. 0. 0. 264. 0.
0. 0. 135. 0. 186. 439. 255. 160. 35. 0. 158. 0.
0. 0. 0. 13. 0. 0. 588. 0. 0. 0. 128. 195.
0. 34. 0. 0. 0. 128. 1294. 0. 0. 390. 0. 34.
0. 0. 0. 166. 0. 128. 35. 0. 0. 0. 149. 4.
114. 0. 255. 6. 217. 1441. 0. 56. 0. 0. 136. 0.
0. 219. 0. 432. 0. 0. 139. 43. 35. 33. 262. 143.
0. 44. 43. 0. 0. 246. 98. 0. 0. 139. 0. 85.
0. 0. 161. 0. 129. 471. 639. 85. 129. 31. 38. 98.
0. 229. 54. 390. 0. 174. 215. 163. 153. 0. 271. 0.
130. 361. 146. 116. 0. 0. 128. 0. 32. 161. 197. 130.
0. 135. 0. 120. 160. 0. 402. 0. 375. 412. 0. 0.
100. 0. 0. 429. 0. 128. 0. 0. 0. 25. 100. 184.
203. 0. 0. 58. 0. 191. 0. 0. 0. 0. 517. 100.
136. 0. 207. 240. 185. 0. 97. 116. 496. 57. 146. 321.
0. 0. 0. 67. 162. 0. 0. 415. 0. 0. 0. 0.
30. 0. 83. 128. 98. 99. 0. 35. 0. 255. 81. 429.
0. 0. 0. 33. 161. 163. 0. 45. 0. 0. 0. 0.
525. 0. 311. 0. 0. 0. 0. 0. 427. 0. 116. 547.
0. 189. 0. 160. 0. 0. 161. 0. 0. 425. 162. 98.
44. 0. 303. 1133. 161. 0. 0. 0. 0. 0. 43. 0.
0. 30. 437. 0. 1030. 163. 270. 0. 130. 34. 209. 191.
0. 0. 318. 189. 508. 0. 0. 259. 114. 0. 230. 54.
0. 495. 0. 40. 160. 0. 161. 0. 53. 362. 164. 161.
0. 0. 39. 182. 128. 73. 0. 80. 274. 41. 130. 104.
0. 236. 40. 44. 277. 0. 163. 43. 369. 132. 20. 0.
58. 35. 345. 90. 0. 0. 0. 0. 0. 0. 0. 0.
0. 314. 0. 0. 410. 0. 159. 0. 128. 160. 0. 0.
0. 130. 162. 229. 0. 162. 0. 0.].
Reshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample.