I am trying to get accuracy score and predict for my 3 features in training model, not understanding the error I am getting since the train/test feature dimensions are the same, I am assuming the day of week using the int representation as well as the time conversion to ordinal is working fine since those were not flagged for error, here is the code
import pandas as pd
import numpy as np
from datetime import datetime
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import SGDClassifier
from sklearn.pipeline import make_pipeline
from statsmodels.stats.proportion import proportion_confint
import seaborn as sns
import matplotlib.pyplot as plt
df_data = pd.read_parquet('est-arrival-ordered.parquet')
df_data.dropna()
df_data['ORDERED'] = np.where(df_data['ORDERED'] == 'Y', 1, 0).astype(np.uint8)
df_data['PRODUCT_LINE_CODE'] = df_data['PRODUCT_LINE_CODE'].astype(int)
df_data['ARRIVAL_HOURS_TOP_LOC'] = df_data['ARRIVAL_HOURS_TOP_LOC'].fillna(0).astype(np.uint16)
df_data['Line_PLCD'] = df_data['LINE'] + '_' + df_data['PRODUCT_LINE_CODE'].astype(str)
df_data['DAY_OF_WEEK'] = df_data['LAST_SOURCED_TS'].dt.dayofweek ## as int
df_data['TIME_OF_DAY'] = df_data['LAST_SOURCED_TS'].apply(lambda x:x.toordinal()) ## convert TS
#df_data['DAY_OF_WEEK'] = df_data['LAST_SOURCED_TS'].dt.day_name()
#df_data['TIME_OF_DAY'] = df_data['LAST_SOURCED_TS'].dt.time
df_data.drop(columns=['LINE', 'ITEM', 'PRODUCT_LINE_CODE'], axis=1, inplace=True)
df_data.head(7)
LAST_SOURCED_TS ARRIVAL_HOURS_TOP_LOC ORDERED Line_PLCD DAY_OF_WEEK TIME_OF_DAY
0 2020-04-02 11:15:44.735 22 1 WIX_1 3 737517
1 2020-04-08 13:26:00.547 18 1 WIX_1 2 737523
2 2020-04-08 14:45:34.767 16 1 WIX_1 2 737523
3 2020-04-10 16:27:27.780 16 1 WIX_1 4 737525
4 2020-04-10 17:50:42.921 68 0 WIX_1 4 737525
5 2020-04-14 12:35:28.880 17 0 WIX_1 1 737529
6 2020-04-14 18:10:59.017 15 1 WIX_1 1 737529
def predicted_values(line_plcd):
df_filtered = df_data[df_data['Line_PLCD'] == line_plcd]
# Pipeline gives the scaling needed by default
clf = SGDClassifier(loss='log', max_iter=1000, tol=1e-3)
pipe = make_pipeline(StandardScaler(), clf)
X = df_filtered.loc[:,['ARRIVAL_HOURS_TOP_LOC', 'DAY_OF_WEEK', 'TIME_OF_DAY']]
y = df_filtered.loc[:, ['ORDERED']].values.ravel()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.35, random_state=0)
print("X_train shape is : ", X_train.shape)
print("X_test shape is : ", X_test.shape)
print("y_train shape is : ", y_train.shape)
print("y_test shape is : ", y_test.shape)
pipe.fit(X_train, y_train)
accuracy_score = pipe.score(X_test, y_test)
#Predicted probability
est_arrival_hours = np.arange(0, 72, 0.5).reshape((-1, 1))
ordered_probability = pipe.predict_proba(est_arrival_hours)[:,1]
predicted = pd.Series(ordered_probability, index=est_arrival_hours.ravel())
return (accuracy_score, predicted)
top_line_plcd = ['FEL_0','WIX_0','STD_9','MOO_0','MRY_1','MPC_2','PTQ_12','BHH_24','WIX_1','GAT_3',
'MTM_0','MSA_0','BBR_8','DOR_15','NAT_0','DOR_6','STD_1','GAT_0','TYC_5','PFP_0',
'IPC_2','PRE_7','IDI_6','MHT_1','IPC_4','DOR_18']
print("All lines: ", df_data.index.size)
for line_plcd in top_line_plcd:
df_observed_counts = get_observed_counts(line_plcd)
accuracy_score, predicted = predicted_values(line_plcd)
print("Accuracy: " +line_plcd + ' ' +accuracy_score.astype(str))
output
All lines: 19094585
X_train shape is : (624545, 3)
X_test shape is : (336294, 3)
y_train shape is : (624545,)
y_test shape is : (336294,)
error msg
ValueError Traceback (most recent call last)
<ipython-input-23-e133ba3a1a20> in <module>
14 df_observed_counts = get_observed_counts(line_plcd)
15
---> 16 accuracy_score, predicted = predicted_values(line_plcd)
17 print("Accuracy: " +line_plcd + ' ' +accuracy_score.astype(str))
18
<ipython-input-22-792ae815c0a7> in predicted_values(line_plcd)
56 #Predicted probability
57 est_arrival_hours = np.arange(0, 72, 0.5).reshape((-1, 1))
---> 58 ordered_probability = pipe.predict_proba(est_arrival_hours)[:,1]
59 predicted = pd.Series(ordered_probability, index=est_arrival_hours.ravel())
60 return (accuracy_score, predicted)
~\Miniconda3\envs\sales-conversion\lib\site-packages\sklearn\utils\metaestimators.py in <lambda>(*args, **kwargs)
117
118 # lambda, but not partial, allows help() to work with update_wrapper
--> 119 out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)
120 # update the docstring of the returned function
121 update_wrapper(out, self.fn)
~\Miniconda3\envs\sales-conversion\lib\site-packages\sklearn\pipeline.py in predict_proba(self, X)
461 Xt = X
462 for _, name, transform in self._iter(with_final=False):
--> 463 Xt = transform.transform(Xt)
464 return self.steps[-1][-1].predict_proba(Xt)
465
~\Miniconda3\envs\sales-conversion\lib\site-packages\sklearn\preprocessing\_data.py in transform(self, X, copy)
789
790 copy = copy if copy is not None else self.copy
--> 791 X = self._validate_data(X, reset=False,
792 accept_sparse='csr', copy=copy,
793 estimator=self, dtype=FLOAT_DTYPES,
~\Miniconda3\envs\sales-conversion\lib\site-packages\sklearn\base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
434
435 if check_params.get('ensure_2d', True):
--> 436 self._check_n_features(X, reset=reset)
437
438 return out
~\Miniconda3\envs\sales-conversion\lib\site-packages\sklearn\base.py in _check_n_features(self, X, reset)
375 )
376 if n_features != self.n_features_in_:
--> 377 raise ValueError(
378 'X has {} features, but this {} is expecting {} features '
379 'as input.'.format(n_features, self.__class__.__name__,
ValueError: X has 1 features, but this StandardScaler is expecting 3 features as input.