I have some code which is listed below. Something strange is happening, I put my rolling window on 2500seconds but the first values in my data frame do not give 'nan' for the first 2500seconds.And it is also not displaying nans at the end of my dataframe. I am puzzling for hours but I cant find out whats wrong. I am sorry for the long code but I do not know what is wrong.
this is how my df_00 looks like:
x_axis y_axis z_axis
datetime
2022-05-16 12:20:03.719 0.220 -0.010 -0.936
2022-05-16 12:20:28.719 0.209 -0.018 -0.927
2022-05-16 12:20:53.719 0.213 -0.026 -0.936
2022-05-16 12:21:18.719 0.224 -0.022 -0.944
df result example:
00_x_axis 00_y_axis ... 07_gradient 07_rolling_mean
datetime ...
2022-06-08 16:45:03.719 -0.035 -0.541 ... NaN NaN
2022-06-08 16:45:28.719 -0.043 -0.549 ... NaN NaN
2022-06-08 16:45:53.719 0.035 -0.549 ... 0.0160 0.016000
2022-06-08 16:46:18.719 -0.024 -0.584 ... 0.0160 0.016000
2022-06-08 16:46:43.719 -0.047 -0.584 ... 0.0000 0.010667
As you it only creates 1 nan in the rolling_mean columns (the first nan is normal)
this is my code:
A_start_01 = datetime.datetime(2022,6,8, 16,45,00)
A_end_01 = datetime.datetime(2022,6,8, 23,10,0)
S_start_01 = datetime.datetime(2022,6,9, 6,00,0)
S_end_01 = datetime.datetime(2022,6,11, 6,00,0)
D_start_01 = datetime.datetime(2022,6,11, 17,00,00)
D_end_01 = datetime.datetime(2022,6, 12, 21,0,0)
path = r'C:\Users\#\Pyth\data\flume'
all_files = glob.glob(path + "/*.csv")
li = []
for filename in all_files:
df = pd.read_csv(filename, index_col=(0), header=0, skiprows=(30), delimiter = ';', names = ['datetime', 'x_axis', 'y_axis', 'z_axis'], parse_dates=['datetime'])
li.append(df)
df_00 = li[0]
df_01 = li[1]
df_02 = li[2]
df_03 = li[3]
df_04 = li[4]
df_05 = li[5]
df_06 = li[6]
df_07 = li[7]
mask = (df_00.index > A_start_01) & (df_00.index <= D_end_01)
df_00 = df_00.loc[mask]
mask = (df_01.index > A_start_01) & (df_01.index <= D_end_01)
df_01 = df_01.loc[mask]
mask = (df_02.index > A_start_01) & (df_02.index <= D_end_01)
df_02 = df_02.loc[mask]
mask = (df_03.index > A_start_01) & (df_03.index <= D_end_01)
df_03 = df_03.loc[mask]
mask = (df_04.index > A_start_01) & (df_04.index <= D_end_01)
df_04 = df_04.loc[mask]
mask = (df_05.index > A_start_01) & (df_05.index <= D_end_01)
df_05 = df_05.loc[mask]
mask = (df_06.index > A_start_01) & (df_06.index <= D_end_01)
df_06 = df_06.loc[mask]
mask = (df_07.index > A_start_01) & (df_07.index <= D_end_01)
df_07 = df_07.loc[mask]
df_00.columns = ['00_x_axis', '00_y_axis', '00_z_axis']
df_01.columns = ['01_x_axis', '01_y_axis', '01_z_axis']
df_02.columns = ['02_x_axis', '02_y_axis', '02_z_axis']
df_03.columns = ['03_x_axis', '03_y_axis', '03_z_axis']
df_04.columns = ['04_x_axis', '04_y_axis', '04_z_axis']
df_05.columns = ['05_x_axis', '05_y_axis', '05_z_axis']
df_06.columns = ['06_x_axis', '06_y_axis', '06_z_axis']
df_07.columns = ['07_x_axis', '07_y_axis', '07_z_axis']
df = pd.merge_asof(df_00, df_01, left_index = True, right_index = True, tolerance=pd.Timedelta("30s"))
df = pd.merge_asof(df, df_02, left_index = True, right_index = True, tolerance=pd.Timedelta("30s"))
df = pd.merge_asof(df, df_03, left_index = True, right_index = True, tolerance=pd.Timedelta("30s"))
df = pd.merge_asof(df, df_04, left_index = True, right_index = True, tolerance=pd.Timedelta("30s"))
df = pd.merge_asof(df, df_05, left_index = True, right_index = True, tolerance=pd.Timedelta("30s"))
df = pd.merge_asof(df, df_06, left_index = True, right_index = True, tolerance=pd.Timedelta("30s"))
df = pd.merge_asof(df, df_07, left_index = True, right_index = True, tolerance=pd.Timedelta("30s"))
window = '2500s'
df['00_gradient']= np.abs(np.gradient(df['00_x_axis']))+np.abs(np.gradient(df['00_y_axis']))+np.abs(np.gradient(df['00_z_axis']))
df['00_rolling_mean'] = df['00_gradient'].rolling(window).mean()
df['01_gradient']= np.abs(np.gradient(df['01_x_axis']))+np.abs(np.gradient(df['01_y_axis']))+np.abs(np.gradient(df['01_z_axis']))
df['01_rolling_mean'] = df['01_gradient'].rolling(window).mean()
df['02_gradient']= np.abs(np.gradient(df['02_x_axis']))+np.abs(np.gradient(df['02_y_axis']))+np.abs(np.gradient(df['02_z_axis']))
df['02_rolling_mean'] = df['02_gradient'].rolling(window).mean()
df['03_gradient']= np.abs(np.gradient(df['03_x_axis']))+np.abs(np.gradient(df['03_y_axis']))+np.abs(np.gradient(df['03_z_axis']))
df['03_rolling_mean'] = df['03_gradient'].rolling(window).mean()
df['04_gradient']= np.abs(np.gradient(df['04_x_axis']))+np.abs(np.gradient(df['04_y_axis']))+np.abs(np.gradient(df['04_z_axis']))
df['04_rolling_mean'] = df['04_gradient'].rolling(window).mean()
df['05_gradient']= np.abs(np.gradient(df['05_x_axis']))+np.abs(np.gradient(df['05_y_axis']))+np.abs(np.gradient(df['05_z_axis']))
df['05_rolling_mean'] = df['05_gradient'].rolling(window).mean()
df['06_gradient']= np.abs(np.gradient(df['06_x_axis']))+np.abs(np.gradient(df['06_y_axis']))+np.abs(np.gradient(df['06_z_axis']))
df['06_rolling_mean'] = df['06_gradient'].rolling(window).mean()
df['07_gradient']= np.abs(np.gradient(df['07_x_axis']))+np.abs(np.gradient(df['07_y_axis']))+np.abs(np.gradient(df['07_z_axis']))
df['07_rolling_mean'] = df['07_gradient'].rolling(window).mean()
print(df)
EDIT
I have inserted the min_periods whit makes the first 2500s 'nan' but it should just work without adding this. And the function might not work properly because it might not do a good 2500s window....