I am currently working on a code that merges and sorts data using two dataframes. However, I have one excel that is manually written and one that is automatically written.
This creates issues due to when it comes to manually write info there's a big chance of it being off, which in my case results in me not being able to match the data.
However, I have come up with a solution to this by removing the last number of what I'm trying to merge. So that both the manual and automatic dataframes are slightly off and easier to match.
So, let's say we got this dataframe:
0 a b c d e
1 blala 1244 12676 2 12345328
2 blaa 1241 12686 3 12345334
3 blabla 1233 12690 4 12345345
And the dataframe:
0 a b c d e
1 blala 1244 12676 2 12345329
2 blaa 1241 12686 3 12345333
3 blabla 1233 12690 4 12345344
If I then try to merge on column "e" it doesn't work because they are not equal even though they are supposed to be. Due to the inaccuracy of manually treating info the only solution is to match them by making them both slightly inaccurate.
My expected output is following:
0 a b c d e
1 blala 1244 12676 2 1234532
2 blaa 1241 12686 3 1234533
3 blabla 1233 12690 4 1234534
And the dataframe:
0 a b c d e
1 blala 1244 12676 2 1234532
2 blaa 1241 12686 3 1234533
3 blabla 1233 12690 4 1234534
CURRENT CODE (don't mind unnecessary imports or comments)
import numpy as np
import os
from numpy import NaN, empty, int32, int64, percentile
import pandas as pd
from pandas.core.base import IndexOpsMixin
from pandas.core.frame import DataFrame
from tkinter import Tk # from tkinter import Tk for Python 3.x
from tkinter.filedialog import askopenfilename
import logging
import sys
import pathlib
from datetime import date, datetime
#tankcapacity = input("What is the fuel capacity of the tank: ")
Tk().withdraw()
filepathname1 = askopenfilename()
filepathname2 = askopenfilename()
print("You have chosen to mix", filepathname1, "and", filepathname2)
pd.set_option("display.max_rows", None, "display.max_columns", 10)
df1 = pd.read_excel(
filepathname1, "CWA107 Event", na_values=["NA"], skiprows=1, usecols="A, B, D, E, F"
)
df2 = pd.read_excel(
filepathname2,
na_values=["NA"],
skiprows=1,
usecols=["Tankad mängd diesel", "Unnamed: 3", "Mätarställning"],
)
# Drop NaN and x
df2.dropna(inplace=True)
# Drop NaN
df1.dropna(inplace=True)
#pd.to_numeric(df2['Mätarställning'])
for col in df2.columns:
print(col)
df1["Difference (%)"] = df1["Bränslenivå (%)"]
df1["Difference (%)"] = df1.loc[:, "Bränslenivå (%)"].diff()
# Renames time-column so that they match
df2.rename(columns={"Unnamed: 3": "Tid"}, inplace=True)
# Filters out the rows with a difference smaller than 2
df1filt = df1[(df1["Difference (%)"] >= 2)]
print(len(df1filt))
# Converts time-column to only year, month and date.
df1filt["Tid"] = pd.to_datetime(df1filt["Tid"]).dt.strftime("%Y%m%d").astype(str)
#print(df1filt)
df1filt.reset_index(level=0, inplace=True)
df1filt.rename(columns={"index": "row"}, inplace=True)
df1filt["Match"] = df1filt["row"]
df1filt["Match"] = df1filt.loc[:, "row"].diff()
df1filt['Match'].values[df1filt['Match'].values > 2] = 0
#print(df1filt)
#print(df1filt)
ROWRANGE = len(df1filt)+1
#input()
thevalue = 0
for currentrow in range(ROWRANGE-1):
#print(df1filt.loc[currentrow, 'row'])
#print(df1filt.loc[currentrow, 'Match'])
if df1filt.loc[currentrow, 'Match'] == 0.0:
df1filt.loc[currentrow-1,'Difference (%)'] = thevalue
df1filt.loc[currentrow-1,'Match'] = "SUMMED"
thevalue = df1filt.loc[currentrow, 'Difference (%)']
if df1filt.loc[currentrow, 'Match'] >= 1.0 and df1filt.loc[currentrow, 'Match'] <= 4:
thevalue += df1filt.loc[currentrow, 'Difference (%)']
#print(thevalue)
#print("----------")
print("LENGHT", (len(df1filt)))
df1filt2 = df1filt[(df1filt['Match'] != 1.0) & (df1filt['Difference (%)'] >= 5) & (df1filt['Match'] != 0.0)]
df1filt2["Refueled amount (l)"] = df1filt2["Difference (%)"]
df1filt2["Refueled amount (l)"] = df1filt2.loc[:, "Difference (%)"]/100 *500
#print(df1filt2)
df1filt2.rename(columns={"Vägmätare (km)": "Mätarställning"}, inplace=True)
df2filt = df2[(df2['Tankad mängd diesel'] != NaN)]
df2filt.drop(df2filt.index[0], inplace=True)
#df2filt.round({'Mätarställning': 0})
#for col in df1filt2.columns:
# print(col)
df2filt['Tid'] = '20' + df2filt['Tid'].astype(str)
#print(df2filt)
#print(df1filt2)
merged_df = df1filt2.merge(df2filt, on='Mätarställning')
print("-----------------------------------------------------")
#print(df2filt)
#print(df1filt2)
print(merged_df)
print("-----------------------------------------------------")
filepathname3 = askopenfilename()
merged_df.to_excel(filepathname3, index = False)
#print(df2filt)
print("Antal tankningar:", len(df1filt2))
input()