Removing the last number of a larger number (specific column)

Viewed 71

I am currently working on a code that merges and sorts data using two dataframes. However, I have one excel that is manually written and one that is automatically written.

This creates issues due to when it comes to manually write info there's a big chance of it being off, which in my case results in me not being able to match the data.

However, I have come up with a solution to this by removing the last number of what I'm trying to merge. So that both the manual and automatic dataframes are slightly off and easier to match.

So, let's say we got this dataframe:

0        a        b        c        d        e
1        blala    1244     12676    2        12345328
2        blaa     1241     12686    3        12345334
3        blabla   1233     12690    4        12345345

And the dataframe:

0        a        b        c        d        e
1        blala    1244     12676    2        12345329
2        blaa     1241     12686    3        12345333
3        blabla   1233     12690    4        12345344

If I then try to merge on column "e" it doesn't work because they are not equal even though they are supposed to be. Due to the inaccuracy of manually treating info the only solution is to match them by making them both slightly inaccurate.

My expected output is following:

0        a        b        c        d        e
1        blala    1244     12676    2        1234532
2        blaa     1241     12686    3        1234533
3        blabla   1233     12690    4        1234534

And the dataframe:

0        a        b        c        d        e
1        blala    1244     12676    2        1234532
2        blaa     1241     12686    3        1234533
3        blabla   1233     12690    4        1234534

CURRENT CODE (don't mind unnecessary imports or comments)

import numpy as np
import os
from numpy import NaN, empty, int32, int64, percentile
import pandas as pd
from pandas.core.base import IndexOpsMixin
from pandas.core.frame import DataFrame
from tkinter import Tk     # from tkinter import Tk for Python 3.x
from tkinter.filedialog import askopenfilename
import logging
import sys
import pathlib
from datetime import date, datetime

#tankcapacity = input("What is the fuel capacity of the tank: ")

Tk().withdraw()
filepathname1 = askopenfilename()
filepathname2 = askopenfilename()

print("You have chosen to mix", filepathname1, "and", filepathname2)
pd.set_option("display.max_rows", None, "display.max_columns", 10)

df1 = pd.read_excel(
    filepathname1, "CWA107 Event", na_values=["NA"], skiprows=1, usecols="A, B, D, E, F"
)
df2 = pd.read_excel(
    filepathname2,
    na_values=["NA"],
    skiprows=1,
    usecols=["Tankad mängd diesel", "Unnamed: 3", "Mätarställning"],
)

# Drop NaN and x
df2.dropna(inplace=True)

# Drop NaN
df1.dropna(inplace=True)

#pd.to_numeric(df2['Mätarställning'])

for col in df2.columns:
    print(col)

df1["Difference (%)"] = df1["Bränslenivå (%)"]
df1["Difference (%)"] = df1.loc[:, "Bränslenivå (%)"].diff()

# Renames time-column so that they match
df2.rename(columns={"Unnamed: 3": "Tid"}, inplace=True)



# Filters out the rows with a difference smaller than 2
df1filt = df1[(df1["Difference (%)"] >= 2)]
print(len(df1filt))

# Converts time-column to only year, month and date.
df1filt["Tid"] = pd.to_datetime(df1filt["Tid"]).dt.strftime("%Y%m%d").astype(str)

#print(df1filt)

df1filt.reset_index(level=0, inplace=True)

df1filt.rename(columns={"index": "row"}, inplace=True)


df1filt["Match"] = df1filt["row"]
df1filt["Match"] = df1filt.loc[:, "row"].diff()

df1filt['Match'].values[df1filt['Match'].values > 2] = 0


#print(df1filt)

#print(df1filt)

ROWRANGE = len(df1filt)+1

#input()

thevalue = 0
for currentrow in range(ROWRANGE-1):
    #print(df1filt.loc[currentrow, 'row'])
    #print(df1filt.loc[currentrow, 'Match'])
    if df1filt.loc[currentrow, 'Match'] == 0.0:
        df1filt.loc[currentrow-1,'Difference (%)'] = thevalue
        df1filt.loc[currentrow-1,'Match'] = "SUMMED"
        thevalue = df1filt.loc[currentrow, 'Difference (%)']
    if df1filt.loc[currentrow, 'Match'] >= 1.0 and df1filt.loc[currentrow, 'Match'] <= 4:
        thevalue += df1filt.loc[currentrow, 'Difference (%)']
    #print(thevalue)
    #print("----------")
print("LENGHT", (len(df1filt)))
df1filt2 = df1filt[(df1filt['Match'] != 1.0) & (df1filt['Difference (%)'] >= 5) & (df1filt['Match'] != 0.0)]

df1filt2["Refueled amount (l)"] = df1filt2["Difference (%)"]
df1filt2["Refueled amount (l)"] = df1filt2.loc[:, "Difference (%)"]/100 *500

#print(df1filt2)

df1filt2.rename(columns={"Vägmätare (km)": "Mätarställning"}, inplace=True)

df2filt = df2[(df2['Tankad mängd diesel'] != NaN)]

df2filt.drop(df2filt.index[0], inplace=True)

#df2filt.round({'Mätarställning': 0})

#for col in df1filt2.columns:
#    print(col)

df2filt['Tid'] = '20' + df2filt['Tid'].astype(str) 


#print(df2filt)
#print(df1filt2)

merged_df = df1filt2.merge(df2filt, on='Mätarställning')

print("-----------------------------------------------------")
#print(df2filt)
#print(df1filt2)
print(merged_df)
print("-----------------------------------------------------")

filepathname3 = askopenfilename()
merged_df.to_excel(filepathname3, index = False)


#print(df2filt)


print("Antal tankningar:", len(df1filt2))

input()
2 Answers

You can perform an integer division by 10:

df['e'] = df['e']//10

or shorter: df['e'] //= 10

output (example on df1):

   0       a     b      c  d       e
0  1   blala  1244  12676  2  123453
1  2    blaa  1241  12686  3  123453
2  3  blabla  1233  12690  4  123453

If you get a TypeError: import_optional_dependency() got an unexpected keyword argument 'errors' with the solution of mozway after updating pandas, one way to make it work is:

df["e"] = df["e"].astype(str).apply(lambda num: num[:-1]).astype(int)

or equivalently (but slightly slower):

df["e"] = df["e"].apply(lambda num: int(str(num)[:-1]))
Related