Merging computed file contents and display previous computed data in output

Question

I am working to 2 files, oldFile.txt and newFile.txt and compute some changes between them. The newFile.txt is updated constantly and any updates will be written to oldFile.txt

I am trying to improve the snippet below by saving previous computed values and add it to a finalOutput.txt. Any idea will be very helpful to accomplish the needed output. Thank you in advance.

import pandas as pd
from time import sleep
def read_file(fn):
    data = {}
    with open(fn, 'r') as f:
        for lines in f:
            line = lines.rstrip()
            pname, cnt, cat = line.split(maxsplit=2)
            data.update({pname: {'pname': pname, 'cnt': int(cnt), 'cat': cat}})
    return data

def process_data(oldfn, newfn):  
    old = read_file(oldfn)
    new = read_file(newfn)

    u_data = {}
    for ko, vo in old.items():
        if ko in new:
            n = new[ko]
            
            old_cnt = vo['cnt']
            new_cnt = n['cnt']
            u_cnt = old_cnt + new_cnt
            tmp_old_cnt = 1 if old_cnt == 0 else old_cnt
            cnt_change = 100 * (new_cnt - tmp_old_cnt) / tmp_old_cnt
            u_data.update({ko: {'pname': n['pname'], 'cnt': new_cnt, 'cat': n['cat'],
                                'curr_change%': round(cnt_change, 0)}})
    for kn, vn in new.items():
        if kn not in old:        
            old_cnt = 1
            new_cnt = vn['cnt']
            cnt_change = 0
            vn.update({'cnt_change': round(cnt_change, 0)})        
            u_data.update({kn: vn})

    pd.options.display.float_format = "{:,.0f}".format
    mydata = []
    
    for _, v in u_data.items():
        mydata.append(v)
        
    df = pd.DataFrame(mydata)
    df = df.sort_values(by=['cnt'], ascending=False)
    
    # Save to text file.
    with open('finalOutput.txt', 'w') as w:
        w.write(df.to_string(header=None, index=False))
        
    # Overwrite oldFile.txt
    with open('oldFile.txt', 'w') as w:
        w.write(df.to_string(header=None, index=False))

    # Print in console.
    df.insert(0, '#', range(1, 1 + len(df)))
    print(df.to_string(index=False,header=True))
    
while True:
    oldfn = './oldFile.txt'
    newfn = './newFile.txt'
    process_data(oldfn, newfn)
    sleep(60)

oldFile.txt

e6c76e4810a464bc 1                   Hello(HLL)
65b66cc4e81ac81d 2           CryptoCars (CCAR)
c42d0c924df124ce 3            GoldNugget (NGT)
ee70ad06df3d2657 4             BabySwap (BABY)
e5b7ebc589ea9ed8 8            Heroes&E... (HE)
7e7e9d75f5da2377 3                Robox (RBOX)

newfile.txt #-- content during 1st reading

e6c76e4810a464bc 34                  Hello(HLL)
65b66cc4e81ac81d 43           CryptoCars (CCAR)
c42d0c924df124ce 95            GoldNugget (NGT)
ee70ad06df3d2657 15             BabySwap (BABY)
e5b7ebc589ea9ed8 37            Heroes&E... (HE)
7e7e9d75f5da2377 23                Robox (RBOX)
755507d18913a944 49             CharliesFactory

newfile.txt #-- content during 2nd reading

924dfc924df1242d 35              AeroDie (ADie)
e6c76e4810a464bc 34                  Hello(HLL)
65b66cc4e81ac81d 73           CryptoCars (CCAR)
c42d0c924df124ce 15            GoldNugget (NGT)
ee70ad06df3d2657 5              BabySwap (BABY)
e5b7ebc589ea9ed8 12            Heroes&E... (HE)
7e7e9d75f5da2377 19                Robox (RBOX)
755507d18913a944 169            CharliesFactory

newfile.txt # content during 3rd reading

924dfc924df1242d 45             AeroDie (ADie)
e6c76e4810a464bc 2                  Hello(HLL)
65b66cc4e81ac81d 4           CryptoCars (CCAR)
c42d0c924df124ce 7            GoldNugget (NGT)
ee70ad06df3d2657 5             BabySwap (BABY)
e5b7ebc589ea9ed8 3            Heroes&E... (HE)
7e7e9d75f5da2377 6                Robox (RBOX)
755507d18913a944 9             CharliesFactory

oldFile.txt #-- Current output that needs improvement

#            pname  cnt               cat   curr_change%
 1 924dfc924df1242d   35    AeroDie (ADie)            29 
 2 755507d18913a944    9   CharliesFactory           -95
 3 c42d0c924df124ce    7  GoldNugget (NGT)           -53
 4 7e7e9d75f5da2377    6      Robox (RBOX)           -68
 5 ee70ad06df3d2657    5   BabySwap (BABY)             0
 6 65b66cc4e81ac81d    4 CryptoCars (CCAR)           -95
 7 e5b7ebc589ea9ed8    3  Heroes&E... (HE)           -75
 8 e6c76e4810a464bc    2        Hello(HLL)           -94

finalOutput.txt #-- Needed Improved Output with additional columns r1, r2 and so on depending on how many update readings

# curr_change% is the latest 3rd reading
# r2% is based on the 2nd reading
# r1% is based on the 1st reading

 #            pname  cnt               cat  curr_change%    r2%      r1%
 1 924dfc924df1242d   35    AeroDie (ADie)            29      0        0
 2 755507d18913a944    9   CharliesFactory           -95    245        0
 3 c42d0c924df124ce    7  GoldNugget (NGT)           -53    -84    3,067
 4 7e7e9d75f5da2377    6      Robox (RBOX)           -68    -17      667
 5 ee70ad06df3d2657    5   BabySwap (BABY)             0    -67      275
 6 65b66cc4e81ac81d    4 CryptoCars (CCAR)           -95     70    2,050
 7 e5b7ebc589ea9ed8    3  Heroes&E... (HE)           -75    -68      362
 8 e6c76e4810a464bc    2        Hello(HLL)           -94      0    3,300

user_6410450 · Answer

Updated for feedback, I made adjustments so that it would handle data that was fed to it live. Whenever new data is loaded, load the file name into process_new_file() function, and it will update the 'finalOutput.txt'.

For simplicity, I named the different files file1, file2, file3, and file4.

I'm doing most of the operations using the pandas Dataframe. I think working with Pandas DataFrames will make the task a lot easier for you.

Overall, I created one function to read the file and return a properly formatted DataFrame. I created a second function that compares the old and the new file and does the calculation you were looking for. I merge together the results of these calculations. Finally, I merge all of these calculations with the last file's data to get the output you're looking for.

import pandas as pd

global global_old_df
global results_df
global count

global_old_df = None
results_df = pd.DataFrame()
count = 0


def read_file(file_name):

    rows = []
    with open(file_name) as f:
      for line in f:
         rows.append(line.split(" ", 2))
    df = pd.DataFrame(rows, columns=['pname', 'cnt', 'cat'])
    df['cat'] = df['cat'].str.strip()
    df['cnt'] = df['cnt'].astype(float)

    return df


def compare_dfs(df_old, df_new, count):

    df_ = df_old.merge(df_new, on=['pname', 'cat'], how='outer')
    df_['r%s' % count] = (df_['cnt_y'] / df_['cnt_x'] - 1) * 100
    df_ = df_[['pname', 'r%s' % count]]
    df_ = df_.set_index('pname')

    return df_


def process_new_file(file):

    global global_old_df
    global results_df
    global count

    df_new = read_file(file)

    if global_old_df is None:
        global_old_df = df_new
        return

    else:
        count += 1
        r_df = compare_dfs(global_old_df, df_new, count)
        results_df = pd.concat([r_df, results_df], axis=1)
        global_old_df = df_new

        output_df = df_new.merge(results_df, left_on='pname', right_index=True)
        output_df.to_csv('finalOutput.txt')
        pd.options.display.float_format = "{:,.1f}".format
        print(output_df.to_string())




files = ['file1.txt', 'file2.txt', 'file3.txt', 'file4.txt']

for file in files:
    process_new_file(file)

This gives the output:

              pname  cnt                cat    r3    r2      r1
0  924dfc924df1242d 45.0     AeroDie (ADie)  28.6   NaN     NaN
1  e6c76e4810a464bc  2.0         Hello(HLL) -94.1   0.0 3,300.0
2  65b66cc4e81ac81d  4.0  CryptoCars (CCAR) -94.5  69.8 2,050.0
3  c42d0c924df124ce  7.0   GoldNugget (NGT) -53.3 -84.2 3,066.7
4  ee70ad06df3d2657  5.0    BabySwap (BABY)   0.0 -66.7   275.0
5  e5b7ebc589ea9ed8  3.0   Heroes&E... (HE) -75.0 -67.6   362.5
6  7e7e9d75f5da2377  6.0       Robox (RBOX) -68.4 -17.4   666.7
7  755507d18913a944  9.0    CharliesFactory -94.7 244.9     NaN

So, to run it live, you'd just replace that last section with:

while True:
    newfn = './newFile.txt'
    process_new_file(newfn)
    sleep(60)

Merging computed file contents and display previous computed data in output

1 Answers