extract data from multiple JSON files present in different folders and add to a dataframe

Viewed 26

I have a folder called data. Within this folder, there are three subfolders: 01, 02, 03. Within these folders are multiple JSON files from which I need to extract the data and add it to data frame. so far I tried to get it for one folder but is confused how to get files from 3 .

import os
def rename(directory):
os.chdir(directory)
num = 1
for file in [file for file in sorted(os.listdir(), key=os.path.getmtime, reverse=False) if os.path.splitext(file)[1] == ".json"]:
    if os.path.splitext(file)[1] == ".json":
        os.rename(file, f"file{num}.json")
        num += 1
    
    
 path = input("Enter path")
 rename(path)
 with open('file1.json', 'r') as f1:              
 data1 = json.load(f1) 
 time_list = []    
 with open("file2.json") as f2:
 data2 = json.load(f2)
 time_string = data2["stop"]
 time_list.append(time_string)

i desire to have 3 dataframes from the 3 folders present in data and then create a single graph combining all the dataframes. Currently worked on only 1...

    ta1 = [data1['price'], data3['price'], data5['price']....]
    df = pd.DataFrame({'price': ta1 , 'time': time_list})
   # Create a scatter chart object.
    chart1 = workbook.add_chart({'type': 'scatter','subtype': 'smooth_with_markers'})
  # Get the number of rows and column index
    max_row = len(df)
    col_x = df.columns.get_loc('time') + 1
    col_y = df.columns.get_loc('price') + 1
  # Create the scatter plot
    chart1.add_series({
      'name':       "price",
      'categories': [sheet_name, 1, col_x, max_row, col_x],
      'values':     [sheet_name, 1, col_y, max_row, col_y],
      'marker':     {'type': 'square', 'size': 6}
    })
    chart1.set_x_axis({'name': 'time'})
    chart1.set_y_axis({'name': 'price',
                  'major_gridlines': {'visible': False}})
1 Answers

Using scandir you can tell whether it's a file or a folder. Then iterate over all folders in the directory and execute your script for each folders path. e.g. os.listdir("./" + currentfolder)

Related