Formatting txt files using pandas/python

Try this:

import re
from pathlib import Path

input_path = Path("path/to/input_folder")
output_path = Path("path/to/output_folder")
run_name_pattern = re.compile("Run\d+")
data_line_pattern = re.compile("(.+?) +(.+?)")


def write_output(input_file: Path, run_name: str, data: str):
    output_file = output_path / f"{input_file.stem}-{run_name}.csv"
    with output_file.open("w") as fp_out:
        fp_out.write(data)


for input_file in input_path.glob("*.txt"):
    with input_file.open() as fp:
        run_name, data, start_reading = "", "", False

        for line in fp:
            # If a line matches "Run...", start a new run name
            if run_name_pattern.match(line):
                run_name = line.strip()
            # If the line matches "Charge (in Coulomb)...",
            # read in the data, starting with the next line
            elif line.startswith("Charge (in Coulomb) temporal evolution"):
                start_reading = True
            # For the data lines, replace spaces in the middle with a comma
            elif start_reading and line != "\n":
                data += data_line_pattern.sub(r"\1,\2", line)
            # If we encounter a blank line, that means the end of data.
            # Flush the data to disk.
            elif line == "\n":
                write_output(input_file, run_name, data)
                run_name, data, start_reading = "", "", False
        else:
            # If we have reached the end of the file but there still
            # data we haven't written to disk, flush it
            if data:
                write_output(input_file, run_name, data)

This works:

import csv
import os
import re

# Where the input
PATH_INPUT = "./test.txt"
# Define the output directory
DIR_OUTPUT = "./output"



def is_section_start(line):
    """Function to check to see if a line is the start of a section
    
    The start of a section is defined as starting with "Run"
    """
    return re.match("^Run", line)


def is_data_line(line):
    """Function to check to see if a line is a data line
    
    A data line is defined if a line starts with a number
    """
    return re.match("^\d", line)


def get_data(line):
    """Split data line into the two numbers"""
    split = line.split(" ")
    split = [s for s in split if s]
    return [float(split[0]), float(split[1])]


if __name__ == "__main__":
    # Open up the input file and read data into a dictionary where the key is the run name and the
    # value is a list of list of the numbers.
    output = {}
    with open(PATH_INPUT) as f_in:
        current_section = None
        for line in f_in.readlines():
            line = line.strip()
            if is_section_start(line) and current_section != line:
                current_section = line
                output[current_section] = []
            
            if is_data_line(line):
                output[current_section].append(get_data(line))
    
    # Write data
    for run, data in output.items():
        with open(os.path.join(DIR_OUTPUT, f"{PATH_INPUT}-{run}.txt"), "w") as f_out:
            writer = csv.writer(f_out)
            writer.writerows(data)

There are a lot of very complicated ways to read the data, but I wanted to present a simpler method:

with open('file.txt') as f:
    data = f.read().split('\n\n')

for run in data:
    run = run.split('\n')
    run_num = run[0]
    df = pd.DataFrame(run[6:])[0].str.split(expand=True).astype(float)
    df.columns = ['Charge (in Coulomb)', 'temporal evolution']
    print(run_num)
    print(df)

Output:

Run1
    Charge (in Coulomb)  temporal evolution
0                  3.61        2.440000e-11
1                  4.11        2.440000e-11
2                  4.61        2.440000e-11
3                  5.11        3.660000e-11
4                  5.63        3.660000e-11
5                  6.14        2.440000e-11
6                  6.66        3.660000e-11
7                  7.14        3.660000e-11
8                  7.67        2.440000e-11
9                  8.19        3.660000e-11
10                 8.70        2.440000e-11
11                 9.20        2.440000e-11
12                 9.72        2.440000e-11
13                10.23        2.440000e-11
14                10.73        2.440000e-11
Run2
   Charge (in Coulomb)  temporal evolution
0                 3.61        2.440000e-11
1                 4.11        2.440000e-11
2                 4.61        2.440000e-11
3                 5.11        3.660000e-11
4                 5.63        3.660000e-11
5                 6.14        2.440000e-11
6                 6.66        3.660000e-11
7                 7.14        3.660000e-11
8                 7.67        2.440000e-11
9                 8.19        3.660000e-11
Run3
    Charge (in Coulomb)  temporal evolution
0                  3.61        2.440000e-11
1                  4.11        2.440000e-11
2                  4.61        2.440000e-11
3                  5.11        3.660000e-11
4                  5.63        3.660000e-11
5                  6.14        2.440000e-11
6                  6.66        3.660000e-11
7                  7.14        3.660000e-11
8                  7.67        2.440000e-11
9                  8.19        3.660000e-11
10                 8.70        2.440000e-11
11                 9.20        2.440000e-11

This approach reads the text file into a single column within a dataframe where each row contains a line. From there each run number and run results are extracted.

import os
import pandas as pd

txt_file_path = r'D:\jchtempnew\SO\ResultFile.txt'
df = pd.read_fwf(txt_file_path, widths=[999999], header=None)

df_out = \
    df[0].str.extract('^(\d+.*?) (\d+.*)').astype(float) \
    .assign(Run=df[0].str.extract('^Run(\d+)').ffill()).dropna() \
    .rename(columns={0:'Charge (in Coulomb)',1:'temporal evolution'})

for run in df_out['Run'].unique():   
    df_out[df_out['Run']==run] \
        .to_csv(f'{os.path.splitext(txt_file_path)[0]}-Run{run}.csv', index=None)