How to scrape text from HTML to dataframe removing header and footer extra information?

You could create a list of dicts from header and values:

header = soup.select_one('pre:nth-of-type(2)').find_previous(text=True).split()[1:]
header[10:10] = ['x','y']

for l in soup.select_one('pre:nth-of-type(2)').text.splitlines():
    d = l.split()
    #d[10:13] = [' '.join([str(x) for x in d[10:13]])]
    # del d[10:12]
    data.append(dict(zip(header,d)))

Tricky part in my opinion is that you have to handle the the last elements in your list to avoid missmatch to headers.

Assuming "X Y ..." belong together:

d[10:13] = [' '.join([str(x) for x in d[10:13]])]

or if they are not needed simply delete them:

del d[10:12]

or adjust the headers instead:

header[10:10] = ['x','y']

Example

import requests
from bs4 import BeautifulSoup
import pandas as pd

URL = "https://www.globalcmt.org/cgi-bin/globalcmt-cgi-bin/CMT5/form?itype=ymd&yr=1976&mo=1&day=1&oyr=1976&omo=1&oday=1&jyr=1976&jday=1&ojyr=1976&ojday=1&otype=nd&nday=365&lmw=0&umw=10&lms=0&ums=10&lmb=0&umb=10&llat=-90&ulat=90&llon=-180&ulon=180&lhd=0&uhd=1000&lts=-9999&uts=9999&lpe1=0&upe1=90&lpe2=0&upe2=90&list=6"
r = requests.get(URL).text

page = requests.get(URL)
soup = BeautifulSoup(page.content, "html5lib")

data = []

header = soup.select_one('pre:nth-of-type(2)').find_previous(text=True).split()[1:]
header[10:10] = ['x','y']

for l in soup.select_one('pre:nth-of-type(2)').text.splitlines():
    d = l.split()
    #d[10:13] = [' '.join([str(x) for x in d[10:13]])]
    # del d[10:12]
    data.append(dict(zip(header,d)))

pd.DataFrame(data)

Output

...

Assuming the data part will always start with "Columns:"

match = re.finditer(r"Columns\:", text) #find the data part with regex
index_data = next(match).start(0) #take the position of the found regex regex

lines = [x.strip().split(" ") for x in text[len("Columns\:"):].split("\n")] #split the lines with line breaks, and split each line with space
columns = lines[0] #first line is the column names
values = lines[1:] # other are the values

print(columns)
print(values)

output:

['lon', 'lat', 'depth', 'mrr', 'mtt', 'mpp', 'mrt', 'mrp', 'mtp', 'iexp', 'name']
[['-176.96', '-29.25', '48', '7.68', '0.09', '-7.77', '1.39', '4.52', '-3.26', '26', 'X', 'Y', '010176A'], ['-75.14', '-13.42', '85', '-1.78', '-0.59', '2.37', '-1.28', '1.97', '-2.90', '24', 'X', 'Y', '010576A'], ['159.50', '51.45', '15', '1.10', '-0.30', '-0.80', '1.05', '1.24', '-0.56', '25', 'X', 'Y', '010676A']]

Hope this help !

You can use the split command to divide the text into sections. I also cleaned up the table to remove the "X Y".

import requests
from bs4 import BeautifulSoup
import pandas as pd
import io

URL = "https://www.globalcmt.org/cgi-bin/globalcmt-cgi-bin/CMT5/form?itype=ymd&yr=1976&mo=1&day=1&oyr=1976&omo=1&oday=1&jyr=1976&jday=1&ojyr=1976&ojday=1&otype=nd&nday=365&lmw=0&umw=10&lms=0&ums=10&lmb=0&umb=10&llat=-90&ulat=90&llon=-180&ulon=180&lhd=0&uhd=1000&lts=-9999&uts=9999&lpe1=0&upe1=90&lpe2=0&upe2=90&list=6"
r = requests.get(URL).text
page = requests.get(URL)
soup = BeautifulSoup(page.content, "html.parser")
text = soup.body.get_text(separator= '\n', strip=True)

split_top, split_main = text.split('\nColumns: ')
split_main, split_end = split_main.split('\nEnd of events found with given criteria.')

# remove the "X Y "
split_main = split_main.replace(' X Y ',' ')

df = pd.read_csv(io.StringIO(split_main),sep='\s+')
df