I have a problem with the code for parsing HTML tables into a CSV file.
I need a code that take all the tables inside a URL and convert into a series of csv files
This is the code:
import csv
from bs4 import BeautifulSoup
import pandas as pd
from urllib.request import urlopen
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
url = 'https://ofi5.mef.gob.pe/invierte/ejecucion/verFichaEjecucion/2459714'
html = urlopen(url).read()
#html = open('test.html').read()
soup = BeautifulSoup(html, features='lxml')
#Specify table name which you want to read.
#Example: <table class="queryResults" border="0" cellspacing="1">
table = soup.select_one('table.queryResults')
def get_all_tables(soup):
return soup.find_all("table")
tbls = get_all_tables(soup)
for i, tablen in enumerate(tbls, start=1):
print(i)
print(tablen)
def get_table_headers(table):
headers = []
for th in table.find("tr").find_all("th"):
headers.append(th.text.strip())
return headers
head = get_table_headers(table)
#print(head)
def get_table_rows(table):
rows = []
for tr in table.find_all("tr")[1:]:
cells = []
# grab all td tags in this table row
tds = tr.find_all("td")
if len(tds) == 0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths = tr.find_all("th")
for th in ths:
cells.append(th.text.strip())
else:
# use regular td tags
for td in tds:
cells.append(td.text.strip())
rows.append(cells)
return rows
table_rows = get_table_rows(table)
#print(table_rows)
def save_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")
save_as_csv("Test_table", head, table_rows)
I have this error in the terminal:
for th in table.find("tr").find_all("th"):
AttributeError: 'NoneType' object has no attribute 'find'
The code is a variation from this thread: Convert HTML into CSV Thanks to Ramineni Ravi Teja for that code.