Cannot make a csv file from html tables, error in find function

Viewed 30

I have a problem with the code for parsing HTML tables into a CSV file.

I need a code that take all the tables inside a URL and convert into a series of csv files

This is the code:

import csv
from bs4 import BeautifulSoup
import pandas as pd
from urllib.request import urlopen
import ssl
ssl._create_default_https_context = ssl._create_unverified_context


url = 'https://ofi5.mef.gob.pe/invierte/ejecucion/verFichaEjecucion/2459714'
html = urlopen(url).read()  

#html = open('test.html').read()
soup = BeautifulSoup(html, features='lxml')
#Specify table name which you want to read.
#Example: <table class="queryResults" border="0" cellspacing="1">
table = soup.select_one('table.queryResults')

def get_all_tables(soup):
    return soup.find_all("table")


tbls = get_all_tables(soup)
for i, tablen in enumerate(tbls, start=1):
    print(i)
    print(tablen)

def get_table_headers(table):
    headers = []
    for th in table.find("tr").find_all("th"):
        headers.append(th.text.strip())
    return headers

head = get_table_headers(table)
#print(head)

def get_table_rows(table):    
    rows = []
    for tr in table.find_all("tr")[1:]:
        cells = []
        # grab all td tags in this table row
        tds = tr.find_all("td")
        if len(tds) == 0:
            # if no td tags, search for th tags
            # can be found especially in wikipedia tables below the table
            ths = tr.find_all("th")
            for th in ths:
                cells.append(th.text.strip())
        else:
            # use regular td tags
            for td in tds:
                cells.append(td.text.strip())
        rows.append(cells)
    return rows

table_rows = get_table_rows(table)
#print(table_rows)

def save_as_csv(table_name, headers, rows):
    pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")

save_as_csv("Test_table", head, table_rows)

I have this error in the terminal:

    for th in table.find("tr").find_all("th"):
AttributeError: 'NoneType' object has no attribute 'find'

The code is a variation from this thread: Convert HTML into CSV Thanks to Ramineni Ravi Teja for that code.

0 Answers
Related