Convert html table to csv in python using beautifulsoup

Viewed 158

I am trying to get table data from a website. The webpage displays some products, I extract the link to these products, and want to go to these one by one, get the table, and combine all information in one big table. The code below returns me information for only 1 product. Could someone help me with extending it to all products? Will be a great help, thanks.

import requests
import xlsxwriter
import csv
from bs4 import BeautifulSoup 
def cpap_spider(max_pages):
    global row_i
    page=1
    while page<=max_pages:
        url= "https://dmesupplyusa.com/bath-aids.html?p=" +str(page)
        source_code= requests.get(url)
        plain_text= source_code.text
        soup= BeautifulSoup(plain_text, 'html.parser')
        for link in soup.findAll("a", {"class":"product-item-link"}):
            href=link.get("href")
            title = link.string
            #worksheet.write(row_i, 0, title)
            each_item(href)
            print(href)
            #print(title)
        page+=1

def each_item(item_url):
    
    source_code= requests.get(item_url)
    plain_text= source_code.text
    soup= BeautifulSoup(plain_text, 'html.parser')

    table = soup.find("table", {"class":"data table additional-attributes"})
    rows = table.find_all("tr")

    with open("editors.csv", "wt+", newline="") as f:
        writer = csv.writer(f)
        for row in rows:
            csv_row = []
            for cell in row.findAll(["td", "th"]):
                csv_row.append(cell.get_text())
            writer.writerow(csv_row)
    
cpap_spider(1)
0 Answers
Related