I am trying to get table data from a website. The webpage displays some products, I extract the link to these products, and want to go to these one by one, get the table, and combine all information in one big table. The code below returns me information for only 1 product. Could someone help me with extending it to all products? Will be a great help, thanks.
import requests
import xlsxwriter
import csv
from bs4 import BeautifulSoup
def cpap_spider(max_pages):
global row_i
page=1
while page<=max_pages:
url= "https://dmesupplyusa.com/bath-aids.html?p=" +str(page)
source_code= requests.get(url)
plain_text= source_code.text
soup= BeautifulSoup(plain_text, 'html.parser')
for link in soup.findAll("a", {"class":"product-item-link"}):
href=link.get("href")
title = link.string
#worksheet.write(row_i, 0, title)
each_item(href)
print(href)
#print(title)
page+=1
def each_item(item_url):
source_code= requests.get(item_url)
plain_text= source_code.text
soup= BeautifulSoup(plain_text, 'html.parser')
table = soup.find("table", {"class":"data table additional-attributes"})
rows = table.find_all("tr")
with open("editors.csv", "wt+", newline="") as f:
writer = csv.writer(f)
for row in rows:
csv_row = []
for cell in row.findAll(["td", "th"]):
csv_row.append(cell.get_text())
writer.writerow(csv_row)
cpap_spider(1)