hey guys i want to scrape categories and subcatégories of this website https://www.materiel-velo.com/c110100-roues-velo-route.html using beautiful soup but all i get is the the first categories and subgategories thank you for helping me expecting output
[{'name': 'ROUES VÉLO ROUTE',
'url': 'https://www.materiel-velo.com/c110100-roues-velo-route.html',
'sub_categories': [{'name': 'Roue vélo carbone',
'url': 'https://www.materiel-velo.com/c110101-roue-velo-carbone.html'},
{'name': 'Roue polyvalente',
'url': 'https://www.materiel-velo.com/c110102-roue-velo-polyvalente.html'}]
}] etc..
this my code
import requests
from bs4 import BeautifulSoup as soup
SITE_URL = "https://www.materiel-velo.com/c110000-route.html"
HEADERS = {
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
'accept': 'application/json, text/javascript, */*; q=0.01',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://www.materiel-velo.com/',
'accept-language': 'en-US,en;q=0.9,fr;q=0.8',
'cookie': '_ga_5P0CXB07LK=GS1.1.1629112243.1.1.1629115778.0',
}
response = requests.get(SITE_URL, headers=HEADERS)
soupe = soup(response.content, 'html5lib')
categories = []
lvl1_1 = soupe.find_all(class_="subcategory")
lvl11 = []
for item in lvl1_1:
lvl1 = {
"name": item.text.strip(),
"url": item.a['href'],
"sub_categories": []
}
lvl11.append(lvl1)
lvl2 = item.find(class_="subsubcategories").find_all('a')
for elt in lvl2:
lvl2 = {
"name": elt.text.strip(),
"url": elt.a['href']
}
lvl1['sub_categories'].append(lvl2)
categories.append(lvl1)
print(categories)