I'm scraping projector items on Suning.com When I count "li" tag on F12 page, there is over 60 items However, although scrolling down, find_all function only returns 30 How can I make my code working?
Here is the URL as below url : https://search.suning.com/%E6%8A%95%E5%BD%B1%E4%BB%AA%E5%AE%B6%E7%94%A8/&iy=0&isNoResult=0&cf=brand_Name_FacetAll:%E5%85%88%E7%A7%91(SAST)&isNoResult=0&cp=0
from selenium import webdriver
import re
import time
import requests
from bs4 import BeautifulSoup
import time
interval = 3
headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-encoding':'gzip, deflate, br',
'accept-language':'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7'}
interval = 2
brand_list = ['先科(SAST)']
page_num = 1
browser = webdriver.Chrome()
browser.maximize_window()
for brand in brand_list:
for page in range(page_num):
print("Current page : ", page+1)
url = "https://search.suning.com/%E6%8A%95%E5%BD%B1%E4%BB%AA%E5%AE%B6%E7%94%A8/&iy=0&isNoResult=0&cf=brand_Name_FacetAll:{}&isNoResult=0&cp={}".format(brand, page)
browser.get(url)
res = requests.get(url, headers=headers)
res.raise_for_status()
soup = BeautifulSoup(res.text, "lxml")
# if no result, break
if len(soup.find_all("div", attrs={"class":"no-find lack-select"})) != 0:
print("No result")
break
# scrolling down
prev_height = browser.execute_script("return document.body.scrollHeight")
while True:
# Scrolling down
browser.execute_script("window.scrollTo(0, document.body.scrollHeight)")
# Wait for loading
time.sleep(interval)
# Update scorll info
curr_height = browser.execute_script("return document.body.scrollHeight")
if curr_height == prev_height:
print("======================= Scrolling done =======================")
break
prev_height = curr_height
# Count "li" tag qty
items = soup.find_all("li", attrs={'class':re.compile('^item-wrap')})
print("li tag qty : ", len(items))
for item in items:
print(item['class'])