Can't seem to find this element <div class="imageSpecInfo_product_img__pQD6e" on this page. I'm trying to download the images inside that div. But the program returns an empty list on print(nsImg)
The end goal is to save the images to a local folder and subsequently use them for machine learning.
Below is my code. Thanks in advance!
from bs4 import BeautifulSoup
import requests
import os
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.keys import Keys
import time
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
# number of pages you wanna crawl
page_num = 10
to_replace = "/",":","*","?","<",">","|" #specify characters that cannot be contained in a filename, for replacement
for i in range(page_num):
i += 1
# page Url
url = "https://search.shopping.naver.com/search/category/100000003?catId=50000002&origQuery&pagingIndex={0}".format(i)
# Crawling route
driver.get(url)
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(1)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
result = BeautifulSoup(driver.page_source, "html.parser")
pageName = result.find("div", {"class", "resultSummary_category_info__VRGme"}).get_text().strip().split('\n')
#create a main folder for the main page
cwd = os.getcwd()
files = os.listdir(cwd)
print(cwd)
for x in to_replace:
if x in pageName[0]:
pageName[0] = pageName[0].replace(x,'_')
if os.path.isdir(os.path.join(cwd, pageName[0])) == False:
os.mkdir(pageName[0])
print(pageName[0] + " page {0} folder created successfully!".format(i))
os.chdir(os.path.join(cwd, pageName[0]))
title = result.find_all("div", {"class", "basicList_title__VfX3c"})
for t in title:
folder = (t.a.text).strip()
for x in to_replace:
if x in folder:
folder = folder.replace(x,'_')
#each page url
url2 =t.a['href']
driver.get(url2)
time.sleep(3)
result2 = BeautifulSoup(driver.page_source, "html.parser")
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(3)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
# find all the images in the page
nsImg = result2.select('div[class*="imageSpecInfo_product_img__pQD6e"]')
#print(nsImg)
num = 1 #image_name
for i in nsImg:
src = requests.get(i['data-src'])
saveName = os.getcwd() + "//" + folder + str(num) + ".jpg"
with open(saveName, "wb") as file:
file.write(src.content)
num += 1
print(folder + " images saved completely!")
print('')
when I print nsImg it returns an empty list.