So I want to scrape the prices for games found under the class name 'newly-added-items__item__price'.
All I want is for Selenium to load up the content so I can then parse through and use beautifulsoup to get what I need.
However, even after the whole page is loaded, the javascript content does not load despite the fact I can view the code using inspect in the very same Chrome window.
I'm just doing a simple webscraping, but I can't figure out why it won't work.
Any thoughts?
#Import selenium
from selenium import webdriver
import time
from bs4 import BeautifulSoup
#import scrolling from selenium
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
#Get url for amiami.com
url = 'https://www.amiami.com/eng/c/videogames/'
#Setup options for chrome
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
options.add_experimental_option('detach', True)
options.add_argument('--disable-blink-features=AutomationControlled')
#Open browser for chrome
driver = webdriver.Chrome(chrome_options=options, executable_path=r"C:\Program Files (x86)\chromedriver.exe")
driver.get(url)
time.sleep(10)
WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.CLASS_NAME, "newly-added-items__item__price")))
#scroll down
last_height = driver.execute_script("return document.body.scrollHeight")
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(10)
page_source = driver.page_source
soup = BeautifulSoup(page_source, 'html.parser')
y = soup.prettify()
print(y)
print ('Done')