Still new('ish) to Selenium but I've scraped a number of pages and I'm super confused as to why I can't get this to work without timing out because it cant find the element.
I'm trying to just pull and save a large amount of Reddit source pages from Web.Archive.org to run separate script on later.
The few issues that have been giving me a headache are noted immediately below ##### ERROR in the script comments below:
import time
import os
import sys
import datetime
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = Options()
options.headless = False
driver = webdriver.Chrome(options=options, executable_path='~/Selenium/chromedriver')
driver.set_window_size(1000, 750)
quickTime = time.time()
print("--- %s seconds ---" % (time.time() - quickTime))
now = datetime.datetime.now()
timeStamp = now.strftime("%H:%M:%S")
dateStamp = now.strftime("%m:%d:%y")
print ("------------------------------")
print ("------------------------------")
print ("INITIALIZED AT:")
print (timeStamp)
driver.get('https://web.archive.org/web/20130307233745/https://www.reddit.com/r/all/')
with open('~/WayBackMachine/WBM-Pull_Log.md', mode='w') as z:
z.write("# ---------------------------------------")
z.write("\n")
z.write("# WayBack Machine Pull Log ---------")
z.write("\n")
z.write("# ---------------------------------------")
z.write("\n")
z.write("\n")
z.write("# Script Initialized At:")
z.write("\n")
z.write("[ ")
z.write(dateStamp)
z.write(" ]")
z.write(" ")
z.write("[ ")
z.write(timeStamp)
z.write(" ]")
z.write("\n")
z.write("\n")
z.write("# ------------------------------------------------------------------------------")
z.write("\n")
z.write("\n")
z.write("\n")
z.write("\n")
for i in range (5): # (This number will be very large in the end)
print ('| Page Run:', i)
##### ERROR - can not locate Page Capture Dates
pageDateMonth = WebDriverWait(driver,30).until(EC.presence_of_element_located(By.XPATH, '//*[@id="displayMonthEl"]'))
pageDateDay = WebDriverWait(driver,30).until(EC.presence_of_element_located(By.XPATH, '//*[@id="displayDayEl"]'))
pageDateYear = WebDriverWait(driver,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="displayYearEl"]'))
fullPageDate = (((pageDateMonth.text) + "-") + ((pageDateDay.text) + "-") + ((pageDateYear.text) + "_"))
fileSaveName = (fullPageDate + "WBM-SOURCE-rALL.html")
with open (
"~/WayBackMachine/WayBackMachine_Page_Sources/%s" % fileSaveName, "w"
) as w:
w.write(driver.page_source)
##### ERROR - can not locate button
nextCapture = WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH, "//img[@src='/_static/images/toolbar/wm_tb_nxt_on.png']")))
actions = ActionChains(driver)
actions.click(nextCapture).perform()
savedSourcePATH = ("~/WayBackMachine/WayBackMachine_Page_Sources/%s" % fileSaveName)
with open('~/WayBackMachine/WBM-Pull_Log.md', mode='w') as l:
l.write("___")
l.write("\n")
l.write("# |-| Page Run: %s" % i)
l.write("\n")
l.write("\n")
l.write(" ")
l.write("Page Source Date - [ %s ]" % fullPageDate)
l.write("\n")
l.write("\n")
l.write(" ")
l.write("Source Saved at Mark - > %s" % (time.time() - quickTime))
l.write("\n")
l.write("\n")
l.write(" ")
l.write("Page Source PATH = `%s`" % savedSourcePATH)
l.write("\n")
l.write("\n")
l.write("\n")
l.write("\n")
print ("| ^^^ [ PASS ] - Page Source Saved ", ("-- time: %s" % (time.time() - quickTime)))
print ("")
print ("---------------------------------------")
print ("---------------------------------------")
print ("---------------------------------------")
print (" ")
print ("Status:")
print ("SUCCESS")
print (" ")
print (" ")
print ("COMPLETED AT: ")
print (timeStamp)
print ("--------------------")
print ("Total Run Time:")
print("%s seconds" % (time.time() - quickTime))
print(" ")
print(" ")
print(" ")
driver.quit()
Every time I just keep getting either:
selenium.common.exceptions.TimeoutException: Message:or- (line 87)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":"[id="displayMonthEl"]"}
I can't for the life of me figure out why these can't find there elements
Can't find anyone with these issues from some Google-fu either so I'm just lost.
Any assistance would be appreciated.