Is it not possible to use Selenium (Python) in web.archive "contained" webpages or am I doing something minor wrong?

Viewed 232

Still new('ish) to Selenium but I've scraped a number of pages and I'm super confused as to why I can't get this to work without timing out because it cant find the element.

I'm trying to just pull and save a large amount of Reddit source pages from Web.Archive.org to run separate script on later.

The few issues that have been giving me a headache are noted immediately below ##### ERROR in the script comments below:

import time
import os
import sys
import datetime
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

options = Options()
options.headless = False

driver = webdriver.Chrome(options=options, executable_path='~/Selenium/chromedriver')
driver.set_window_size(1000, 750)




quickTime = time.time()
print("--- %s seconds ---" % (time.time() - quickTime))

now = datetime.datetime.now()
timeStamp = now.strftime("%H:%M:%S")
dateStamp = now.strftime("%m:%d:%y")





print ("------------------------------")
print ("------------------------------")
print ("INITIALIZED AT:")
print (timeStamp)

driver.get('https://web.archive.org/web/20130307233745/https://www.reddit.com/r/all/')

with open('~/WayBackMachine/WBM-Pull_Log.md', mode='w') as z:
    z.write("# ---------------------------------------")
    z.write("\n")
    z.write("# WayBack Machine Pull Log      ---------")
    z.write("\n")
    z.write("# ---------------------------------------")
    z.write("\n")
    z.write("\n")
    z.write("# Script Initialized At:")
    z.write("\n")
    z.write("[ ")
    z.write(dateStamp)
    z.write(" ]")
    z.write(" ")
    z.write("[ ")
    z.write(timeStamp)
    z.write(" ]")
    z.write("\n")
    z.write("\n")
    z.write("# ------------------------------------------------------------------------------")
    z.write("\n")
    z.write("\n")
    z.write("\n")
    z.write("\n")







for i in range (5): # (This number will be very large in the end)

    print ('| Page Run:', i)

    ##### ERROR - can not locate Page Capture Dates
    pageDateMonth = WebDriverWait(driver,30).until(EC.presence_of_element_located(By.XPATH, '//*[@id="displayMonthEl"]'))
    pageDateDay = WebDriverWait(driver,30).until(EC.presence_of_element_located(By.XPATH, '//*[@id="displayDayEl"]'))
    pageDateYear = WebDriverWait(driver,30).until(EC.presence_of_element_located((By.XPATH, '//*[@id="displayYearEl"]'))

    fullPageDate = (((pageDateMonth.text) + "-") + ((pageDateDay.text) + "-") + ((pageDateYear.text) + "_"))

    fileSaveName = (fullPageDate + "WBM-SOURCE-rALL.html")



    with open (
        "~/WayBackMachine/WayBackMachine_Page_Sources/%s" % fileSaveName, "w"
        ) as w:
        w.write(driver.page_source)

    ##### ERROR - can not locate button
    nextCapture = WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH, "//img[@src='/_static/images/toolbar/wm_tb_nxt_on.png']")))
    actions = ActionChains(driver)
    actions.click(nextCapture).perform()



    savedSourcePATH = ("~/WayBackMachine/WayBackMachine_Page_Sources/%s" % fileSaveName)

    with open('~/WayBackMachine/WBM-Pull_Log.md', mode='w') as l:
        l.write("___")
        l.write("\n")
        l.write("# |-| Page Run: %s" % i)
        l.write("\n")
        l.write("\n")
        l.write("    ")
        l.write("Page Source Date - [ %s ]" % fullPageDate)
        l.write("\n")
        l.write("\n")
        l.write("    ")
        l.write("Source Saved at Mark - > %s" % (time.time() - quickTime))
        l.write("\n")
        l.write("\n")
        l.write("    ")
        l.write("Page Source PATH = `%s`" % savedSourcePATH)
        l.write("\n")
        l.write("\n")
        l.write("\n")
        l.write("\n")


    print ("| ^^^ [ PASS ] - Page Source Saved ", ("-- time: %s" % (time.time() - quickTime)))
    print ("")



print ("---------------------------------------")
print ("---------------------------------------")
print ("---------------------------------------")
print (" ")
print ("Status:")
print ("SUCCESS")
print (" ")
print (" ")
print ("COMPLETED AT: ")
print (timeStamp)
print ("--------------------")
print ("Total Run Time:")
print("%s seconds" % (time.time() - quickTime))
print(" ")
print(" ")
print(" ")


driver.quit()

Every time I just keep getting either:

  • selenium.common.exceptions.TimeoutException: Message: or
  • (line 87) selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":"[id="displayMonthEl"]"}

I can't for the life of me figure out why these can't find there elements

Can't find anyone with these issues from some Google-fu either so I'm just lost.

Any assistance would be appreciated.

*Side Note: There are probably a number of other issue with my code but this is the first hurdle I've run into

0 Answers
Related