I am trying to download excel files from a website using selenium in headless mode. While it's working perfectly fine in most cases, there are a few cases(some months of an year) where the driver.find_element_by_xpath() fails to work like expected. I have been through many posts and though that the element might not have appeared when the driver was looking for it, but that isn't case as I thoroughly checked it and also tried to slow down the process using time.sleep(), on a side note I also use driver.implicitly_wait() to make things easier as the website actually takes a while to load content on the page. I couldn't use requests because it doesn't show any data in the response of get request. My script is as follows:
from selenium import webdriver
import datetime
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
import os
import shutil
import time
import calendar
currentdir = os.path.dirname(__file__)
Initial_path = 'whateveritis'
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_experimental_option("prefs", {
"download.default_directory": f"{Initial_path}",
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": True
})
def save_hist_data(year, months):
def waitUntilDownloadCompleted(maxTime=1200):
driver.execute_script("window.open()")
# switch to new tab
driver.switch_to.window(driver.window_handles[-1])
# navigate to chrome downloads
driver.get('chrome://downloads')
# define the endTime
endTime = time.time() + maxTime
while True:
try:
# get the download percentage
downloadPercentage = driver.execute_script(
"return document.querySelector('downloads-manager').shadowRoot.querySelector('#downloadsList downloads-item').shadowRoot.querySelector('#progress').value")
# check if downloadPercentage is 100 (otherwise the script will keep waiting)
if downloadPercentage == 100:
# exit the method once it's completed
return downloadPercentage
except:
pass
# wait for 1 second before checking the percentage next time
time.sleep(1)
# exit method if the download not completed with in MaxTime.
if time.time() > endTime:
break
starts_on = 1
for month in months:
no_month = datetime.datetime.strptime(month, "%b").month
no_of_days = calendar.monthrange(year, no_month)[1]
print(f"{no_of_days} days in {month}-{year}")
driver = webdriver.Chrome(executable_path="whereeveritexists", options=chrome_options)
driver.maximize_window() #For maximizing window
driver.implicitly_wait(20)
driver.get("https://www.iexindia.com/marketdata/areaprice.aspx")
select = Select(driver.find_element_by_name('ctl00$InnerContent$ddlPeriod'))
select.select_by_visible_text('-Select Range-')
driver.find_element_by_xpath("//input[@name='ctl00$InnerContent$calFromDate$txt_Date']").click()
select = Select(driver.find_element_by_xpath("//td[@class='scwHead']/select[@id='scwYears']"))
select.select_by_visible_text(str(year))
select = Select(driver.find_element_by_xpath("//td[@class='scwHead']/select[@id='scwMonths']"))
select.select_by_visible_text(month)
#PROBLEM IS WITH THIS BLOCK
test=None
while not test:
try:
driver.find_element_by_xpath(f"//td[@class='scwCells' and contains(text(),'{starts_on}')]").click()
test=True
except IndentationError:
print('Entered except block -IE')
driver.find_element_by_xpath(f"//td[@class='scwCellsWeekend' and contains(text(), '{starts_on}')]").click()
test=True
except:
print('Entered except block -IE-2')
driver.find_element_by_xpath(f"//td[@class='scwInputDate' and contains(text(), '{starts_on}')]").click()
test=True
driver.find_element_by_xpath("//input[@name='ctl00$InnerContent$calToDate$txt_Date']").click()
select = Select(driver.find_element_by_xpath("//td[@class='scwHead']/select[@id='scwYears']"))
select.select_by_visible_text(str(year))
select = Select(driver.find_element_by_xpath("//td[@class='scwHead']/select[@id='scwMonths']"))
select.select_by_visible_text(month)
#PROBLEM IS WITH THIS BLOCK
test=None
while not test:
try:
driver.find_element_by_xpath(f"//td[@class='scwCells' and contains(text(), '{no_of_days}')]").click()
# time.sleep(4)
test=True
except IndentationError:
print('Entered except block -IE')
driver.find_element_by_xpath(f"//td[@class='scwCellsWeekend' and contains(text(), '{no_of_days}')]").click()
# time.sleep(4)
test=True
except:
# time.sleep(2)
driver.find_element_by_xpath(f"//td[@class='scwInputDate' and contains(text(), '{no_of_days}')]").click()
test=True
driver.find_element_by_xpath("//input[@name='ctl00$InnerContent$btnUpdateReport']").click()
driver.find_element_by_xpath("//a[@title='Export drop down menu']").click()
print("Right before excel button click")
driver.find_element_by_xpath("//a[@title='Excel']").click()
waitUntilDownloadCompleted(180)
print("After the download potentially!")
filename = max([Initial_path + f for f in os.listdir(Initial_path)],key=os.path.getctime)
shutil.move(filename,os.path.join(Initial_path,f"{month}{year}.xlsx"))
driver.quit()
def main():
# years = list(range(2013,2015))
# months = ['Jan', 'Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
# for year in years:
# try:
save_hist_data(2018, ['Mar'])
# except:
# pass
if __name__== '__main__':
main()
The while loops are basically being used to select the date element on the calendar(month and year are already being selected from the drop downs). Because the website has different tags if the date falls on weekday or weekend, I used try and except blocks to try all possible xpaths but the weird thing is, some months of an year simply don't work like expected. This is the link btw "https://www.iexindia.com/marketdata/areaprice.aspx". Especially, in the case of Mar-2018, searching for xpaths on the chrome browser manually works and it locates 31st of Mar-2018, but when the python script is being executed it throws and error saying selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//td[@class='scwInputDate' and contains(text(), '31')]"} (Session info: headless chrome=84.0.4147.105)