Selenium Max Retries exceeded with url target machine refused connection

Question

I'm trying to get the ranking of all the players for each day using selenium. The problem is that my functions are calling the url multiple times which is causing an error. Is there a way to get the stuff I'm looking for without calling the site multiple times? The error is as follows:

raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=63292): Max retries exceeded with url: /session/34f9dbbc49369c1ab6634be1ca08820d/url (Caused by NewConnectionError(': Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))

The code is as shown below


import pandas as pd
import requests
import re
import datetime
import os
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
date_list = pd.date_range(start = "1971-02-01", end=datetime.date.today(), freq='1d')


PATH = "C:\Program Files (x86)\chromedriver"
driver = webdriver.Chrome(PATH)
def get_batsmen(date):
    url = f'https://www.icc-cricket.com/rankings/mens/player-rankings/odi/batting?at={date}'
    
    driver.get(url)
    players = [my_elem.text for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "td.table-body__cell.rankings-table__name.name a[href^='/rankings/mens/player-rankings']")))]
    
    player_list = []
    find_top = driver.find_element_by_class_name('rankings-block__banner--name-large')
    player_list.append(find_top.text)
    player_list.extend(players)
    df = pd.DataFrame(player_list, columns = ['Player Name'])
    driver.quit()
    return df

def get_bowler(date):
    url = f'https://www.icc-cricket.com/rankings/mens/player-rankings/odi/bowling?at={date}'
   
    driver.get(url)
    players = [my_elem.text for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "td.table-body__cell.rankings-table__name.name a[href^='/rankings/mens/player-rankings']")))]
    
    player_list = []
    find_top = driver.find_element_by_class_name('rankings-block__banner--name-large')
    player_list.append(find_top.text)
    player_list.extend(players)
    df = pd.DataFrame(player_list, columns = ['Player Name'])
    driver.quit()
    return df

def get_allrounder(date):
    url = f'https://www.icc-cricket.com/rankings/mens/player-rankings/odi/all-rounder?at={date}'
   
    driver.get(url)
    players = [my_elem.text for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "td.table-body__cell.rankings-table__name.name a[href^='/rankings/mens/player-rankings']")))]
    
    player_list = []
    find_top = driver.find_element_by_class_name('rankings-block__banner--name-large')
    player_list.append(find_top.text)
    player_list.extend(players)
    df = pd.DataFrame(player_list, columns = ['Player Name'])
    driver.quit()
    return df



for date in date_list:
    year = date.year
    month = date.month
    day = date.day
    newpath = rf'C:\Users\divya{year}'
    if not os.path.exists(newpath):
        os.makedirs(newpath)
    newpath1 = rf'C:\Users\divya{year}\{month}'
    if not os.path.exists(newpath1):
        os.makedirs(newpath1)
    newpath2 = rf'C:\Users\divya{year}\{month}\{day}'
    if not os.path.exists(newpath2):
        os.makedirs(newpath2)
    get_batsmen(date).to_csv(newpath2+'/batsmen.csv')
    get_bowler(date).to_csv(newpath2+'/bowler.csv')
    get_allrounder(date).to_csv(newpath2+'/allrounder.csv')

Selenium Max Retries exceeded with url target machine refused connection

0 Answers