Selenium cannot click on Element even after page is loaded

Viewed 305

I'm trying to scrape some data off of a page which can only be reached after following a three-step process of clicking text to expose checkboxes, checking a box, and then clicking a button to take me to the next page, from which point I will scrape the data. I am using Python's Selenium package to make the three clicks and then using driver.page_source to send to BeautifulSoup to scrape the data.

The webpage is here: https://www.betonline.ag/sportsbook. On the left hand side, you have a list of sports which you can click to expose checkboxes. This is what am seemingly not able to do with Selenium. I cannot find in the HTML any element which I can click. If I manually complete this step, then the rest of the script works great. It's a little tricky to get the xpath to check the box, but using bs4 and this awesome function by ergoithz xpath_soup I can complete Step 2. Then the rest is easy.

Question How can I complete step one with Selenium to click "Baseball" or the "+" to expose the list of checkboxes?

Provided Below is

  1. Screenshot of the steps
  2. Full Code
  3. Stack trace if I try to skip step 1 (it doesn't allow me to interact with the checkbox if it's not exposed)

These screenshots layout the process more specifically:

Screenshots of Steps 1-3


import pandas as pd
import bs4
from selenium import webdriver

def xpath_soup(element):
     ##### Awesome function by ergoithz (removed docstring to save space)
    #https://gist.github.com/ergoithz/6cf043e3fdedd1b94fcf
    components = []
    child = element if element.name else element.parent
    for parent in child.parents:  # type: bs4.element.Tag
        siblings = parent.find_all(child.name, recursive=False)
        components.append(
            child.name if 1 == len(siblings) else '%s[%d]' % (
                child.name,
                next(i for i, s in enumerate(siblings, 1) if s is child)
                )
            )
        child = parent
    components.reverse()
    return '/%s' % '/'.join(components)


# Call up driver and 
driver = webdriver.Chrome()
driver.get('https://www.betonline.ag/sportsbook')
soup = bs4.BeautifulSoup(driver.page_source, 'html.parser')
divs = soup.find_all('div', class_='mainSportsLinks')

# Get 'south korea baseball' index (inelegant but it works)
table = divs[0].find_all('a')
sk_indices = []
for index, row in enumerate(table):
    try:
        #if bool(re.match(reg, row['cfg'])):
        if 'South Korea' in row['cfg']:
            sk_indices.append(index)
            print(index, row)
    except:
        pass

sk_index = sk_indices[0]

# use awesome xpath_soup function to create an xpath to find checkbox
# function by ergoithz @ https://gist.github.com/ergoithz/6cf043e3fdedd1b94fcf
bs4_tag = table[sk_index].parent.parent.find_all('div')[1].input
xpath_text = xpath_soup(bs4_tag)

### -----------------------------
### Missing Step 1 - how to click "Baseball" to expose the checkbox
### -----------------------------

# Step 2 - Click checkbox
button = driver.find_element_by_xpath(xpath_text)
#print(button.get_attribute("type"))
button.click()

# Step 3 - Click 'View Selected'
view_selected = driver.find_element_by_id('viewSelectedId')
view_selected.click()

# Pass to bs4 for scraping 
page_source = driver.page_source
soup_bets = bs4.BeautifulSoup(driver.page_source, 'html.parser')
driver.quit()

# Scrape using pandas read_html ...
#df_raw = pd.read_html(page_source, match='South Korea KBO')[0]
#df = df_raw.dropna(thresh=3).dropna(thresh=3, axis=1)
#df = df.loc[1:] # eliminate blank first column

Stack Trace if I try to "skip" Step 1. It doesn't allow me to interact with the checkbox unless I expose the list:

---------------------------------------------------------------------------
ElementNotInteractableException           Traceback (most recent call last)
<ipython-input-7-3ddcc15ccda8> in <module>
      5 button = driver.find_element_by_xpath(xpath_text)
      6 #print(button.get_attribute("type"))
----> 7 button.click()
      8 
      9 # Step 3 - Click 'View Selected'

C:\python38\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
     78     def click(self):
     79         """Clicks the element."""
---> 80         self._execute(Command.CLICK_ELEMENT)
     81 
     82     def submit(self):

C:\python38\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self, command, params)
    631             params = {}
    632         params['id'] = self._id
--> 633         return self._parent.execute(command, params)
    634 
    635     def find_element(self, by=By.ID, value=None):

C:\python38\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
    319         response = self.command_executor.execute(driver_command, params)
    320         if response:
--> 321             self.error_handler.check_response(response)
    322             response['value'] = self._unwrap_value(
    323                 response.get('value', None))

C:\python38\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
    240                 alert_text = value['alert'].get('text')
    241             raise exception_class(message, screen, stacktrace, alert_text)
--> 242         raise exception_class(message, screen, stacktrace)
    243 
    244     def _value_or_default(self, obj, key, default):

ElementNotInteractableException: Message: element not interactable
  (Session info: chrome=81.0.4044.129)
1 Answers

You can use below xPath to click the basket ball

driver.find_element_by_xpath('//a[@cfg="{type:\'h2h\',level1:\'Basketball\'}"]').click()
Related