How would I scroll container within website using Selnium in Python?

MOK_Z 0
Viewed 49

I would like to scroll a container within a webpage - not the webpage itself - using selenium within Python. Would anyone have any solutions to this?

I have tried the standard:

driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

Of course, this is fine for traversing the full web page, but not a container itself.

EDIT: The webpage is https://www.mixcloud.com/

The container shows when you use the search bar at the top of the page.

web-scraping selenium python
1 Answers

This is one way to scroll the page (it will stop once it goes over 300 songs):

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchShadowRootException, NoSuchElementException

import time as t
import pandas as pd


chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')

chrome_options.add_argument("window-size=1280,720")

webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)
actions = ActionChains(browser)
wait = WebDriverWait(browser, 20)


url = "https://www.mixcloud.com/"
browser.get(url)
s_box = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'input[name="mixcloud_query"]')))
s_box.click()
s_box.send_keys('house')
s_box.send_keys(Keys.ENTER)
music_div = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'body[class="rebrand"]')))
music_div.click()
while True:
    t.sleep(5)
    
    music_div.send_keys(Keys.END)
    print('scrolled to bottom')
    songs = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div[class^="SearchAudioCard__SearchAudioCardContainer"]')))
    print('songs:', len(songs))
    print('______________-')
    if len(songs) > 300:
        print('got over 300 songs, stopping')
        break
browser.quit()

Selenium documentation can be found at https://www.selenium.dev/documentation/

Barry the Platipus0
Related