Python Selenium Message: no such element: Unable to locate element: {"method":"css selector","selector":".st"}

user_18730940 0
Viewed 57

I am new to web scraping and I am trying to access the text of search results on google. When I inspect for class names of the text I can get varying class names and the example I was following simply used "st". I have tried using different class names based on what I can find in inspecting the search results but it always throws the same error just with a different selector name.

Here is my code

def main():
    # Number of pages to scrape
    numPages = 5

    # All the scraped data
    infoAll = []

    # Scraped data from page 1
    infoAll.extend(scrape())

    for i in range(0 , numPages - 1):
        nextButton = driver.find_element(By.LINK_TEXT,'Next')
        nextButton.click()
        infoAll.extend(scrape())

    df = pd.DataFrame(infoAll)
    fileName = keyword + '_' + str(numPages) + '.csv'
    df.to_csv(filename)

#path to chromedriver.exe
s = Service('C:\Program Files (x86)\ChromeDriver\chromedriver.exe')

#creat instance of webdriver
driver = webdriver.Chrome(service = s)

#google url
url = 'https://www.google.com'

#Code to open a specific url
driver.get(url)

# set the keyword you want to search for
keyword = 'stocks' 

# we find the search bar using it's name attribute value
searchBar = driver.find_element(by=By.NAME, value='q')

# first we send our keyword to the search bar followed by the enter # key
searchBar.send_keys(keyword)
searchBar.send_keys('\n')

def scrape():
    pageInfo = []
    try:
        #wait for search results to be fetched
        WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "g")))
    except Exception as e:
        print(e)
        driver.quit()
    #contains the search results
    searchResults = driver.find_elements(By.CLASS_NAME, 'g')

    for result in searchResults:
        element = result.find_element(By.CSS_SELECTOR, 'a')
        link = element.get_attribute('href')
        header = result.find_element(By.CSS_SELECTOR, 'h3').text
        text = result.find_element(By.CLASS_NAME, 'st').text
        pageInfo.append({'header' : header, 'link' : link, 'text' : text})

    return pageInfo

if __name__ == "__main__":
    main()
selenium python
1 Answers

You just need proper xpaths to find spesific elements you are looking for. For example something like this could be used to get all header link texts:

elements = driver.find_elements(By.XPATH, '//*[contains(@class,"g tF2Cxc")]//div[@data-header-feature]/div/a[@href]')
for elem in elements:
    print(elem.get_attribute("href"))
user_181712710
Related