Python selenium code scrolls down successfully but id can`t print content of tweet

user_15363193 0
Viewed 39
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import time
import re
import os
from selenium.webdriver.common.keys import Keys 
import selenium.webdriver.support.ui as ui
driver = webdriver.Chrome('C:\SeleniumDrivers\chromedriver')
def find_hashtags(hashtag):
    driver.get('https://twitter.com/hashtag/' + hashtag + '?src=hash')
    body=driver.find_element_by_tag_name('body')
    wait = ui.WebDriverWait(driver,0.5)
    while True:
            try:
                    wait.until(EC.visibility_of_element_located((By.XPATH,"//span[contains(@class, 'Icon Icon--large Icon--logo')]")))
                    print("error")
                    break
            except:
                    body.send_keys(Keys.END)

    tweets=driver.find_elements_by_class_name('tweet-text')
    for tweet in tweets:
            print (tweet.text)
find_hashtags("any hashtag")

When i run it on chrome it scrolls sccessfully but it cant print tweets content so there arent any problems in webdriver.

selenium-webdriver selenium python
1 Answers

First, you locators are incorrect. Use CSS selectors.

Second, if you want text to be printed in a loop, put it there. And last: you need to wait for your elements to become visible.

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import time
import re
import os

from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import selenium.webdriver.support.ui as ui
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome(executable_path='/snap/bin/chromium.chromedriver')


def find_hashtags(hashtag):
    driver.get('https://twitter.com/hashtag/' + hashtag + '?src=hash')
    body = driver.find_element_by_tag_name('body')
    wait = ui.WebDriverWait(driver, 5)
    while True:
        try:
            wait.until(
                EC.visibility_of_element_located((By.XPATH, "//span[contains(@class, 'Icon Icon--large Icon--logo')]")))
            print("error")
            break
        except:
            body.send_keys(Keys.END)
            wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, ".css-901oao.r-18jsvk2.r-1qd0xha.r-a023e6.r-16dba41.r-rjixqe.r-bcqeeo.r-bnwqim.r-qvutc0>span:nth-of-type(1)")))
            tweets = driver.find_elements_by_css_selector('.css-901oao.r-18jsvk2.r-1qd0xha.r-a023e6.r-16dba41.r-rjixqe.r-bcqeeo.r-bnwqim.r-qvutc0>span:nth-of-type(1)')
            for tweet in tweets:
                print(tweet.text)


find_hashtags("any hashtag")

Also, note that 0.5 seconds wait is too small.

user_127301120
Related