Python Selenium store all data from loop in dataframe

Question

I'm new to python and I've created a web scraper, which scrapes all tweets from a specific company up to a specific time. The scraper should scrape not only the tweets from one company but for many, therefore I read in several company tickers from a csv file. That works fine and the scraper gets first the tweets from one company and then from the other.

with open('C:\Users\veron\Desktop\Test.csv') as csv_file:
    csv = csv.reader(csv_file)
    
    for row in csv:
        username = driver.find_element_by_xpath('//input[@placeholder = "Symbol or @Username"]')
        username.send_keys(row)

        time.sleep(2)

        username.send_keys(Keys.ENTER)

        time.sleep(2)

Then I wrote the code to get the tweets and each element (user, text, date) is stored in a list and the loop breaks when a specific date or the endof scrolling is reached.

        def get_tweet(element):
            try:
                user = element.find_element_by_xpath('.//span[@class = "st_2JY3sEE"]/a[contains(@href, "/")]/span[text()]').text
                text = element.find_element_by_xpath('.//div[@class="st_3SL2gug"]').text
                date = element.find_element_by_xpath('.//a[@class ="st_28bQfzV st_1E79qOs st_3TuKxmZ st_1VMMH6S"]').text
                #date = date_old.replace("
", "")
                tweet_data = [user, text, date]

            except:
                tweet_data = ['user', 'text', 'date']
            return tweet_data


        user_data = []
        text_data = []
        date_data = []
        scrolling = True

        while scrolling: 

            tweets = WebDriverWait(driver, 5).until(
                EC.presence_of_all_elements_located((By.XPATH, '//div[@class = "st_2o0zabc st_jGV698i st_PLa30pM"]')))
            #print(len(tweets))
            for tweet in tweets:  # you can change this number with the number of tweets in a website || NOTE: ONLY THOSE LOADED IN THE last page will be considered while those from previous page will be forgotten (example: scroll all the way down and then try to find an @username that it's on top --> it won't find it)
                tweet_list = get_tweet(tweet)
                user_data.append(tweet_list[0])
                text_data.append(" ".join(tweet_list[1].split()))
                date_data.append(tweet_list[2])

            # Get the initial scroll height
            last_height = driver.execute_script("return document.body.scrollHeight")
            str1 = "4/25/22"

            while True:
                # Scroll down to bottom
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

                # Wait to load page
                time.sleep(2)
                # Calculate new scroll height and compare it with last scroll height
                new_height = driver.execute_script("return document.body.scrollHeight")

                #check if the date substring from above is in the date list, thats the condition 1
                res = any(str1 in string for string in date_data)
                if res is True:
                    scrolling = False
                    break
                # condition 2
                if new_height == last_height:  # if the new and last height are equal, it means that there isn't any new page to load, so we stop scrolling
                    scrolling = False
                    break

                else:
                    last_height = new_height
                    break

At the end I want the data to be stored in a dataframe and then be extracted in a csv file.

driver.quit()
df_tweets = pd.DataFrame({'user': user_data, 'text': text_data, 'date': date_data})
df_tweets.to_csv('stocktweets.csv', index=False)

However, only the tweets from the last ticker from the csv file are stored in the dataframe and not the tweets from the previous ticker.

Does anyone has tips how to get all data stored in that dataframe?

Python Selenium store all data from loop in dataframe

0 Answers