How do I press load more button while scraping comments on Instagram with Selenium Python

Viewed 156

I'm working on a project that can scrape comments off posts on instagram and write them into an excel file.

Here's my code:

from selenium.webdriver.common.by import By
from selenium import webdriver
import time
import sys
import pandas as pd
from pandas import ExcelWriter
import os.path
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains


url = [
    "https://www.instagram.com/p/CcVTqRtJ2gj/",
    "https://www.instagram.com/p/CcXpLHepve-/",
]
user_names = []
user_comments = []
driver = driver = webdriver.Chrome("C:\chromedriver.exe")
driver.get(url[0])
time.sleep(3)

username = WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[name='username']")))
password = WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.CSS_SELECTOR,"input[name='password']")))
username.clear()
username.send_keys("username")
password.clear()
password.send_keys("pwd")
Login_button = (
    WebDriverWait(driver, 2)
    .until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[type='submit']")))
    .click()
)

time.sleep(4)
not_now = (
    WebDriverWait(driver, 30)
    .until(
        EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Not Now")]'))
    )
    .click()
)

for n in url:
    try:
        driver.get(n)
        time.sleep(3)
        load_more_comment = driver.find_element_by_xpath("//button[class='wpO6b  ']")
        print("Found {}".format(str(load_more_comment)))
        i = 0
        while load_more_comment.is_displayed() and i < 10:
            load_more_comment.click()
            time.sleep(1.5)
            load_more_comment = driver.find_element_by_xpath(
                "//button[class='wpO6b  ']"
            )
            print("Found {}".format(str(load_more_comment)))
            i += 1

        user_names.pop(0)
        user_comments.pop(0)

    except Exception as e:
        print(e)
        pass
    comment = driver.find_elements_by_class_name("gElp9 ")
    for c in comment:
        container = c.find_element_by_class_name("C4VMK")
        name = container.find_element_by_class_name("_6lAjh ").text
        content = container.find_element_by_class_name("MOdxS ").text
        content = content.replace("\n", " ").strip().rstrip()
        user_names.append(name)
        user_comments.append(content)
        print(content)
    user_names.pop(0)
    user_comments.pop(0)

    # export(user_names, user_comments)

driver.close()
df = pd.DataFrame(list(zip(user_names, user_comments)), columns=["Name", "Comments"])
# df.to_excel("Anime Content Engagement.xlsx")
print(df)

And the load-more-comments part, doesn't seem to work.

Since there are more than one buttons with the same class name, I"m not able to choose the right button to click on. And I'm a beginner so if there's anyone with any solution to how I can solve this it would be great.

1 Answers

you can select by aria-label text:

driver.find_element_by_css_selector("svg._8-yf5[aria-label='TEXT']")

i believe the text inside changes according to instagram language, put it according to what appears on your

Related