presence_of_all_elements_located returns partial result

Question

Below is my code. I try to retrieve the comment owner's thumbnail. But I do not get link of the complete set. Few links values are shown as None. XPath point to correct location ,but unable to retrieve the value.

import time
import pandas as pd
import json
import mysql.connector as conn

import os
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.chrome.service import Service

options = Options()
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument("--headless")
options.add_argument("start-maximized")
options.add_argument('--window-size=1920,1080')
options.add_argument("--start-maximized")

caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "eager"

if __name__ == "__main__":
    DRIVER_PATH = 

    wd = webdriver.Chrome(executable_path=DRIVER_PATH , desired_capabilities=caps, options=options)

    url = "https://www.youtube.com/watch?v=8DkCAN1z-3U"

    wd.get(url)
    wd.maximize_window()
    wait = WebDriverWait(wd, 20)
    last_height = wd.execute_script("return document.documentElement.scrollHeight")
    while True:
        # Scroll down to bottom
        wd.execute_script("window.scrollTo(0, arguments[0]);", last_height)
        # Wait to load page
        time.sleep(1)

        # Calculate new scroll height and compare with last scroll height
        new_height = wd.execute_script("return document.documentElement.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height

    time.sleep(5)
    comm_thumbNail = wait.until(EC.presence_of_all_elements_located((By.XPATH,
                                                                     '//*[@id="contents"]/ytd-comment-thread-renderer//*[@id="author-thumbnail"]//*[@id="img"]')))
    i = 1
    for comm_img in comm_thumbNail:
        print(f"Comment {i}")
        print(comm_img.get_attribute("src"))
        i += 1

Please find the Element details which I have used to capture the thumbnail:

Could anyone please help me to understand what I am missing here ? Thanks.

walker · Answer

I'm not sure what get_attribute("src") looks for but to get a link you should use get_attribute("href"). Also if nothing else works, use BeautifulSoup to parse the links.

from bs4 import BeautifulSoup

source = driver.page_source
soup = BeautifulSoup(source, 'html.parser')
soup.find_all("tag", attrs={"type": "element"})

udpated

soup = BeautifulSoup(driver.page_source, 'html.parser')
text = soup.find_all("a", attrs={"class": "yt-simple-endpoint style-scope ytd-comment-renderer"})

presence_of_all_elements_located returns partial result

1 Answers