Python Youtube Web Scraper not working properly

user_12540222 0
Viewed 975

So i built this small script that would give back a URL of any searched video on youtube. But after opening it up again turns out that the web scraping with youtube is not working out properly. As when printing soup it returns something completely different than from what can be seen with inspect element on Youtube. Can someone help me solve this... Heres My Code:

import requests
from lxml import html
import webbrowser
from bs4 import BeautifulSoup
import time
import tkinter
from pytube import YouTube

headers= {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36"}

def video_finder():
    word = input("Enter video title: ")
    if ' ' in word:
        new = word.replace(' ', '+')
        print(new)
    else:
        pass

    vid = requests.get('https://www.youtube.com/results?search_query={}'.format(new))
    soup = BeautifulSoup(vid.text, features='lxml')
    all_vids = soup.find_all('div', id_='contents')
    print(all_vids)
    video1st = all_vids[0]
    a_Tag = video1st.find('a', class_="yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink spf-link", href=True)
    Video_name = a_Tag.text
    Video_id = a_Tag['href']
    video_link = 'https://www.youtube.com' + Video_id
    print(Video_name)
    print(video_link)

Its not the best but ye... thank you

web-scraping youtube python
1 Answers

To get correct result from Youtube page, set User-Agent HTTP header to Googlebot, and use html.parser in BeautifulSoup.

For example:

import requests
from bs4 import BeautifulSoup


headers= {"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
def video_finder():
    word = input("Enter video title: ")

    params = {
        'search_query': word
    }

    vid = requests.get('https://www.youtube.com/results', params=params, headers=headers)
    soup = BeautifulSoup(vid.content, features='html.parser')
    a_Tag = soup.find('a', class_="yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink spf-link", href=lambda h: h.startswith('/watch?'))
    Video_name = a_Tag.text
    Video_id = a_Tag['href']
    video_link = 'https://www.youtube.com' + Video_id
    print(Video_name)
    print(video_link)

video_finder()

Prints:

Enter video title: sailor moon
Sailor Moon Opening (English) *HD*
https://www.youtube.com/watch?v=5txHGxJRwtQ
user_100359850
Related