I want to get a list with all video_ids from a certain youtube channel. This channel uploaded 1230 videos and 17 livestreams (23/09/2022). I know the youtube api is restricted to only 500 results. So I wanted to use the publishedBefore-parameter to get more than these 500 results.
When I use my code, I only get 551 results. Do I have a mistake in my code? I can't see a systemtic mistake in which videos are not included.
My Code:
import pandas as pd
import requests
import datetime
api_key = '...'
channel_id = 'UCXDi1F7Q-cJQ4mGGavtfYRQ'
# build dataframe
df = pd.DataFrame(columns=['channel_id',
'video_id',
'video_title'
'published_date',
'type'])
# first request
url = 'https://youtube.googleapis.com/youtube/v3/search?part=snippet&channelId=' + channel_id + '&maxResults=50&order=date&type=video&key=' + api_key
response = requests.get(url=url).json()
total_results = response['pageInfo']['totalResults']
# If there are more than 50 results, the response gives back a nextPageToken to get the other results, so we need to save this Token
if 'nextPageToken' not in response:
page_token = []
elif 'nextPageToken' in response:
page_token = response['nextPageToken']
# save the channel_id and video_id in a dataframe
for i in response['items']:
channel_id = i['snippet']['channelId']
video_id = i['id']['videoId']
published_date = i['snippet']['publishedAt']
video_title = i['snippet']['title']
vid_type = i['id']['kind']
df = pd.concat([df, pd.DataFrame([{
'channel_id': channel_id,
'video_id': video_id,
'video_title': video_title,
'published_date': published_date,
'type': vid_type
}])], ignore_index=True)
# CHANNELS WITH <50 VIDEOS
if total_results < 500 and page_token == []:
df.to_csv('C:\\Users\\...\\data\\video_ids_' + datetime.datetime.now().strftime('%Y-%m-%d') + '.csv')
# CHANNELS WITH >50 AND <500 VIDEOS
elif total_results < 500 and page_token != []:
# with the while loop we get maximum 500 results
while page_token != []:
# make the second call for the next results with the nextPageToken
url = 'https://youtube.googleapis.com/youtube/v3/search?part=snippet&channelId=' + channel_id + '&maxResults=50&order=date&type=video&pageToken=' + page_token + '&key=' + api_key
response = requests.get(url=url).json()
# If there are more than 50 results, the response gives back a nextPageToken to get the other results, so we need to save this Token
if 'nextPageToken' not in response:
page_token = []
elif 'nextPageToken' in response:
page_token = response['nextPageToken']
# save the channel_id and video_id in a dataframe
for i in response['items']:
channel_id = i['snippet']['channelId']
video_id = i['id']['videoId']
published_date = i['snippet']['publishedAt']
video_title = i['snippet']['title']
vid_type = i['id']['kind']
df = pd.concat([df, pd.DataFrame([{
'channel_id': channel_id,
'video_id': video_id,
'video_title': video_title,
'published_date': published_date,
'type': vid_type
}])], ignore_index=True)
df.to_csv('C:\\Users\\...\\data\\video_ids_' + datetime.datetime.now().strftime('%Y-%m-%d') + '.csv')
# CHANNELS WITH >500 VIDEOS
# to get more than 500 results we need another loop - we will do this by using the published_date variable
else:
while page_token != []:
# make the second call for the next results with the nextPageToken
url = 'https://youtube.googleapis.com/youtube/v3/search?part=snippet&channelId=' + channel_id + '&maxResults=50&order=date&type=video&pageToken=' + page_token + '&key=' + api_key
response = requests.get(url=url).json()
# If there are more than 50 results, the response gives back a nextPageToken to get the other results, so we need to save this Token
if 'nextPageToken' not in response:
page_token = []
elif 'nextPageToken' in response:
page_token = response['nextPageToken']
# save the channel_id and video_id in a dataframe
for i in response['items']:
channel_id = i['snippet']['channelId']
video_id = i['id']['videoId']
published_date = i['snippet']['publishedAt']
video_title = i['snippet']['title']
vid_type = i['id']['kind']
df = pd.concat([df, pd.DataFrame([{
'channel_id': channel_id,
'video_id': video_id,
'video_title': video_title,
'published_date': published_date,
'type': vid_type
}])], ignore_index=True)
else:
url = 'https://youtube.googleapis.com/youtube/v3/search?part=snippet&channelId=' + channel_id + '&maxResults=50&order=date&type=video&publishedBefore=' + published_date + '&key=' + api_key
response = requests.get(url=url).json()
# If there are more than 50 results, the response gives back a nextPageToken to get the other results, so we need to save this Token
if 'nextPageToken' not in response:
page_token = []
elif 'nextPageToken' in response:
page_token = response['nextPageToken']
# save the channel_id and video_id in a dataframe
for i in response['items']:
channel_id = i['snippet']['channelId']
video_id = i['id']['videoId']
published_date = i['snippet']['publishedAt']
video_title = i['snippet']['title']
vid_type = i['id']['kind']
df = pd.concat([df, pd.DataFrame([{
'channel_id': channel_id,
'video_id': video_id,
'video_title': video_title,
'published_date': published_date,
'type': vid_type
}])], ignore_index=True)
df.to_csv('C:\\Users\\...\\data\\video_ids_' + datetime.datetime.now().strftime('%Y-%m-%d') + '.csv')
Thank you in advance!
EDIT: I found no solution, but another approach to get the videos. Just use the publishedBefore-parameter. Now I got 1114 results but these are still not the full results...
# build dataframe
df = pd.DataFrame(columns=['channel_id',
'video_id',
'video_title'
'published_date',
'type'])
# first request
url = 'https://youtube.googleapis.com/youtube/v3/search?part=snippet&channelId=' + channel_id + '&maxResults=50&order=date&type=video&key=' + api_key
response = requests.get(url=url).json()
total_results = response['pageInfo']['totalResults']
# save the channel_id and video_id in a dataframe
for i in response['items']:
channel_id = i['snippet']['channelId']
video_id = i['id']['videoId']
published_date = i['snippet']['publishedAt']
video_title = i['snippet']['title']
vid_type = i['id']['kind']
df = pd.concat([df, pd.DataFrame([{
'channel_id': channel_id,
'video_id': video_id,
'video_title': video_title,
'published_date': published_date,
'type': vid_type
}])], ignore_index=True)
while df['video_id'][len(df)-1] != df['video_id'][len(df)-2]:
url = 'https://youtube.googleapis.com/youtube/v3/search?part=snippet&channelId=' + channel_id + '&maxResults=50&order=date&type=video&publishedBefore=' + published_date + '&key=' + api_key
response = requests.get(url=url).json()
total_results = response['pageInfo']['totalResults']
for i in response['items']:
channel_id = i['snippet']['channelId']
video_id = i['id']['videoId']
published_date = i['snippet']['publishedAt']
video_title = i['snippet']['title']
vid_type = i['id']['kind']
df = pd.concat([df, pd.DataFrame([{
'channel_id': channel_id,
'video_id': video_id,
'video_title': video_title,
'published_date': published_date,
'type': vid_type
}])], ignore_index=True)
# because the last row is a duplicate we need to delete the last row
df.drop(df.tail(1).index, inplace=True)
df.to_csv('C:\\Users\\...\\data\\video_ids_' + datetime.datetime.now().strftime('%Y-%m-%d') + '.csv')
If I try my methods with another channel which has only 28 videos uploaded everything seems to work.
Use this ID: channel_id:'UCWz3hMEDel0NGkR4jue2q1A'