I have adopted this code from a medium post, currently it works for the first id but is not looping through the ranges:
import requests
from bs4 import BeautifulSoup
import json
import pandas as pd
understat_ids = range(16376, 16755+1)
match_data = []
for id in understat_ids:
base_urls = f'https://understat.com/match/{id}'
#Use requests to get the webpage and BeautifulSoup to parse the page
res = requests.get(base_urls)
soup = BeautifulSoup(res.content, 'lxml')
scripts = soup.find_all('script')
#get only the shotsData
strings = scripts[1].string
# strip unnecessary symbols and get only JSON data
ind_start = strings.index("('")+2
ind_end = strings.index("')")
json_data = strings[ind_start:ind_end]
json_data = json_data.encode('utf8').decode('unicode_escape')
#convert string to json format
data = json.loads(json_data)
x = []
y = []
xG = []
result = []
team = []
minute = []
data_away = data['a']
data_home = data['h']
date = []
player = []
situation = []
shotType = []
matchId = []
for index in range(len(data_home)):
for key in data_home[index]:
if key == 'X':
x.append(data_home[index][key])
if key == 'Y':
y.append(data_home[index][key])
if key == 'h_team':
team.append(data_home[index][key])
if key == 'xG':
xG.append(data_home[index][key])
if key == 'minute':
minute.append(data_home[index][key])
if key == 'result':
result.append(data_home[index][key])
if key == 'situation':
situation.append(data_home[index][key])
if key == 'date':
date.append(data_home[index][key])
if key == 'player':
player.append(data_home[index][key])
if key == 'shotType':
shotType.append(data_home[index][key])
if key == 'match_id':
matchId.append(data_home[index][key])
for index in range(len(data_away)):
for key in data_away[index]:
if key == 'X':
x.append(data_away[index][key])
if key == 'Y':
y.append(data_away[index][key])
if key == 'a_team':
team.append(data_away[index][key])
if key == 'xG':
xG.append(data_away[index][key])
if key == 'minute':
minute.append(data_away[index][key])
if key == 'result':
result.append(data_away[index][key])
if key == 'situation':
situation.append(data_away[index][key])
if key == 'date':
date.append(data_away[index][key])
if key == 'player':
player.append(data_away[index][key])
if key == 'shotType':
shotType.append(data_away[index][key])
if key == 'match_id':
matchId.append(data_away[index][key])
col_names = ['x','y','xG','minute','result','situation','date','player','shotType','team','match_id']
df = pd.DataFrame([x,y,xG,minute,result,situation,date,player,shotType,team,matchId],index=col_names)
df = df.T
The above code works for one match the first ID specified in range, how can I edit this code to loop through each match id appending them on to a pandas dataframe?
It seems like the code needs a small edit to be able to do this.