AttributeError: 'NoneType' object has no attribute 'text' - BeautifulShop

user_8918431 0
Viewed 478

I have a little code for scraping info from fbref (link for data: https://fbref.com/en/comps/9/stats/Premier-League-Stats) and it worked well but now I have some problems with some features (I've checked that the fields which don't work now are"player","nationality","position","squad","age","birth_year"). I have also checked that the fields have the same name in the web that it used to be. Any ideas/help to solve the problem?

Many Thanks!


import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import re
import sys, getopt
import csv

def get_tables(url):
    res = requests.get(url)
    ## The next two lines get around the issue with comments breaking the parsing.
    comm = re.compile("<!--|-->")
    soup = BeautifulSoup(comm.sub("",res.text),'lxml')
    all_tables = soup.findAll("tbody")
    team_table = all_tables[0]
    player_table = all_tables[1]
    return player_table, team_table

def get_frame(features, player_table):
    pre_df_player = dict()
    features_wanted_player = features
    rows_player = player_table.find_all('tr')
    for row in rows_player:
        if(row.find('th',{"scope":"row"}) != None):
    
            for f in features_wanted_player:
                cell = row.find("td",{"data-stat": f})
                a = cell.text.strip().encode()
                text=a.decode("utf-8")
                if(text == ''):
                    text = '0'
                if((f!='player')&(f!='nationality')&(f!='position')&(f!='squad')&(f!='age')&(f!='birth_year')):
                    text = float(text.replace(',',''))
                if f in pre_df_player:
                    pre_df_player[f].append(text)
                else:
                    pre_df_player[f] = [text]
    df_player = pd.DataFrame.from_dict(pre_df_player)
    return df_player

stats = ["player","nationality","position","squad","age","birth_year","games","games_starts","minutes","goals","assists","pens_made","pens_att","cards_yellow","cards_red","goals_per90","assists_per90","goals_assists_per90","goals_pens_per90","goals_assists_pens_per90","xg","npxg","xa","xg_per90","xa_per90","xg_xa_per90","npxg_per90","npxg_xa_per90"]

def frame_for_category(category,top,end,features):
    url = (top + category + end)
    player_table, team_table = get_tables(url)
    df_player = get_frame(features, player_table)
    return df_player

top='https://fbref.com/en/comps/9/'
end='/Premier-League-Stats'
df1 = frame_for_category('stats',top,end,stats)

df1
nonetype web-scraping beautifulsoup python
2 Answers

I suggest loading the table with panda's read_html. There is a direct link to this table under Share & Export --> Embed this Table.

import pandas as pd
df = pd.read_html("https://widgets.sports-reference.com/wg.fcgi?css=1&site=fb&url=%2Fen%2Fcomps%2F9%2Fstats%2FPremier-League-Stats&div=div_stats_standard", header=1)

This outputs a list of dataframes, the table can be accessed as df[0]. Output df[0].head():

Rk Player Nation Pos Squad Age Born MP Starts Min 90s Gls Ast G-PK PK PKatt CrdY CrdR Gls.1 Ast.1 G+A G-PK.1 G+A-PK xG npxG xA npxG+xA xG.1 xA.1 xG+xA npxG.1 npxG+xA.1 Matches
0 1 Patrick van Aanholt nl NED DF Crystal Palace 30-190 1990 16 15 1324 14.7 0 1 0 0 0 1 0 0 0.07 0.07 0 0.07 1.2 1.2 0.8 2 0.08 0.05 0.13 0.08 0.13 Matches
1 2 Tammy Abraham eng ENG FW Chelsea 23-156 1997 20 12 1021 11.3 6 1 6 0 0 0 0 0.53 0.09 0.62 0.53 0.62 5.6 5.6 0.9 6.5 0.49 0.08 0.57 0.49 0.57 Matches
2 3 Che Adams eng ENG FW Southampton 24-237 1996 26 22 1985 22.1 5 4 5 0 0 1 0 0.23 0.18 0.41 0.23 0.41 5.5 5.5 4.3 9.9 0.25 0.2 0.45 0.25 0.45 Matches
3 4 Tosin Adarabioyo eng ENG DF Fulham 23-164 1997 23 23 2070 23 0 0 0 0 0 1 0 0 0 0 0 0 1 1 0.1 1.1 0.04 0.01 0.05 0.04 0.05 Matches
4 5 Adrián es ESP GK Liverpool 34-063 1987 3 3 270 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 Matches
user_113807950

If you're only after the player stats, change player_table = all_tables[1] to player_table = all_tables[2], because now you are feeding team table into get_frame function.

I tried it and it worked fine after that.

user_132075950
Related