BeautifulSoup returns multiple Div values

Question

Any help is appreciated!

soup1 = BeautifulSoup(updated_url, "lxml")
confStar = soup1.find_all('div', attrs={'data-testid': 'ConfidenceRating'})
print(confStar)

Above is the snippet of code I am having issues with. Every time I run my program it returns 15 divs when I only want one. I know it is running through each url item and returning all my values, but I cant figure out how I would go about only showing one pages results at a time.

import requests
import re
from bs4 import BeautifulSoup
import pprint


headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) '
                         'Chrome/39.0.2171.95 Safari/537.36'}
games = {}
endUrl = 0

# ---------------------------PickWise Game Grabber-------------------------------------------------#
page = requests.get("https://www.pickswise.com/nfl/", headers={'Cache-Control': 'no-cache'})
soup1 = BeautifulSoup(page.content, 'lxml')
box = soup1.find_all("div", class_="predictions_eventGrid__xDIPy")

game_urls = []
bb = []
info = {f"Game{endUrl}": {"Record": {"team1": "",
                                         "team2": ""},
                              "Projected W": "",
                              "Trust Factor": 0,
                              "+/-": 0,
                              }
            }

# ----- Grabs all URLs for the upcoming games and adds it to game_urls list.
for time in box:
    links = time.find('div', class_='EventGrid_eventContainer__39T3x')
    for urls in links:
        url = urls.get('href')
        game_urls.append(url)

# ------------Visits each game url and populated dict with info------------------------#
for i in game_urls:
    endUrl += 1
    # ----- Finds URLS and fills data from webpage ---- #
    updated_url = requests.get(f"https://www.pickswise.com{i}").text
    soup = BeautifulSoup(updated_url, "lxml")
    box = soup.find_all('div', class_="PredictionHeaderTeam_name__Pf64F")
    record = soup.find('table', class_="PredictionEnhancedInfo_predictionEnhancedInfo__35ubN")
    test = soup.find('div', class_="PredictionHeaderTeam_enhancedInfo__3lhgr")

    # ------------Gets the confidence rating for each team on PickWise-------------------
    soup1 = BeautifulSoup(updated_url, "lxml")
    confStar = soup1.find_all('div', attrs={'data-testid': 'ConfidenceRating'})
    print(confStar)

Below is the style of output I am looking for, yet it produces 15. (One for each NFL team)

[
, 
, ]

chitown88 · Answer

Few things:

probabably better seperate this updated_url = requests.get(f"https://www.pickswise.com{i}").text to make it little more readable code
no need to create 2 soup objects from the same html (as you do with soup and soup1
find_all() will return, as it states ALL those found elements. If you want just the first, us .find() or pull out the first element from that list that is returned from the .find_all().
Usually if I have a counter, similar to what you are doing with the endUrl count, I would just use enumerate() to handle that

Code:

import requests
import re
from bs4 import BeautifulSoup
import pprint


headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) '
                         'Chrome/39.0.2171.95 Safari/537.36'}


# ---------------------------PickWise Game Grabber-------------------------------------------------#
url = "https://www.pickswise.com/nfl/"

page = requests.get(url, headers=headers)
soup = BeautifulSoup(page.content, 'lxml')


# ----- Grabs all URLs for the upcoming games and adds it to game_urls list.
game_urls = []
games = soup.find_all('a', {'data-testid':'EventInfo'})
game_urls = [a['href'] for a in games]


# ------------Visits each game url and populated dict with info------------------------#
rows = []
for endUrl, link in enumerate(game_urls, start=1):
    # ----- Finds URLS and fills data from webpage ---- #
    updated_url = f"https://www.pickswise.com{link}"
    
    response = requests.get(updated_url)
    soup = BeautifulSoup(response.text, "lxml")
    
    # ------------Gets the confidence rating for each team on PickWise-------------------
    records = soup.find('td', text='Record')
    team_1_record = records.find_previous('td').text
    team_2_record = records.find_next('td').text
    
    spread = soup.find_all('div', {'class':re.compile('SelectionInfo_outcome.*')})[0].find(text=True)
    projectedW = spread
    
    confidence = soup.find('div', {'data-testid':'ConfidenceRating'})
    stars = confidence.find_all('span')
    starsFilled = len([x for x in stars if 'empty' not in ' '.join(x['class'])])
    
    over_under = soup.find_all('div', {'class':re.compile('SelectionInfo_outcome.*')})[1].find(text=True)
    
    data = {
        f"Game{endUrl}": {
            "Records": {
                "team1":team_1_record,
                "team2":team_2_record
                        },
            "Projected W":projectedW,
            "Trust Factor": starsFilled,
            "+/-": over_under
            }
        }
    
    rows.append(data)
    print(data)

Output:

Note I ran this after all Sunday games were done, so it'll only return the 2 remaing MNF games.

{'Game1': {'Records': {'team1': '0-1', 'team2': '1-0'}, 'Projected W': 'TEN Titans +10.0', 'Trust Factor': 1, '+/-': 'Under 48.0'}}
{'Game2': {'Records': {'team1': '1-0', 'team2': '1-0'}, 'Projected W': 'MIN Vikings +2.5', 'Trust Factor': 3, '+/-': 'Over 50.5'}}

BeautifulSoup returns multiple Div values

1 Answers