Create dataframe by scraping from different two or more <div>s etc

user_17865491 0
Viewed 73

I'm stuck on a little problem and hope you can help. I want to create a df by scraping from two parts of a web page. I seem to be stuck on the second part. My requirement is to get a df with each Horse name and the associated odds. eg. Horse Odds name1 odd1 name2 odd2

I've used a sample page in the script but it will be the same for any : base url https://www.racingtv.com/racecards/tomorrow : then select any time to get another page with the horse name and odds details etc.

import requests
import pandas as pd
from bs4 import BeautifulSoup


def main():
    # base url is https://www.racingtv.com/racecards/tomorrow
    # select any time to get the horse name and odds details etc.

    url = 'https://www.racingtv.com/racecards/catterick-bridge/372180-watch-racing-tv-now-novices-hurdle-gbb-race?'

    res = requests.get(url)
    soup = BeautifulSoup(res.content, "html.parser")
    strike = soup.select('div', class_='data-strike-out-group')

    # this bit seems to be working    
    for data in soup.find_all('div',
                              class_='racecard__runner__column racecard__runner__name'):
        for a in data.find_all('a'):
            print(a.text)

            # this bit sort of works but it seems to repeat the first three items of data
    for odds in soup.find_all('div',
                              class_='racecard__runner__column racecard__runner__column--price'):
        for odd1 in odds.find_all('ruk-odd'):
            print(odd1.text)

            # I tried this to work out how to stop getting the three duplicates but it does not work
    for odds in strike.select('div',
                              class_='racecard__runner__column racecard__runner__column--price'):
        for odd1 in odds.find_all('ruk-odd'):
            print(odd1.text)

    return


if __name__ == '__main__':
    main()
web-scraping python
2 Answers
class_='data-strike-out-group'

this isn't a class, check the raw html. It's an attribute of the div... weird

Glad you posted this, might end up using this site for a personal project. Figured you'd be interested in this code:

import requests
from bs4 import BeautifulSoup
import pandas as pd
from datetime import datetime

headers =   {
    'accept':'*/*',
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
    }

url = 'https://www.racingtv.com/racecards/catterick-bridge/372180-watch-racing-tv-now-novices-hurdle-gbb-race?'

resp = requests.get(url,headers=headers)
print(resp)

soup = BeautifulSoup(resp.text,'html.parser')
table = soup.find('div',{'class':'page__content__section racecard'})

race_id = url.split('/')[-1].split('-')[0]
race_name = soup.find('div',class_='race__name').text.strip()
race_date = soup.find('div',class_='race__date').text.strip()
clean_date = datetime.strptime(race_date,'%d %b %Y').strftime('%Y%m%d')
race_info1 = soup.find_all('div',class_='race__subtitle')[0].text.strip()
race_info2 = soup.find_all('div',class_='race__subtitle')[1].text.strip()

final = []
for row in table.find_all('div',class_='racecard__runner--content'):
    try:
        num = row.find('div',class_='racecard__runner__cloth-number').text.strip()
        last_days_ugly = row.find('div',class_='racecard__runner__name').find('a').find('sup').text
        horse_name = row.find('div',class_='racecard__runner__name').find('a').text.strip().replace(last_days_ugly,'')
        horse_link = 'http://www.racingtv.com'+row.find('div',class_='racecard__runner__name').find('a')['href']
        last_race_days = last_days_ugly.strip().replace('(','').replace(')','')
        for people in row.find_all('div',class_='racecard__runner__person'):
            if 'J:' in people.getText():
                jockey = people.find('a').text.strip()
                jockey_link = 'http://www.racingtv.com'+people.find('a')['href']
            if 'T:' in people.getText():
                trainer = people.find('a').text.strip()
                trainer_link = 'http://www.racingtv.com'+people.find('a')['href']
        form = row.find('div',class_='racecard__runner__column--form_lr').find_all('div')[0].text.strip()
        equip = row.find('div',class_='racecard__runner__column--form_lr').find_all('div')[1].text.strip()
        weight = row.find('div',class_='racecard__runner__column--weight_age').find_all('div')[0].text.strip()
        age = row.find('div',class_='racecard__runner__column--weight_age').find_all('div')[1].text.strip()
        o_r = row.find('div',class_='racecard__runner__column--or').text.strip()
        odds = row.find('div',class_='racecard__runner__column--price').getText()
        odds_dec = row.find('div',class_='racecard__runner__column--price').find('ruk-odd')['data-js-odds-decimal']
        odds_data = row.find('div',class_='racecard__runner__column--price').find('ruk-odd')['data-js-odd-alternatives']

    except AttributeError: #skip blank starting gates
        continue

    item = {
        'race_url' : url,
        'race_id': race_id,
        'race_name':race_name,
        'race_date':clean_date,
        'race_info1':race_info1,
        'race_info2':race_info2,
        'num': num,
        'horse_name':horse_name,
        'horse_link':horse_link,
        'last_race_days':last_race_days,
        'jockey':jockey,
        'jockey_link':jockey_link,
        'trainer':trainer,
        'trainer_link':trainer_link,
        'form':form,
        'equip':equip,
        'weight':weight,
        'age':age,
        'o_r':o_r,
        'odds':odds,
        'odds_dec':odds_dec,
        'odds_data':odds_data
    }
    final.append(item)

df = pd.DataFrame(final)
df.to_csv('racingtv.csv',index=False)

print('Saved to racingtv.csv')
user_113389000

Following on from the script supplied kindly by bushcat69 and my subsequent question "how to get the race time into the df" I have cobbled together some code (cut and paste from other sites). I thought you may be interested. It may not be elegant but it seems to work. The section:

race_data.extend(get_racecards_data(url_race, date, racetime

is used to pass the url etc to the bushcat69 script. Thanks again.

def get_meetings():
global date
global date_ext
odds_date = date_ext

url = f'https://www.racingtv.com/racecards/{date_ext}'

try:
    res = requests.get(url, headers = headers)
except:
    print('Date or Connection error occured! \nTry again!!')
    return

soup = BeautifulSoup(res.text, 'html.parser')
meetings = soup.select('.race-selector__times__race')
course_num = len(meetings)

meetings1 = [a['href'] for a in soup.select('.race-selector__times__race')]
course_num = len(meetings1)

cnt01 = 0

if course_num == 0:
  print('Provide a upcoming valid date')
  return

for track in meetings1[:course_num]:
    cnt01 = cnt01 + 1
    trackref = track.split("/")[2]
    print(cnt01, ": ", trackref)

need = input(f'{course_num} courses found \nHow many courses to scrape? Press \'a\' for all :\n')
if need == 'a':
    n = course_num
else:
    try:
        n = int(need)
    except:
        print('Invalid input !')
        return

cnt01 = 0
race_data = []
for mtm in meetings[:course_num]:
    cnt01 = cnt01 + 1
    racetime = mtm.text
    href = mtm.attrs
    htxt = Text(href)
    url_race = htxt.partition("/")[2]
    url_race = "/" + url_race.rpartition("'")[0]
    print(cnt01, racetime, url_race)
    time.sleep(1)
    race_data.extend(get_racecards_data(url_race, date, racetime))
    print(f"Meeting {url_race.split('/')[2]} scraping completed")
    if cnt01 == n:
        break

df_race = pd.DataFrame(race_data)
df = df_race
user_178654910
Related