I'm trying to scrape bettingdata from https://www.bingoal.be/nl/Sport#/sc/98. Via the network tab I'm able to retrieve the cURL which is downloading the data, via a cURL converter I'm able to convert it to Python and get JSON data. So far so good, unfortunately after some time the cookies aren't valid anymore so I want to use selenium to download the cookies and update my python code with the most recent cookie. I found out that the "UST" cookie is the one which is "blocking" my code to work. If I update the UST cookie with the most valid UST cookie, I'm not getting any response anymore. Can somefind trigger out why this is happening / how to fix this?
My code so far:
import logging
import requests
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from seleniumwire import webdriver as wd
from webdriver_manager.chrome import ChromeDriverManager
def getCookieForBingoal():
WINDOW_SIZE = "1920,1080"
logging.getLogger('WDM').setLevel(logging.NOTSET)
chrome_options = Options()
chrome_options.add_argument("--window-size=%s" % WINDOW_SIZE)
cookie_value = ""
s = Service(ChromeDriverManager().install())
driver = wd.Chrome(service=s, options=chrome_options)
url = "https://www.bingoal.be/nl/Sport#/sc/98"
driver.get(url=url)
cookies = driver.get_cookies()
cookie_ust = ""
for cookie in cookies:
print(cookie)
if cookie["name"] == 'ust':
cookie_ust = cookie["value"]
driver.quit()
return cookie_ust
cookies = {
'ust': '202209231216223df1ffa351a67f681192931f1083a943b1591c7b710bfa6a45553872ac91c80d',
'_hjFirstSeen': '1',
'_hjSession_2935299': 'eyJpZCI6IjYyY2RjZDdiLTEyZGYtNDUzMy1hYmEwLTBlMTEyZjM3ZDc3MiIsImNyZWF0ZWQiOjE2NjM5MjgxODg1NzcsImluU2FtcGxlIjpmYW12ZX0=',
'_hjAbsoluteSessionInProgress': '0',
'cookieMode': 'all',
'lastType': '1',
'lastBets': '',
'spoMenu': '',
'_gcl_au': '1.1.1550563930.1663928193',
'_gid': 'GA1.2.669725060.1663928194',
'_hjSessionUser_2935299': 'eyJpZCI6IjYxNjIxNTE2LTcwMmEtNWI0ZC1iNzg1LTg3ZmE2MzQwOGZmYSIsImNyZWF0ZWQiOjE2NjM5MjgxODgzMzYsImV4aXN0aW5nIjp0cnVlf1==',
'_pprv': '%7B%22consent%22%3A%7B%220%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%221%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%222%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%223%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%224%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%225%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%226%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%227%22%3A%7B%22mode%22%3A%22opt-in%22%7D%7D%7D',
'cX_S': 'l8ebvw3w479czvck',
'_pcid': '%7B%22browserId%22%3A%22l8ebvvxoapnar88y%22%7D',
'cX_P': 'l8ebvvxoapnar88y',
'cX_G': 'cx%3A3m35j2emeppmg27b3qdpqaiyew%3A2y3o0k6wj1a5b',
'_pctx': '%7Bu%7DN4IgDghg5gpgagSxgdwJIBMQC4QBsAcMAxgAxEwQBGAbGAM67XX4gA0IArnTAE53YA7DrlzsuvAMoAXCFK7YQEAQHsBbEHQRSYGBQEY9AZnzUSAJj35DJPQHYAnLaaHqIAL5A',
'cto_bundle': 'K-TZMl9CTXNiVTBMdHpBTHklMkYlMkIlMkY1anhjOFVzTVZLb01LM3BqVGw0NE05MXF4Ynowd0NZZzRCT3JWOGdCU1VTNGw4RVlDSXJCOWFWNERlWUlVdjVaaGdRTmRKNDVtckhicFVveGhxUWFIeVp1YTJLWU1PYiUyQkN3SVBVM25NY1VBT25ETzQ2Q29hSHFzMzZWaEZYU2RwU1VIc0dTdyUzRCUzRA',
'CSPSESSIONID-SP-80-UP-': '042002000000278yxI6nZu1EwBHOc5pl8Y6hfVB6wWKa9g0SoU',
'_ga_2J0LMSM6JQ': 'GS1.1.1663928400.1.1.1663929830.60.0.0',
'_ga': 'GA1.1.1024746643.1663928194',
}
headers = {
'Accept': '*/*',
'Accept-Language': 'nl-NL,nl;q=0.9,en-US;q=0.8,en;q=0.7',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
# Requests sorts cookies= alphabetically
# 'Cookie': 'ust=202209231216223df1ffa351a67f681192931f1083a943b1591c7b710bfa6a45553872ac91c80d; _hjFirstSeen=1; _hjSession_2935299=eyJpZCI6IjYyY2RjZDdiLTEyZGYtNDUzMy1hYmEwLTBlMTEyZjM3ZDc3MiIsImNyZWF0ZWQiOjE2NjM5MjgxODg1NzcsImluU2FtcGxlIjpmYWxzZX0=; _hjAbsoluteSessionInProgress=0; cookieMode=all; lastType=1; lastBets=; spoMenu=; _gcl_au=1.1.1550563930.1663928193; _gid=GA1.2.669725060.1663928194; _hjSessionUser_2935299=eyJpZCI6IjYxNjIxNTE2LTcwMmEtNWI0ZC1iNzg1LTg3ZmE2MzQwOGZmYSIsImNyZWF0ZWQiOjE2NjM5MjgxODgzMzYsImV4aXN0aW5nIjp0cnVlfQ==; _pprv=%7B%22consent%22%3A%7B%220%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%221%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%222%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%223%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%224%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%225%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%226%22%3A%7B%22mode%22%3A%22opt-in%22%7D%2C%227%22%3A%7B%22mode%22%3A%22opt-in%22%7D%7D%7D; cX_S=l8ebvw3w479czvck; _pcid=%7B%22browserId%22%3A%22l8ebvvxoapnar88y%22%7D; cX_P=l8ebvvxoapnar88y; cX_G=cx%3A3m35j2emeppmg27b3qdpqaiyew%3A2y3o0k6wj1a5b; _pctx=%7Bu%7DN4IgDghg5gpgagSxgdwJIBMQC4QBsAcMAxgAxEwQBGAbGAM67XX4gA0IArnTAE53YA7DrlzsuvAMoAXCFK7YQEAQHsBbEHQRSYGBQEY9AZnzUSAJj35DJPQHYAnLaaHqIAL5A; cto_bundle=K-TZMl9CTXNiVTBMdHpBTHklMkYlMkIlMkY1anhjOFVzTVZLb01LM3BqVGw0NE05MXF4Ynowd0NZZzRCT3JWOGdCU1VTNGw4RVlDSXJCOWFWNERlWUlVdjVaaGdRTmRKNDVtckhicFVveGhxUWFIeVp1YTJLWU1PYiUyQkN3SVBVM25NY1VBT25ETzQ2Q29hSHFzMzZWaEZYU2RwU1VIc0dTdyUzRCUzRA; CSPSESSIONID-SP-80-UP-=042002000000278yxI6nZu1EwBHOc5pl8Y6hfVB6wWKa9g0SoU; _ga_2J0LMSM6JQ=GS1.1.1663928400.1.1.1663929830.60.0.0; _ga=GA1.1.1024746643.1663928194',
'Origin': 'https://www.bingoal.be',
'Referer': 'https://www.bingoal.be/nl/Sport',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
'sec-ch-ua': '"Google Chrome";v="105", "Not)A;Brand";v="8", "Chromium";v="105"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
}
data = {
'func': 'sc',
'ver': '1',
'id': '98',
'ts': '20220923 13:43:49',
'k': '7115',
}
response = requests.post('https://www.bingoal.be/A/sport', cookies=cookies, headers=headers, data=data)
try:
print(response.json())
except:
print("Error")
So this code is working. But when I do something like ust_cookie = getCookieForBingoal() and pass this ust_cookie to the ust value of the cookie, then you will see you don't get any response anymore.
Thanks a lot already. KR, Stef