I am trying to scrape shopee item information using python.
As it is using ajax, I am trying to extract item info from: https://shopee.com.my/api/v2/item/get?itemid=5859069631&shopid=206039726
After a few requests, i found out that the json it responded turned out to be fake values (such as its actual rating is 4.78 but it returns 0.24).
I tried to solve this problem by changing headers and ip/proxy, but still not working.
Is there any other way that can potentially solve this?
def get_info(url,itemurl):
requests.adapters.DEFAULT_RETRIES = 5
s = requests.session()
s.keep_alive = False
try:
fake_ua=UserAgent()
headers = {'User-Agent':fake_ua.random,
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.5',
'X-Shopee-Language': 'en',
'X-Requested-With': 'XMLHttpRequest',
'X-API-SOURCE': 'pc',
'If-None-Match-': '55b03-2ff39563c299cbdc937f8ab86ef322ab',
'DNT': '1',
'Referer': referer,
'TE': 'Trailers'}
ip = get_daili()
proxies = {"proxies":{"https":ip}}
response = requests.get(url, headers = headers, proxies = proxies, verify=False)
#response = requests.request("GET", url, headers=headers, data=payload)
if response.status_code == 200:
shop_info = response.json()
except requests.ConnectionError as e:
print(f' {url} error', e.args)
shop_name = shop_info['data']['name']
followers = shop_info['data']['follower_count']
ratinggood = shop_info['data']['rating_good']
ratingbad = shop_info['data']['rating_bad']
ratingnormal = shop_info['data']['rating_normal']
try:
fake_ua=UserAgent()
headers = {'User-Agent':fake_ua.random,
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.5',
'X-Shopee-Language': 'en',
'X-Requested-With': 'XMLHttpRequest',
'X-API-SOURCE': 'pc',
'If-None-Match-': '55b03-2ff39563c299cbdc937f8ab86ef322ab',
'DNT': '1',
'Referer': referer,
'TE': 'Trailers'}
ip = get_daili()
proxies = {"proxies":{"https":ip}}
response = requests.get(itemurl, headers = headers, proxies = proxies, verify=False)
#response = requests.request("GET", itemurl, headers=headers, data=payload)
if response.status_code == 200:
item_info = response.json()
except requests.ConnectionError as e:
print(f' {url} error', e.args)
#print(json.dumps(item_info, indent=4))
print(itemurl)