I can't find a POST request when I check the Network tab in dev tools. I only see GET requests. Am I doing something wrong???
No, just you can't see POST requests because the data you send is passed as query string, it means that data is encoded in the url.
If you open Dev Tools and click on the first request once you have clicked the button (or even a link), you can go to params section and get all the data you sent.
Using the requests module the form data in sent with the data keyword with a POST request, while the parameters that are encoded in the url are passed to the params keyword with both GET and POST methods.
for example, this is exactly the request I made when I added the item to the watch list:
url = 'https://www.ebay.com/myb'
headers = {
'Host': 'www.ebay.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko'
'/20100101 Firefox/61.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q='
'0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'https://www.ebay.com/itm/Rockville-HTS56-1000w-5-1-Channel-'
'Home-Theater-System-Bluetooth-USB-8-Subwoofer/302495341359?_trkpar'
'ms=pageci%3Ac45ae602-b2bb-11e8-91a9-74dbd1807185%7Cparentrq%3Ab4e1'
'b5ee1650a9ccac0001e5fffeeebd%7Ciid%3A1',
'Cookie': 'JSESSIONID=6E3675DDC01917342E915A485425DE14; ebay=%5Ecv%3D15'
'555%5Esbf%3D%2310000100000%5Epsi%3DATBv5t4c*%5Ejs%3D1%5E; dp1=bu1p'
'/QEBfX0BAX19AQA**5d73f732^bl/IT5f552ab2^pbf/#800080000080000000000'
'05f552ac5^; s=CgAD4ACBblBUyYjU0YzFiZjkxNjUwYTk5Yjc4NzAzZjhhZmZjNjJ'
'iNjClGshN; nonsession=CgAAIABxbulCyMTUzNjM0NDk4OHgzMDI0OTUzNDEzNTl'
'4MHgyTgDKACBk+MUyYjU0YzFiZjkxNjUwYTk5Yjc4NzAzZjhhZmZjNjJiNjAAywABW'
'5LK0jZ87uR5; ak_bmsc=CBC404CFCC021D437EAEB56AB0A505C80212FF677D1B0'
'0009DC3925B8422CF41~pl1XkGGUVkQmdzLfSzxOHqS7a6B5bt6IE+YZ9pBQsojU23'
'4gAkOREldw07haa9wqBjRKkfaGqXnWck+XkoiOMH75VNvp7RX0Tswwmgd2XI2DLpTf'
'Z3Wic4ULyIjHQiolAXprZboWAssr45zCzbT1DEfphZ+3vHtD2sZcfcIUj/u5hrbWmX'
'WcqZHABtvn/XDI5z8ul1rnRe0ZM87TfkySxS09SXR1c+HoE8BVBm0WeSB6o=; npii'
'=btguid/b54c1bf91650a99b78703f8affc62b605d73f72e^cguid/b54c2316165'
'0ac3c480165a5fe7d7b1e5d73f72e^; cssg=b54c1bf91650a99b78703f8affc62'
'b60; bm_sv=43E32CFCCEEB3DE24A7853BCD296554D~5g3PXHhS+OOCPoJYdO/hGo'
'GEWmrSON6AvaW8RYaPM31Yhe4afGf1MM/OmSgHoFPrTLloRcPphW1KrOy4IjnUiiHU'
'BHq60fazRhTC9rdF6bweXE9Oyz02T4zoySTDLYfL8SJtb99/tNa5v1jarB5cjA==; '
'AMCV_A71B5B5B54F607AB0A4C98A2%40AdobeOrg=-1758798782%7CMCIDTS%7C17'
'782%7CMCMID%7C80964471016865310950758990401017778531%7CMCAID%7CNON'
'E%7CMCOPTOUT-1536352227s%7CNONE; AMCVS_A71B5B5B54F607AB0A4C98A2%40'
'AdobeOrg=1; ds2=sotr/b7pwxzzzzzzz^',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'DNT': '1',
}
params = {
'_trksid:p2047675.l1359',
'SubmitAction.AddToListVI': 'x',
'item': '302495341359',
'rt': 'nc',
'srt': '0100030000005006d271d47b3a0557eff3cbcd450ad13d38dd94ca9e'\
'2d8918de753cf5a1dfc6eeb0648b5e9c433cbf106609c6d81bed4ad1fa6'\
'fa0fdbeca4bc2e3ae88a523453c4a5620551f91a45384f9d5a4054f8e56',
'etn': 'Watch list',
'tagId': '-99',
'wt': 'f1cc17761369fcda30b0792ff44e1a09',
'ssPageName': 'VIP:watchlink:top:en',
'sourcePage': '4340',
}
requests.get(url, headers=headers, params=params)
[EDIT 1]
Working code example to scrape "Add to watch list" button:
from requests import Session
from bs4 import BeautifulSoup
url = 'https://www.ebay.com/itm/Rockville-HTS56-1000w-5-1-Cha'\
'nnel-Home-Theater-System-Bluetooth-USB-8-Subwoofer/302'\
'495341359?_trkparms=pageci%3Ac45ae602-b2bb-11e8-91a9-7'\
'4dbd1807185%7Cparentrq%3Ab4e1b5ee1650a9ccac0001e5fffee'\
'ebd%7Ciid%3A1'
def get_watch_list_url(page_source):
"""
Return "Add to watch list" button url.
"""
soup = BeautifulSoup(page_source)
for button in soup.find_all('a'):
if button.get_text() == 'Add to watch list':
return button.get('href')
def main():
with Session() as session:
response = session.get(url) # visit item page and scrape button
add_to_watch_list_url = get_watch_list_url(response.text)
print('Url is:', add_to_watch_list_url)
response = session.get(add_to_watch_list_url) # add item to watch list
if response.ok:
print('Item successfully added to watch list')
if __name__ == '__main__':
main()
[EDIT 2]
Example of implementation with aiohttp:
import aiohttp
import asyncio
from bs4 import BeautifulSoup
url = 'https://www.ebay.com/itm/Rockville-HTS56-1000w-5-1-Cha'\
'nnel-Home-Theater-System-Bluetooth-USB-8-Subwoofer/302'\
'495341359?_trkparms=pageci%3Ac45ae602-b2bb-11e8-91a9-7'\
'4dbd1807185%7Cparentrq%3Ab4e1b5ee1650a9ccac0001e5fffee'\
'ebd%7Ciid%3A1'
def get_watch_list_url(page_source):
soup = BeautifulSoup(page_source, "html.parser")
for button in soup.find_all('a'):
if button.get_text() == 'Add to watch list':
return button.get('href')
async def main():
async with aiohttp.ClientSession() as session:
# Send request and get response.
async with session.get(url) as response:
html = await response.text()
# Extract button url.
add_to_watch_list_url = get_watch_list_url(html)
# Add item to watch list.
async with session.get(add_to_watch_list_url) as response:
status = response.status
print('Request status code:', status) # print the request status
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
