I want to scrape prices of the orders in amazon.com with Requests Library
The problem is, Amazon does not give me the related data because I can not choose a delivery address. How can I add a delivery address to the cookies?
Example; When I selected delivery on chrome
when I didn't select
my code:
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
import pandas as pd
import random
import requests
from datetime import datetime
import re
from lxml import etree
from random import choice
import json
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
random1 = random.randrange(7, 10, 1)
random2 = random.randrange(70, 79, 1)
random3 = random.randrange(100, 130, 1)
random4 = random.randrange(5000, 9999, 1)
headers = {
'User-Agent': "Mozilla/5.0 (Windows NT "+str(random1)+".0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/"+str(random2)+".0."+str(random4)+"."+str(random3)+" Safari/537.36",
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'TE': 'Trailers'}
url = "https://www.amazon.com/gp/offer-listing/B07PXGQC1Q/ref=olp_f_new?ie=UTF8&f_new=true"
req = requests.get(url, headers=headers)
soup = BeautifulSoup(req.text, 'lxml')
pricedata=soup.select(".olpOfferPrice")
print(pricedata)