I tried running the below program, but it only returns first 9 entries from the webpage but I need all the listings shown in the url which is approx 40
import requests
from bs4 import BeautifulSoup
import re
url = 'https://www.zillow.com/san-francisco-ca/?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22usersSearchTerm%22%3A%22San%20Francisco%2C%20CA%22%2C%22mapBounds%22%3A%7B%22west%22%3A-122.55177585009766%2C%22east%22%3A-122.31488314990234%2C%22south%22%3A37.68215383963642%2C%22north%22%3A37.86831288097759%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A20330%2C%22regionType%22%3A6%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%2C%22ah%22%3A%7B%22value%22%3Atrue%7D%7D%2C%22isListVisible%22%3Atrue%2C%22mapZoom%22%3A12%7D'
headers = {
"User-Agent": 'My User Agent 1.0'
}
req = requests.get(url, headers=headers)
soup = BeautifulSoup(req.content,"html.parser")
container = soup.find_all('li')
for lst in container:
articles = lst.findAll("div", {"class": "list-card-price"})
print(articles)