Can't scrape customized property links from zillow using requests

user_7180194 0
Viewed 246

I'm trying to parse different property links which are populated when I select two dropdowns from zillow. When I'm done choosing the options, I can see the results in json in dev tools. However, when I do the same using the script below, I get some weird text.

Manual actions:

  1. Navigated to that site
  2. Chose options from first dropdown
  3. Chose options from second dropdown

This is how I've tried to automate:

import json
import requests
from pprint import pprint

link = 'https://www.zillow.com/search/GetSearchPageState.htm?'

params = {
    'searchQueryState': {"pagination":{},"usersSearchTerm":"Vista, CA","mapBounds":{"west":-117.44051346728516,"east":-116.99488053271484,"south":33.126944633035116,"north":33.27919773006566},"regionSelection":[{"regionId":41517,"regionType":6}],"isMapVisible":True,"filterState":{"doz":{"value":"6m"},"isForSaleByAgent":{"value":False},"isForSaleByOwner":{"value":False},"isNewConstruction":{"value":False},"isForSaleForeclosure":{"value":False},"isComingSoon":{"value":False},"isAuction":{"value":False},"isPreMarketForeclosure":{"value":False},"isPreMarketPreForeclosure":{"value":False},"isRecentlySold":{"value":True},"isAllHomes":{"value":True},"hasPool":{"value":True},"hasAirConditioning":{"value":True},"isApartmentOrCondo":{"value":False}},"isListVisible":True,"mapZoom":11},
    'wants': {"cat1":["listResults","mapResults"]},
    'requestId': 2
}

with requests.Session() as s:
    s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
    res = s.get(link,params=json.dumps(params))
    pprint(res.content)

This is the output it produces:

b'<!-- This page outputs JSON instead of anything written here. -->'

How can I parse customized property links from zillow using requests?

python-requests web-scraping python-3.x python json
1 Answers

You have to encode the query string as it appears in the request URL.

To do so you need:

urllib.parse.urlencode()

Here's a working example:

import json
import urllib.parse

import requests

link = 'https://www.zillow.com/search/GetSearchPageState.htm?'

params = {
    'searchQueryState': {
        "pagination": {},
        "usersSearchTerm": "Vista, CA",
        "mapBounds": {
            "west": -117.44051346728516,
            "east": -116.99488053271484,
            "south": 33.126944633035116,
            "north": 33.27919773006566
        },
        "regionSelection": [{"regionId": 41517, "regionType": 6}],
        "isMapVisible": True,
        "filterState": {
            "doz": {"value": "6m"}, "isForSaleByAgent": {"value": False},
            "isForSaleByOwner": {"value": False}, "isNewConstruction": {"value": False},
            "isForSaleForeclosure": {"value": False}, "isComingSoon": {"value": False},
            "isAuction": {"value": False}, "isPreMarketForeclosure": {"value": False},
            "isPreMarketPreForeclosure": {"value": False},
            "isRecentlySold": {"value": True}, "isAllHomes": {"value": True},
            "hasPool": {"value": True}, "hasAirConditioning": {"value": True},
            "isApartmentOrCondo": {"value": False}
        },
        "isListVisible": True,
        "mapZoom": 11
    },
    'wants': {"cat1": ["listResults"]},
    'requestId': 2
}

with requests.Session() as s:
    s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
    s.headers["x-requested-session"] = "BE6D8DA620E60010D84B55EB18DC9DC8"
    s.headers["cookie"] = f"JSESSIONID={s.headers['x-requested-session']}"
    data = json.dumps(
        json.loads(s.get(f"{link}{urllib.parse.urlencode(params)}").content),
        indent=2
    )
    print(data)

Output:

{
  "user": {
    "isLoggedIn": false,
    "hasHousingConnectorPermission": false,
    "savedSearchCount": 0,
    "savedHomesCount": 0,
    "personalizedSearchGaDataTag": null,
    "personalizedSearchTraceID": "607a9ecb5aabe489c361c1d91f368b37",
    "searchPageRenderedCount": 0,
    "guid": "33b7add3-bfd3-4d85-a88a-d9d99256d2a2",
    "zuid": "",
    "isBot": false,
    "userSpecializedSEORegion": false
  },
  "mapState": {
    "customRegionPolygonWkt": null,
    "schoolPolygonWkt": null,
    "isCurrentLocationSearch": false,
    "userPosition": {
      "lat": null,
      "lon": null
    },
    "regionBounds": {
      "north": 33.275284,
      "east": -117.145153,
      "south": 33.130865,
      "west": -117.290241
    }
  },

and much much more ...

Note: Go easy on that site as they have a pretty sensitive anti-bot measures and if you keep requesting the data too quickly, they'll throw a CAPTCHA at you.

user_61067910
Related