i am ubable to crawl multiple pages of zillow.com don't know , search on google , stackoverflow but didn't get any proper answer or resone of this error it is giving perfect result for first page but not crawling multiple pages
getting this error again and again .
TypeError(f"Request url must be str, got {type(url).name}") TypeError: Request url must be str, got NoneType
here is my code
import scrapy
from ..utils import URL, cookie_parser, parse_new_url
from ..items import ZillowItem
from scrapy.loader import ItemLoader
import json
class ZillowhousesSpider(scrapy.Spider):
name = 'zillowhouses'
allowed_domains = ['www.zillow.com']
def start_requests(self):
yield scrapy.Request(
url=URL,
callback=self.parse,
cookies=cookie_parser(),
meta={
'currentPage': 1
}
)
def parse(self, response):
current_page = response.meta['currentPage']
json_resp = json.loads(response.body) #to convert json object into python dict
houses = json_resp.get('cat1').get('searchResults').get('listResults')
for house in houses:
loader = ItemLoader(item=ZillowItem())
loader.add_value('id', house.get('id'))
loader.add_value('img_src', house.get('imgSrc'))
loader.add_value('details_url', house.get('detailUrl'))
loader.add_value('address', house.get('address'))
loader.add_value('area_sqft', house.get('area'))
loader.add_value('zipcode', house.get('addressZipcode'))
loader.add_value('broker_name', house.get('brokerName'))
loader.add_value('status_text', house.get('statusText'))
loader.add_value('status_type', house.get('statusType'))
loader.add_value('latitude', house.get('latLong').get('latitude'))
loader.add_value('longitude', house.get('latLong').get('longitude'))
loader.add_value('price', house.get('price'))
loader.add_value('baths', house.get('baths'))
loader.add_value('beds', house.get('beds'))
yield loader.load_item()
total_pages = json_resp.get('cat1').get('searchList').get('totalPages')
if current_page <= total_pages:
current_page+= 1
yield scrapy.Request(
url = parse_new_url(URL, page_number = current_page),
callback=self.parse,
cookies=cookie_parser(),
meta= {
'currentPage':current_page
}
)
here is my items.py
import scrapy
from scrapy.loader.processors import TakeFirst
class ZillowItem(scrapy.Item):
id = scrapy.Field(
output_processor= TakeFirst()
)
img_src = scrapy.Field(
output_processor= TakeFirst()
)
details_url = scrapy.Field(
output_processor= TakeFirst()
)
address = scrapy.Field(
output_processor= TakeFirst()
)
area_sqft = scrapy.Field(
output_processor= TakeFirst()
)
zipcode = scrapy.Field(
output_processor= TakeFirst()
)
broker_name = scrapy.Field(
output_processor= TakeFirst()
)
status_text = scrapy.Field(
output_processor= TakeFirst()
)
status_type = scrapy.Field(
output_processor= TakeFirst()
)
latitude = scrapy.Field(
output_processor= TakeFirst()
)
longitude = scrapy.Field(
output_processor= TakeFirst()
)
price = scrapy.Field(
output_processor= TakeFirst()
)
baths = scrapy.Field(
output_processor= TakeFirst()
)
beds = scrapy.Field(
output_processor= TakeFirst()
)
Here is my setting.py
BOT_NAME = 'zillow'
SPIDER_MODULES = ['zillow.spiders']
NEWSPIDER_MODULE = 'zillow.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/105.0.0.0 Safari/537.36'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 10
here is my utils.py
from http.cookies import SimpleCookie
from urllib.parse import urlparse, parse_qs, urlencode
import json
URL ='https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=%7B%22usersSearchTerm%22%3A%22Miami%2C%20FL%22%2C%22mapBounds%22%3A%7B%22west%22%3A-80.5178131866171%2C%22east%22%3A-80.07149360653898%2C%22south%22%3A25.63146189889158%2C%22north%22%3A25.934422885146812%7D%2C%22mapZoom%22%3A11%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A12700%2C%22regionType%22%3A6%7D%5D%2C%22isMapVisible%22%3Afalse%2C%22filterState%22%3A%7B%22isAllHomes%22%3A%7B%22value%22%3Atrue%7D%2C%22sortSelection%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D&wants={%22cat1%22:[%22listResults%22],%22cat2%22:[%22total%22]}&requestId=7'
def cookie_parser():
cookie_string = 'zguid=24|%24bd1cc7fb-3a96-4355-82ad-eb54b6582fc0; zgsession=1|c8c7aa18-a869-4fa5-acc2-0974c45c5397; _ga=GA1.2.2098963959.1662992109; _gid=GA1.2.646099340.1662992109; zjs_user_id=null; zg_anonymous_id=%2272ad9701-5772-4341-9553-2d5c6dbbb727%22; zjs_anonymous_id=%22bd1cc7fb-3a96-4355-82ad-eb54b6582fc0%22; _gcl_au=1.1.435090296.1662992115; KruxPixel=true; DoubleClickSession=true; pxcts=53406f29-32a5-11ed-9caa-487079506e4a; _pxvid=53403445-32a5-11ed-9caa-487079506e4a; _cs_c=0; _hp2_id.1215457233=%7B%22userId%22%3A%224388082776070223%22%2C%22pageviewId%22%3A%225521802898055122%22%2C%22sessionId%22%3A%22112719343168408%22%2C%22identity%22%3Anull%2C%22trackerVersion%22%3A%224.0%22%7D; KruxAddition=true; __pdst=222e1aef96394f48950db65677af5fb6; _cs_id=bc3f13b6-b504-a600-8f00-aa54819814a3.1662992134.1.1662992134.1662992134.1.1697156134010; utag_main=v_id:0183320dff7a0008f6511b590fb90506f00170670086e$_sn:1$_se:1$_ss:1$_st:1662993921725$ses_id:1662992121725%3Bexp-session$_pn:1%3Bexp-session$dcsyncran:1%3Bexp-session$tdsyncran:1%3Bexp-session$dc_visit:1$dc_event:1%3Bexp-session$dc_region:eu-central-1%3Bexp-session; _fbp=fb.1.1662992137787.1566635397; _pin_unauth=dWlkPU5EazROemRsTnpRdE5HVmxNUzAwT0RVeUxUbGpaV1F0WXpreFpUWmpaVEUyTjJRNA; __gads=ID=3a3aac85687f3a37-22e4e67cd5d50051:T=1662992221:S=ALNI_MZ4vIKVY--1rhaS-Dlmyq494eHCRA; __gpi=UID=00000accff902937:T=1662992221:RT=1662992221:S=ALNI_MbsEUqyyMfnfGRtmDdAAnNWlAQEtA; _clck=1ar1pmr|1|f4t|0; G_ENABLED_IDPS=google; _uetsid=5a9c690032a511ed945bf72202409c10; _uetvid=5a9cc21032a511ed9a012f92552709bc; _gat=1; AWSALB=ihmWxilxzEdWoOVOnUogTwdbVoGwbhOlUpMcMQ/Dg6X5L0UASrqwYngE9rS1l4TJD4lpP/qc1/uW+eAPWsvyrESYOcPEdG+h2WLYuNjwZl83rVNkanY/kFlI1Wc1; AWSALBCORS=ihmWxilxzEdWoOVOnUogTwdbVoGwbhOlUpMcMQ/Dg6X5L0UASrqwYngE9rS1l4TJD4lpP/qc1/uW+eAPWsvyrESYOcPEdG+h2WLYuNjwZl83rVNkanY/kFlI1Wc1; JSESSIONID=F4D9C55C95F31C49899E6A8955CC3AF7; search=6|1665600742919%7Crect%3D25.934422885146812%252C-80.07149360653898%252C25.63146189889158%252C-80.5178131866171%26rid%3D12700%26disp%3Dmap%26mdm%3Dauto%26p%3D2%26z%3D1%26fs%3D1%26fr%3D0%26mmm%3D0%26rs%3D0%26ah%3D0%26singlestory%3D0%26housing-connector%3D0%26abo%3D0%26garage%3D0%26pool%3D0%26ac%3D0%26waterfront%3D0%26finished%3D0%26unfinished%3D0%26cityview%3D0%26mountainview%3D0%26parkview%3D0%26waterview%3D0%26hoadata%3D1%26zillow-owned%3D0%263dhome%3D0%26featuredMultiFamilyBuilding%3D0%09%0912700%09%09%09%09%09%09'
cookie = SimpleCookie()
cookie.load(cookie_string)
cookies = {}
for key, morsel in cookie.items():
cookies[key] = morsel.value
return cookies
def parse_new_url(url, page_number):
url_parsed = urlparse(url)
query_string = parse_qs(url_parsed.query)
search_query_state = json.loads(query_string.get('searchQueryState')[0])
search_query_state['pagination'] = {"currentPage": page_number}
query_string.get('searchQueryState')[0] = search_query_state
encoded_qs = urlencode(query_string, doseq=1)
new_url = f"https://www.zillow.com/search/GetSearchPageState.htm?{encoded_qs}"
print(new_url)
parse_new_url(URL, 3)