TypeError(f"Request url must be str, got {type(url).__name__}") TypeError: Request url must be str, got NoneType

Viewed 31

i am ubable to crawl multiple pages of zillow.com don't know , search on google , stackoverflow but didn't get any proper answer or resone of this error it is giving perfect result for first page but not crawling multiple pages

getting this error again and again .

TypeError(f"Request url must be str, got {type(url).name}") TypeError: Request url must be str, got NoneType

here is my code

import scrapy
from ..utils import URL, cookie_parser, parse_new_url
from ..items import ZillowItem
from scrapy.loader import ItemLoader
import json

class ZillowhousesSpider(scrapy.Spider):
    name = 'zillowhouses'
    allowed_domains = ['www.zillow.com']
    
    def start_requests(self):
        yield scrapy.Request(
            url=URL,
            callback=self.parse,
            cookies=cookie_parser(),
            meta={
                'currentPage': 1
            }
        )
        
    def parse(self, response):
        current_page = response.meta['currentPage']
        json_resp = json.loads(response.body)   #to convert json object into python dict
        houses = json_resp.get('cat1').get('searchResults').get('listResults')
        
        for house in houses:
            loader = ItemLoader(item=ZillowItem())
            loader.add_value('id', house.get('id'))
            loader.add_value('img_src', house.get('imgSrc'))
            loader.add_value('details_url', house.get('detailUrl'))
            loader.add_value('address', house.get('address'))
            loader.add_value('area_sqft', house.get('area'))
            loader.add_value('zipcode', house.get('addressZipcode'))
            loader.add_value('broker_name', house.get('brokerName'))
            loader.add_value('status_text', house.get('statusText'))
            loader.add_value('status_type', house.get('statusType'))
            loader.add_value('latitude', house.get('latLong').get('latitude'))
            loader.add_value('longitude', house.get('latLong').get('longitude'))
            loader.add_value('price', house.get('price'))
            loader.add_value('baths', house.get('baths'))
            loader.add_value('beds', house.get('beds'))
            
            yield loader.load_item()
            
        total_pages = json_resp.get('cat1').get('searchList').get('totalPages')
        
        if current_page <= total_pages:
            current_page+= 1 
            yield scrapy.Request(
                url = parse_new_url(URL, page_number = current_page),
                callback=self.parse,
                cookies=cookie_parser(),
                meta= {
                    'currentPage':current_page 
                }
                )

here is my items.py

import scrapy
from scrapy.loader.processors import TakeFirst


class ZillowItem(scrapy.Item):
    id = scrapy.Field(
        output_processor= TakeFirst()
    )
    img_src = scrapy.Field(
        output_processor= TakeFirst()
    )
    details_url = scrapy.Field(
        output_processor= TakeFirst()
    )
    address = scrapy.Field(
        output_processor= TakeFirst()
    )
    area_sqft = scrapy.Field(
        output_processor= TakeFirst()
    )
    zipcode = scrapy.Field(
        output_processor= TakeFirst()
    )
    broker_name = scrapy.Field(
        output_processor= TakeFirst()
    )
    status_text = scrapy.Field(
        output_processor= TakeFirst()
    )
    status_type = scrapy.Field(
        output_processor= TakeFirst()
    )       
    latitude  = scrapy.Field(
        output_processor= TakeFirst()
    )       
    longitude = scrapy.Field(
        output_processor= TakeFirst()
    )       
    price  = scrapy.Field(
        output_processor= TakeFirst()
    )
    baths = scrapy.Field(
        output_processor= TakeFirst()
    )
    beds = scrapy.Field(
        output_processor= TakeFirst()        
    )

Here is my setting.py

BOT_NAME = 'zillow'

SPIDER_MODULES = ['zillow.spiders']
NEWSPIDER_MODULE = 'zillow.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) 
Chrome/105.0.0.0 Safari/537.36'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 10

here is my utils.py

from http.cookies import SimpleCookie
from urllib.parse import urlparse, parse_qs, urlencode
import json 


URL ='https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=%7B%22usersSearchTerm%22%3A%22Miami%2C%20FL%22%2C%22mapBounds%22%3A%7B%22west%22%3A-80.5178131866171%2C%22east%22%3A-80.07149360653898%2C%22south%22%3A25.63146189889158%2C%22north%22%3A25.934422885146812%7D%2C%22mapZoom%22%3A11%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A12700%2C%22regionType%22%3A6%7D%5D%2C%22isMapVisible%22%3Afalse%2C%22filterState%22%3A%7B%22isAllHomes%22%3A%7B%22value%22%3Atrue%7D%2C%22sortSelection%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D&wants={%22cat1%22:[%22listResults%22],%22cat2%22:[%22total%22]}&requestId=7'

def cookie_parser():
    cookie_string = 'zguid=24|%24bd1cc7fb-3a96-4355-82ad-eb54b6582fc0; zgsession=1|c8c7aa18-a869-4fa5-acc2-0974c45c5397; _ga=GA1.2.2098963959.1662992109; _gid=GA1.2.646099340.1662992109; zjs_user_id=null; zg_anonymous_id=%2272ad9701-5772-4341-9553-2d5c6dbbb727%22; zjs_anonymous_id=%22bd1cc7fb-3a96-4355-82ad-eb54b6582fc0%22; _gcl_au=1.1.435090296.1662992115; KruxPixel=true; DoubleClickSession=true; pxcts=53406f29-32a5-11ed-9caa-487079506e4a; _pxvid=53403445-32a5-11ed-9caa-487079506e4a; _cs_c=0; _hp2_id.1215457233=%7B%22userId%22%3A%224388082776070223%22%2C%22pageviewId%22%3A%225521802898055122%22%2C%22sessionId%22%3A%22112719343168408%22%2C%22identity%22%3Anull%2C%22trackerVersion%22%3A%224.0%22%7D; KruxAddition=true; __pdst=222e1aef96394f48950db65677af5fb6; _cs_id=bc3f13b6-b504-a600-8f00-aa54819814a3.1662992134.1.1662992134.1662992134.1.1697156134010; utag_main=v_id:0183320dff7a0008f6511b590fb90506f00170670086e$_sn:1$_se:1$_ss:1$_st:1662993921725$ses_id:1662992121725%3Bexp-session$_pn:1%3Bexp-session$dcsyncran:1%3Bexp-session$tdsyncran:1%3Bexp-session$dc_visit:1$dc_event:1%3Bexp-session$dc_region:eu-central-1%3Bexp-session; _fbp=fb.1.1662992137787.1566635397; _pin_unauth=dWlkPU5EazROemRsTnpRdE5HVmxNUzAwT0RVeUxUbGpaV1F0WXpreFpUWmpaVEUyTjJRNA; __gads=ID=3a3aac85687f3a37-22e4e67cd5d50051:T=1662992221:S=ALNI_MZ4vIKVY--1rhaS-Dlmyq494eHCRA; __gpi=UID=00000accff902937:T=1662992221:RT=1662992221:S=ALNI_MbsEUqyyMfnfGRtmDdAAnNWlAQEtA; _clck=1ar1pmr|1|f4t|0; G_ENABLED_IDPS=google; _uetsid=5a9c690032a511ed945bf72202409c10; _uetvid=5a9cc21032a511ed9a012f92552709bc; _gat=1; AWSALB=ihmWxilxzEdWoOVOnUogTwdbVoGwbhOlUpMcMQ/Dg6X5L0UASrqwYngE9rS1l4TJD4lpP/qc1/uW+eAPWsvyrESYOcPEdG+h2WLYuNjwZl83rVNkanY/kFlI1Wc1; AWSALBCORS=ihmWxilxzEdWoOVOnUogTwdbVoGwbhOlUpMcMQ/Dg6X5L0UASrqwYngE9rS1l4TJD4lpP/qc1/uW+eAPWsvyrESYOcPEdG+h2WLYuNjwZl83rVNkanY/kFlI1Wc1; JSESSIONID=F4D9C55C95F31C49899E6A8955CC3AF7; search=6|1665600742919%7Crect%3D25.934422885146812%252C-80.07149360653898%252C25.63146189889158%252C-80.5178131866171%26rid%3D12700%26disp%3Dmap%26mdm%3Dauto%26p%3D2%26z%3D1%26fs%3D1%26fr%3D0%26mmm%3D0%26rs%3D0%26ah%3D0%26singlestory%3D0%26housing-connector%3D0%26abo%3D0%26garage%3D0%26pool%3D0%26ac%3D0%26waterfront%3D0%26finished%3D0%26unfinished%3D0%26cityview%3D0%26mountainview%3D0%26parkview%3D0%26waterview%3D0%26hoadata%3D1%26zillow-owned%3D0%263dhome%3D0%26featuredMultiFamilyBuilding%3D0%09%0912700%09%09%09%09%09%09' 
    cookie = SimpleCookie()
    cookie.load(cookie_string)
    
    cookies = {}
    for key, morsel in cookie.items():
        cookies[key] = morsel.value
    
    return cookies

def parse_new_url(url, page_number):
    url_parsed = urlparse(url)
    query_string = parse_qs(url_parsed.query)
    search_query_state = json.loads(query_string.get('searchQueryState')[0])
    search_query_state['pagination'] = {"currentPage": page_number}
    query_string.get('searchQueryState')[0] = search_query_state
    encoded_qs = urlencode(query_string, doseq=1)
    new_url = f"https://www.zillow.com/search/GetSearchPageState.htm?{encoded_qs}"
    print(new_url)
    
parse_new_url(URL, 3)
0 Answers
Related