Simple scraper with Scrapy API

user_6248379 0
Viewed 95

I am writing a scraper with Scrapy within a larger project, and I'm trying to keep it as minimal as possible (without create a whole scrapy project). This code downloads a single URL correctly:

import scrapy
from scrapy.crawler import CrawlerProcess

class WebsiteSpider(scrapy.Spider):
    """ 
    https://docs.scrapy.org/en/latest/
    """        
    custom_settings = {'DOWNLOAD_DELAY': 1, 'DEPTH_LIMIT': 3}
    name = 'my_website_scraper'
    
    def parse(self,response):
        html = response.body
        url = response.url
        # process page here


process = CrawlerProcess() 
process.crawl(WebsiteSpider, start_urls=['https://www.bbc.co.uk/'])
process.start()

How can I enrich this code to keep scraping the links found in the start URLs (with a maximum depth, for example of 3)?

scrapy
1 Answers

Try this.

from simplified_scrapy import Spider, SimplifiedDoc, SimplifiedMain


class WebsiteSpider(Spider):
    name = 'bbc.co.uk'
    allowed_domains = ['.bbc.co.uk']
    start_urls = ['https://www.bbc.co.uk/']

    # refresh_urls = True # For debug. If efresh_urls = True, start_urls will be crawled again.

    def extract(self, url, html, models, modelNames):
        doc = SimplifiedDoc(html)
        lstA = doc.listA(url=url["url"])  # Get link data for subsequent crawling
        data = [{"title": doc.title.text}]  # Get target data

        return {"Urls": lstA, "Data": data}  # Return data to framework


SimplifiedMain.startThread(WebsiteSpider())  # Start crawling
user_124558790
Related