Get Scraped data using scrapy to a variable instead of file/database

user_17939220 0
Viewed 64

I am trying to run scrapy as a python script and want to process the data scraped instead of storing in a file/database. The code looks like

import scrapy
import scrapy.crawler as crawler
from scrapy.utils.log import configure_logging
from multiprocessing import Process, Queue
from twisted.internet import reactor

# spider
class QuotesSpider(scrapy.Spider):
    name = "quotes"
    start_urls = ['http://quotes.toscrape.com/tag/humor/']

    def parse(self, response):
        yield {"html_data": response.text}


# the wrapper to make it run more times
def run_spider(spider):
    def f(q):
        try:
            runner = crawler.CrawlerRunner()
            deferred = runner.crawl(spider)
            deferred.addBoth(lambda _: reactor.stop())
            reactor.run()
            q.put(None)
        except Exception as e:
            q.put(e)

    q = Queue()
    p = Process(target=f, args=(q,))
    p.start()
    result = q.get()
    p.join()

    if result is not None:
        raise result

configure_logging()

x = run_spider(QuotesSpider)

I want to run the spider when it is called. How this can be done

scrapy python
1 Answers

As I understand, you want to crawl the links using scrapy, and instead of storing these links on a file, you want to return them to the script.

Using your method

import scrapy
import scrapy.crawler as crawler
from scrapy.utils.log import configure_logging
from multiprocessing import Process, Queue
from twisted.internet import reactor

all_html_data = []


# spider
class QuotesSpider(scrapy.Spider):
    name = "quotes"
    start_urls = ['http://quotes.toscrape.com/tag/humor/']
    html_data = []

    def parse(self, response):
        self.html_data.append(response.text)

    def close(self, reason):
        global all_html_data
        all_html_data = self.html_data


# the wrapper to make it run more times
def run_spider(spider):
    def f(q):
        try:
            runner = crawler.CrawlerRunner()
            deferred = runner.crawl(spider)
            deferred.addBoth(lambda _: reactor.stop())
            reactor.run()
            q.put(None)
        except Exception as e:
            q.put(e)

    q = Queue()
    p = Process(target=f, args=(q,))
    p.start()
    result = q.get()
    p.join()

    if result:
        raise result


configure_logging()

run_spider(QuotesSpider)
data = all_html_data  # this is the data

But if you want to crawl the links and use the HTML response, I think it will be better if you use another library like Requests or use HtmlResponse from scrapy like

from scrapy.http import HtmlResponse
def parse(url):
    response = HtmlResponse(url=url)
    return response.text

data=parse('example.com')
user_151306850
Related