Source : https://docs.twisted.org/en/twisted-17.9.0/core/howto/time.html
My Code:
import scrapy
from scrapy.crawler import CrawlerRunner
from twisted.internet import task
from twisted.internet import reactor
class Livescores1(scrapy.Spider):
name = 'Total'
runner = CrawlerRunner(settings = {
"FEEDS": {
"Total.json": {"format": "json", "overwrite": True},
},
})
def start_requests(self):
yield scrapy.Request('https://www.livescores.com/football/turkey/super-lig/?tz=3&table=league-total')
def parse(self, response):
for total in response.css('td'):
yield{
'total': total.css('::text').get()
}
class Livescores2(scrapy.Spider):
name = 'Home'
runner = CrawlerRunner(settings = {
"FEEDS": {
"Home.json": {"format": "json", "overwrite": True},
},
})
def start_requests(self):
yield scrapy.Request('https://www.livescores.com/football/turkey/super-lig/?tz=3&table=league-home')
def parse(self, response):
for total in response.css('td'):
yield{
'total': total.css('::text').get()
}
class Livescores3(scrapy.Spider):
name = 'Away'
runner = CrawlerRunner(settings = {
"FEEDS": {
"Away.json": {"format": "json", "overwrite": True},
},
})
def start_requests(self):
yield scrapy.Request('https://www.livescores.com/football/turkey/super-lig/?tz=3&table=league-away')
def parse(self, response):
for total in response.css('td'):
yield{
'total': total.css('::text').get()
}
loopTimes = 99999999999999999999999999999
failInTheEnd = False
_loopCounter = 0
def runEverySecond():
"""
Called at ever loop interval.
"""
runner.crawl(Livescores1)
runner.crawl(Livescores2)
runner.crawl(Livescores3)
global _loopCounter
if _loopCounter < loopTimes:
_loopCounter += 1
print('A new second has passed.')
return
loop = task.LoopingCall(runEverySecond)
# Start looping every 1 second.
loopDeferred = loop.start(1.0)
reactor.run()
This code has an error
"runner" is not defined
All i want is to save crawling to different json files. For this I added a separate CrawlerRunner(settings) for each class. But I should add another runner = CrawlerRunner(settings) for runEverySecond function, otherwise the code does not work. If I add this (although in these settings I can give a single json file name) all classes obey this rule and don't save data to different files! This structure has a function that works repeatly. That has looping function for spiders.
Can I add different settings for all classes to CrawlerRunner(settings) which I create inside runEverySecond? Is this possible or should I try a different solution?
Could you help me to fix this?
Thank you very much