Why is scrapy so slow when using proxy rotator?

Viewed 67

I'm new to scrapy and I'm trying to scrape 300 urls per spider for my project, but for testing purposes, I only scrape 2 urls. When I use proxy rotator it takes 120 secs to 500 secs to finish my script, but when I'm not using proxy it takes 2 to 20 secs to finish my script. Is there any solution that would improve the overall performance of my script? Also, I use proxy rotator from proxyrotator.com

I often get TimeoutError when using proxy rotator. Any advice or answer would be appreciated.

settings.py

#settings.py
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 0.25
RETRY_TIMES = 3
CONNECTION_TIMEOUT = 30
COOKIES_ENABLED = False
DEFAULT_REQUEST_HEADERS = {
    "Accept": "application/json, text/javascript, */*; q=0.01",
    "DNT": "1",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
}
CONCURRENT_REQUESTS = 100
CONCURRENT_REQUESTS_PER_DOMAIN = 100
AUTOTHROTTLE_ENABLED = False
DOWNLOAD_TIMEOUT = 30
DOWNLOADER_MIDDLEWARES = {
     'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
}

static_spider.py

class ArticleStaticSpider(scrapy.Spider):
    logfile("server.log", maxBytes=1e6, backupCount=3)
    name = "article_static"

    custom_settings = {
        'ITEM_PIPELINES': {'news_extractor.pipelines.StaticExtractorPipeline': 300},
        "FEEDS": {"articles.json": {"format": "json"}},
    }

    def __init__(self, urls=None):
        self.urls = urls

    def start_requests(self):
        for url in self.urls:
            proxy = get_proxy() # get request for proxy rotator
            ip = proxy['ip']
            port = proxy['port']
            meta_proxy = f"http://{ip}:{port}"
            headers = {
                "User-Agent": proxy['randomUserAgent']
            }
            meta = {
                "proxy": meta_proxy                }
            yield scrapy.Request(url, callback=self.parse,headers=headers, meta=meta, errback=self.errback_httpbin)

    def parse(self, response):                                                                                               
        src = StaticSource(response.url) # module for cleaning html tags
        text_format = src.text
        news = News(response.url, text_format) # module for extracting data
        data = news.generate_data()
        print(data)

scraper.py

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
import time

ulrs = [
    "http://www.nytimes.com/2021/02/28/technology/seniors-vaccines-technology.html",
    "http://www.nytimes.com/2021/02/28/us/schools-reopening-philadelphia-parents.html"
]

def main():
    process = CrawlerProcess(get_project_settings())
    process.crawl('article_static', urls=urls)
    process.crawl('article_static', urls=urls)
    process.start()


if __name__ == "__main__":
    t1 = time.perf_counter()
    main()
    t2 = time.perf_counter()
    print(f'Finished in {round(t2-t1, 2)} second(s).....')
0 Answers
Related