how do you optimize your setting.py in order to avoid being blocked by websites? in Scrapy
what is the best way to optimize it ?
I just know few methods want feedback from experts
if this is your setting.py file
BOT_NAME = 'xyz'
SPIDER_MODULES = ['xyz.spiders']
NEWSPIDER_MODULE = 'xyz.spiders'
#ROBOTSTXT_OBEY = True
#CONCURRENT_REQUESTS = 32
# USER_AGENT =
DOWNLOAD_DELAY = 10
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
#COOKIES_ENABLED = False
#TELNETCONSOLE_ENABLED = False
# DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
# }
# SPIDER_MIDDLEWARES = {
# 'millions.middlewares.MillionsSpiderMiddleware': 543,
# }
# Enable or disable downloader middlewares
# DOWNLOADER_MIDDLEWARES = {
# 'millions.middlewares.MillionsDownloaderMiddleware': 543,
# }
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
# }
# ITEM_PIPELINES = {
# 'millions.pipelines.MillionsPipeline': 300,
# }
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
#AUTOTHROTTLE_MAX_DELAY = 60
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
#AUTOTHROTTLE_DEBUG = False
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'