so I'm trying to run scrapy crawler through another script the crawler runs fine but it shows an error while running the crawler and I can't seem to understand why this error occurs. The Code is given below to the script
class run():
def PostCrawler(self):
os.chdir(r'E:\pythonProject\Redit\redit\redit\spiders')
l=['PTI','PMLN','PPP']
while True:
os.system(f'scrapy crawl Reddit -a query={l[0]}')
l.pop(0)
if len(l)==0:
break
c = run()
c.PostCrawler()
Here is the spider
import scrapy
from redit.redit.items import RedditscrapperItem
from selenium.webdriver.chrome.options import Options
from selenium.webdriver import Chrome**
class Redditspider(scrapy.Spider):
name = 'Reddit'
def __init__(self,query=''):
self.url=f"https://www.reddit.com/search/?q={query}"
def start_requests(self):
options = Options()
options.add_argument('--headless')
path = "C:\Program Files (x86)\chromedriver.exe"
driver = Chrome(executable_path=path, options=options)
driver.get(self.url)
sn = 10
for i in range(1, sn):
driver.execute_script(f"window.scrollTo(1,50000)")
# time.sleep(2)
file = open('Post.html', 'w', encoding="utf-8")
file.write(driver.page_source)
yield scrapy.Request(url='file:///E:/pythonProject/Redit/redit/redit/spiders/Post.html')
file.close()
driver.close()
def parse(self, response):
table = response.css('div.QBfRw7Rj8UkxybFpX-USO')
Data = table.css('div._1oQyIsiPHYt6nx7VOmd1sz')
Image = []
votes = []
Comments = []
Awards = []
date = []
Post = []
detail = []
for i in range(len(Data)):
val = Data[i].css('span._vaFo96phV6L5Hltvwcox::text').getall()
vot = val[0]
com = val[1]
award = val[2]
if (vot == None):
votes.append('None')
else:
votes.append(vot)
if (com == None):
Comments.append('None')
else:
Comments.append(com)
if (award == None):
Awards.append('None')
else:
Awards.append(award)
val = Data[i].css('div._1poyrkZ7g36PawDueRza-J')
post_date = val.css('span._2VF2J19pUIMSLJFky-7PEI::text').getall()
if (post_date == None):
date.append('None')
else:
date.append(post_date)
rec = Data[i].css('h3._eYtD2XCVieq6emjKBH3m::text').getall()
star = ""
rec = star.join(rec)
if (rec == None):
Post.append('None')
else:
Post.append(rec)
ref = Data[i].css('div._2MkcR85HDnYngvlVW2gMMa')
Details = ref.css('a::attr("href")').get()
if (Details == None):
detail.append('None')
else:
detail.append(Details)
img = Data[i].css('div._2MkcR85HDnYngvlVW2gMMa')
pic = img.css(
'div._2c1ElNxHftd8W_nZtcG9zf._33Pa96SGhFVpZeI6a7Y_Pl._2r9BZFotuBbLYnijAaskeZ::attr("style")').get()
if (pic == None):
Image.append('None')
else:
spos = pic.find('(')
epos = pic.find(')')
pic = pic[spos + 1:epos]
Image.append(pic)
item = RedditscrapperItem()
for i in range(len(date)):
item['Post_Date'] = date[i]
item['Posts'] = Post[i]
item['Detail'] = detail[i]
item['Img'] = Image[i]
item['Comments'] = Comments[i]
item['Awards'] = Awards[i]
item['Votes'] = votes[i]
yield item
and here is the error
Traceback (most recent call last):
File "C:\Users\Intag\New folder (2)\lib\runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\Intag\New folder (2)\lib\runpy.py", line 86, in _run_code
exec(code, run_globals)
File "C:\Users\Intag\.virtualenvs\Redit\Scripts\scrapy.exe\__main__.py", line 7, in <module>
File "C:\Users\Intag\.virtualenvs\Redit\lib\site-packages\scrapy\cmdline.py", line 123, in execute
settings = get_project_settings()
File "C:\Users\Intag\.virtualenvs\Redit\lib\site-packages\scrapy\utils\project.py", line 68, in get_project_settings
settings.setmodule(settings_module_path, priority='project')
File "C:\Users\Intag\.virtualenvs\Redit\lib\site-packages\scrapy\settings\__init__.py", line 287, in setmodule
module = import_module(module)
File "C:\Users\Intag\New folder (2)\lib\importlib\__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
File "<frozen importlib._bootstrap>", line 1004, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'redit.settings'
So if anyone has passed through this issue kindly let me know