I have created a scrapy spider that has to crawl the whole webpage and extract the urls. now I have to remove the social media URL for that I want to make a list of the URLs, but somehow it's not working. when I try to append each URL in list it just continuously make list of urls.
import re
import scrapy
all_urls = []
class QuotesSpider(scrapy.Spider):
name = 'quotes'
start_urls = [
'https://www.wireshark.org/docs/dfref/i/ip.html',
]
def parse(self, response):
page= response.url.split("/")[-2]
filename='quotes-%s.html' % page
with open(filename, 'wb') as f:
f.write(response.body)
for r in response.css('a'):
url = r.css('::attr(href)').get()
print('all the urls are here', url)
for i in url:
all_urls.append(url)
print(all_urls)
Edit:
I have removed the social media links and other info from the list now I want to scrape each link further kindly see if my approach is fine.
import requests
import scrapy
all_urls = []
remove = ["twitter"]
class QuotesSpider(scrapy.Spider):
name = 'quotess'
start_urls = [
'https://www.wireshark.org/docs/dfref/i/ip.html',
]
def parse(self, response):
for url in response.css('a::attr(href)').getall():
if url.startswith("http"):
if 'twitter' not in url:
all_urls.append(url)
print(all_urls)
print(all_urls[0])
for i in all_urls:
response = requests.get(i)
# print(response)
yield response