#!/usr/bin/env python3
import re
import requests
import click
from bs4 import BeautifulSoup
def get_html(url):
webpage = requests.get(url)
reqStatus = webpage.status_code
DecodeURL = (webpage.content.decode())
if reqStatus != 200:
print(url)
print(f'HTTP response code: {reqStatus}. Not 200. Exiting script.')
exit(1)
return DecodeURL
def get_links(url):
html = get_html(url)
CodeU = BeautifulSoup(html, 'html.parser')
all_links = CodeU.findAll('a')
checkurl = url
with open('crawl.txt', 'a') as wr:
for link in all_links:
wr.write(str(link.get('href')))
wr.write('\n')
with open('crawl.txt', 'r') as r:
pattern = re.findall(r'\b'+checkurl+r'.*', r.read())
with open('crawl.txt', 'w') as w:
for line in pattern:
checkline = line.rstrip()
removechar = checkline.rstrip('/')
if removechar != checkurl:
w.write(line)
w.write('\n')
with open('crawl.txt', 'r') as r:
lines = r.readlines()
newlines = []
for line in lines:
if line not in newlines and line != line+'/':
newlines.append(line)
with open('crawl.txt', 'w') as w:
for line in newlines:
w.write(line)
@click.command()
@click.option('--url', '-u', prompt='Web URL', help='URL of webpage to extract from.')
@click.option('--crawldepth', '-c', default=0, help='If links are found, how many to scan
through?')
def main(url, crawldepth):
get_html(url)
get_links(url)
if int(crawldepth) == 0:
exit(0)
with open('crawl.txt', 'r') as r:
n = 1
crawllist = r.readlines()
for line in crawllist:
URL = line
try:
iterations(URL, crawldepth)
except SystemExit:
pass
n = n + 1
if n > int(crawldepth):
exit(0)
def iterations(URL, crawldepth):
url = URL
get_html(url)
get_links(url)
if __name__=='__main__':
main()
My goal here is to check a webpage for all its links that match to the initial 'url' input by user. All of these links are then written to a file called 'crawl.txt'. I want to be able to go a certain depth equal to 'crawldepth'. For example, if crawldepth = 1, the functions 'get_html' and 'get_links' are called twice(once with 'main', once with 'iteraitons'), and the first line of crawl.txt is read as the new 'url'. Crawl.txt is ammended with more links. The 'error' I receive is built into 'get_html'; if the response code is NOT equal to 200, the url variable is printed, and the actual response code is printed as well. Then the code exits with 1. This is expected IF the URL isn't giving a code 200.
Here in lies the problem. Using the example above, I get a 404 code when 'crawldepth' is > 0 and running through a second time. BUT when python prints the url to terminal, it is the first line of crawl.txt. If I search that url manually i.e. entered into my webbrowser, the page loads up without any issues. I am baffled by this.
I expect the code response to be 200 since when I search it manually, thats what I get. But this clearly is not the case when running the code. I don't get any errors from python. It seems like it runs error free.
I am AWARE that there is an issue with how I am filtering the links in 'get_links'. The second run of code if crawldepth > 0 will often give me an empty crawl.txt. This might throw an error if int(crawldepth) > number of lines in crawl.txt. This is what I will be working on next.
I've tried the following:
enter the first line of crawl.txt when first running the script: it runs successfully.
placing 'print(url') before the exit code if statement. I get the FIRST url entered when setting the options via terminal, then the second url, which is actually the first line of crawl.txt. But this is where the script exits as explained above.
To summarize, if I run the code with crawldepth = 0, I get no issues. It's ONLY when crawldepth > 0, and after the first run through. So basically, if crawldepth is >0, the function 'iterations' gets called, then 'get_html' exits the code with printing the 'url' and the response code.
I'm running the code with './' in terminal. Everything is updated and upgraded.