How skip repeated lines?

user_13907533 0
Viewed 50

how do I make the section described unable to test repeated links? I tried to do it by comparison, I couldn't, the script was very slow.

import re
from bs4 import BeautifulSoup
import requests
import urllib.request

r = requests.get( 'http://www.google.com' )
html = r.text
soup = BeautifulSoup( html , 'lxml' )
links = soup.find_all( 'a' , attrs={'href' : re.compile( r'^https?://' )} )
for i in links :
    href = i['href']

# Test Section
    req = requests.get( href )
    resp = req.status_code
    if resp is None or resp in [400 , 404 , 403 , 408 , 409 , 501 , 502 , 503] :
        print( resp + '=' + resp.reason + '===>' + href )
        with open( 'Document_ERROR.txt' , 'a' ) as arq :
           arq.write( href )
           arq.write( '\n' )
           arq.write( resp.reason )
           arq.close( )
    else :
       print( 'Response is {} ===> `{}'.format( resp , href ) )
       with open( 'Document_OK.txt' , 'a' ) as arq :
          arq.write( href )
          arq.write( '\n' )
          arq.close( )
beautifulsoup python-3.x python regex
1 Answers

If I understand you correctly, you want to skip the testing code, when you have already tested the link.

You can have a set named seen_links, that will hold all links tested so far:

import re
from bs4 import BeautifulSoup
import requests
import urllib.request


r = requests.get('http://www.google.com')
soup = BeautifulSoup(r.content, 'lxml')
links = soup.find_all('a',attrs={'href': re.compile( r'^https?://' )})


seen_links = set()  # <-- set that will hold all seen links so far

for i in links :
    href = i['href']

    # have we seen the link before?
    if href in seen_links:
        continue    # yes, continue the loop

    # no, add it to seen_links
    seen_links.add(href)

    req = requests.get( href )
    resp = req.status_code
    if resp is None or resp in [400 , 404 , 403 , 408 , 409 , 501 , 502 , 503]:
        print( resp + '=' + resp.reason + '===>' + href )
        with open( 'Document_ERROR.txt' , 'a' ) as arq :
            print(href, file=arq)
            print(resp.reason, file=arq)
    else :
        print( 'Response is {} ===> `{}'.format( resp , href ) )
        with open( 'Document_OK.txt' , 'a' ) as arq :
            print(href, file=arq)
user_100359850
Related