AttributeError: 'builtin_function_or_method' object has no attribute 'decode'

sherlock 0
Viewed 28

I am trying to scrape emails from a website and when I run the code I am getting an error and I don't really understand the error

Traceback (most recent call last):
  File "Email_Scrapper.py", line 37, in <module>
    parts = urllib.parse.urlsplit(url)
  File "C:\Users\HP\AppData\Local\Programs\Python\Python38-32\lib\urllib\parse.py", line 423, in urlsplit
    url, scheme, _coerce_result = _coerce_args(url, scheme)
  File "C:\Users\HP\AppData\Local\Programs\Python\Python38-32\lib\urllib\parse.py", line 124, in _coerce_args
    return _decode_args(args) + (_encode_result,)
  File "C:\Users\HP\AppData\Local\Programs\Python\Python38-32\lib\urllib\parse.py", line 108, in _decode_args
    return tuple(x.decode(encoding, errors) if x else '' for x in args)
  File "C:\Users\HP\AppData\Local\Programs\Python\Python38-32\lib\urllib\parse.py", line 108, in <genexpr>
    return tuple(x.decode(encoding, errors) if x else '' for x in args)
AttributeError: 'builtin_function_or_method' object has no attribute 'decode'

this is my code:

scraped_url = set()
emails = set() 

#ici on fait un compte jusqu'a ce que 
# les emails recuperer atteignent 20 et on sort de la boucle
count = 0
try:
    while len(urls):
        count += 1
        if count == 20:
            break
        url = urls.popleft 
        scraped_url.add(url)

        parts = urllib.parse.urlsplit(url)
        base_url = '{0.scheme}://{0.netloc}'.format(parts)
        
        path = url[:url.rfind('/')+1] if '/' in parts.path else url

        print('[%d] Processing %s' % count(count, url))
        try:
           response = requests.get()
        except(requests.exceptions.MissingSchema, requests.exceptions.ConnectionError):
            continue

        new_emails = set(re.findall(r'[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+',response.text, re.I))
        soup = BeautifulSoup(response.text, features="lxml")

        for anchor in soup.find_all("a"):
            link = anchor.attrs['href'] if 'href' in anchor.attrs else ''
            if link.startswith('/'):
                link = base_url + link
            elif not link.startswith('http'):
                link = path + link 
            if not link in urls and not link in scraped_url:
                urls.append(link)
except KeyboardInterrupt:
    print('[-]closing')

for mails in emails:
    print(mails)
python
1 Answers

popleft is a method, and not an attribute, so you need to call it like so:

scraped_url = set()
emails = set() 

#ici on fait un compte jusqu'a ce que 
# les emails recuperer atteignent 20 et on sort de la boucle
count = 0
try:
    while len(urls):
        count += 1
        if count == 20:
            break
        url = urls.popleft() # note the change here
        scraped_url.add(url)

        parts = urllib.parse.urlsplit(url)
        base_url = '{0.scheme}://{0.netloc}'.format(parts)
        
        path = url[:url.rfind('/')+1] if '/' in parts.path else url

        print('[%d] Processing %s' % count(count, url))
        try:
           response = requests.get()
        except(requests.exceptions.MissingSchema, requests.exceptions.ConnectionError):
            continue

        new_emails = set(re.findall(r'[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+',response.text, re.I))
        soup = BeautifulSoup(response.text, features="lxml")

        for anchor in soup.find_all("a"):
            link = anchor.attrs['href'] if 'href' in anchor.attrs else ''
            if link.startswith('/'):
                link = base_url + link
            elif not link.startswith('http'):
                link = path + link 
            if not link in urls and not link in scraped_url:
                urls.append(link)
except KeyboardInterrupt:
    print('[-]closing')

for mails in emails:
    print(mails)
Yuval.R0
Related