python BeaitifulSoup problem'NoneType' object has no attribute 'find_all'

userMO7O 0
Viewed 45

Hi I'm new to web scraping and I faced this problem that no other answer was good for it

from bs4 import BeautifulSoup
import requests
import csv
from itertools import zip_longest
import re

jobTitle = []
companyName = []
location = []
employmentReq = []
exp = []
startTime = []
links = []
salary = []
experience = []
pageNum = 0
result = requests.get(f"https://wuzzuf.net/search/jobs/?a=hpb&q=web&start={pageNum}")

src = result.content

soup = BeautifulSoup(src, "html.parser")

jobTitles = soup.find_all("h2", {"class": "css-m604qf"})
companyNames = soup.find_all("a", {"class": "css-17s97q8"})
locations = soup.find_all("span", {"class": "css-5wys0k"})
employment = soup.find_all("div", {"class": "css-1lh32fc"})

for i in range(len(jobTitles)):
    jobTitle.append(jobTitles[i].text)
    links.append("https://wuzzuf.net" + jobTitles[i].find("a").attrs['href'])
    companyName.append(companyNames[i].text)
    location.append(locations[i].text)
    employmentReq.append(employment[i].text)
    years = re.sub(r'[^0-9-]', '', soup.find_all("div", {"class": "css-y4udm8"})[i].find_all("div")[1].find_all("span")[0].text)
    experience.append(years)

for link in links:
    result = requests.get(link)
    src = result.content
    soup = BeautifulSoup(src, "html.parser")
    a = soup.find("main")
    print(a)
    b=a.find("section",{"class":"css-3kx5e2"})
    print(b)
    c =b.find_all("div")
    print(c)
    d =c.find_all("span")
    print(d)

    #if sal_span != "Confidential":
    #    salaries = re.sub(r'E.*$', '', sal_span)
    #else:
    #    salaries = sal_span
    #salary.append(salaries)

#print(salary)
#fileList = [jobTitle, companyName, location, employmentReq, links, experience, salary]
#exported = zip_longest(*fileList)
#with open("D:\T1t4nProject\python\wuzzuf.csv", "w") as excel_sheet:
#    wr = csv.writer(excel_sheet)
#    wr.writerow(["job title", "company name", "location", "full or part time", "links", "Years of experience", "Salary"])
#    wr.writerows(exported)

the error that i get is this

    sal_span =soup.find("main").find("section",{"class":"css-3kx5e2"}).find_all("div")[3].find_all("span")[1].find("span").text
AttributeError: 'NoneType' object has no attribute 'find_all'

i don't know what is the problem i used the same method on the variable years and on such errors

beware that the variable years is taking from another page

compiler-errors python-re beautifulsoup web-scraping python
1 Answers

Try to avoid dynamic class names, if this is difficult, put them in a separate variable to quickly return the parser to work. I made a simple example that will get the fields you need from the first 3 pages.

import pandas as pd
import requests
from bs4 import BeautifulSoup


def get_jobs(page_count):
    jobs = []
    for page in range(page_count):
        url = f'https://wuzzuf.net/search/jobs/?a=hpb&q=web&start={page}'
        response = requests.get(url)
        soup = BeautifulSoup(response.text, 'lxml')
        job_block_class = 'css-pkv5jc'
        job_type_class = 'css-n2jc4m'
        for job in soup.find_all('div', class_=job_block_class):
            jobs.append({
                'Link': 'https://wuzzuf.net' + job.find('h2').find('a').get('href'),
                'Title': job.find('h2').get_text(),
                'Company': job.find('h2').findNext('div').find('a').get_text().strip(' -'),
                'Location': job.find('h2').findNext('div').findNext('span').get_text(),
                'Type': ', '.join([jt.get_text() for jt in job.findAll('a', class_=job_type_class)]),
                'Experienced': job.find('div', attrs={'class': None}).findNext('span').get_text()[2:]
            })
    return jobs


df = pd.DataFrame(get_jobs(3))
# df.to_csv('filename.csv')
print(df.to_string())

OUTPUT:

    Link                                                   Title                              Company                         Location                                            Type        Experienced
0                                       https://wuzzuf.net/jobs/p/YbxjPYIASBeD-Web-Developer-WEBFLOW-ENGLISH-Giza-Egypt?o=1&l=sp&t=sj&a=web|search-v3|hpb                         Web Developer WEBFLOW (ENGLISH)                         Confidential                     Giza, Egypt                             Full Time, Part Time   4 - 5 Yrs of Exp
1                              https://wuzzuf.net/jobs/p/4JeWRMrh02cL-Web-Design-Instructor-Arabic-Localizer-Giza-Egypt?o=2&l=sp&t=sj&a=web|search-v3|hpb                                   Web Design Instructor                     Arabic Localizer        Mohandessin, Giza, Egypt                   Part Time, Freelance / Project   3 - 4 Yrs of Exp
2                                         https://wuzzuf.net/jobs/p/ymGucmZ1g9bu-Web-Developer-Flojics-Alexandria-Egypt?o=3&l=sp&t=sj&a=web|search-v3|hpb                                           Web Developer                              Flojics               Alexandria, Egypt                                        Full Time   0 - 1 Yrs of Exp
3                                         https://wuzzuf.net/jobs/p/BZgw5FvOYM9n-Web-Developer-Usual-Agency-Cairo-Egypt?o=4&l=sp&t=sj&a=web|search-v3|hpb                                           Web Developer                         Usual Agency        Heliopolis, Cairo, Egypt                                        Full Time   2 - 4 Yrs of Exp
4   https://wuzzuf.net/jobs/p/wy1BWQF9OItu-Junior-Web-Developer-Back-End---Alexandria-Arabic-Localizer-Alexandria-Egypt?o=5&l=sp&t=sj&a=web|search-v3|hpb            Junior Web Developer (Back-End) - Alexandria                     Arabic Localizer  San Stefano, Alexandria, Egypt                                        Full Time   0 - 3 Yrs of Exp
5                                    https://wuzzuf.net/jobs/p/PaP6bc0KD9NB-Front-End-Web-Developer-ReactJS-Cairo-Egypt?o=6&l=sp&t=sj&a=web|search-v3|hpb                      Front-End Web Developer (React.JS)                         Confidential         New Cairo, Cairo, Egypt                                        Full Time      2+ Yrs of Exp
6                    https://wuzzuf.net/jobs/p/VnU8hMLxuERr-Translator-Web-Content-Writer-INTERNET-SOLUTIONS-Giza-Egypt?o=7&l=sp&t=sj&a=web|search-v3|hpb                           Translator-Web Content Writer                   INTERNET SOLUTIONS       Sheikh Zayed, Giza, Egypt                                        Full Time   2 - 4 Yrs of Exp
7                                                https://wuzzuf.net/jobs/p/z7kkIx8RUP5V-Web-Programmer-Alexandria-Egypt?o=8&l=sp&t=sj&a=web|search-v3|hpb                                          Web Programmer                         Confidential    Cleopatra, Alexandria, Egypt                                        Full Time   3 - 7 Yrs of Exp
8                               https://wuzzuf.net/jobs/p/W43NaUFQdk4H-UI-Web-Developer-Perfect-Presentation-Giza-Egypt?o=9&l=sp&t=sj&a=web|search-v3|hpb                                        UI Web Developer                 Perfect Presentation     6th of October, Giza, Egypt                                        Full Time      5+ Yrs of Exp
9           https://wuzzuf.net/jobs/p/cuIQdXTtU7Kv-Senior-NetAngular-Web-Developer-Ultimate-Solutions-Egypt-Giza-Egypt?o=10&l=sp&t=sj&a=web|search-v3|hpb                       Senior .Net/Angular Web Developer             Ultimate Solutions Egypt     6th of October, Giza, Egypt                                        Full Time      4+ Yrs of Exp
10                             https://wuzzuf.net/jobs/p/2CHzjc7E8zz5-Senior-Java-Web-Developer-FlairsTech-Cairo-Egypt?o=11&l=sp&t=sj&a=web|search-v3|hpb                               Senior Java Web Developer                           FlairsTech             Maadi, Cairo, Egypt                                        Full Time  3 - 10 Yrs of Exp
11                                 https://wuzzuf.net/jobs/p/X3gNnVZM3Y3u-UIUX-Web-Designer-Nile-Creations-Cairo-Egypt?o=12&l=sp&t=sj&a=web|search-v3|hpb                                      UI/UX Web Designer                       Nile Creations        Heliopolis, Cairo, Egypt                                        Full Time      3+ Yrs of Exp
12                                          https://wuzzuf.net/jobs/p/61mX9Z4mLmOc-Web-Developer-Etisal-EG-Cairo-Egypt?o=13&l=sp&t=sj&a=web|search-v3|hpb                                           Web Developer                            Etisal EG         New Cairo, Cairo, Egypt                                        Full Time      3+ Yrs of Exp
13                                        https://wuzzuf.net/jobs/p/tePwhoJIBK2V-Senior-Web-Developer-Turbo-Giza-Egypt?o=14&l=sp&t=sj&a=web|search-v3|hpb                                    Senior Web Developer                                Turbo                     Giza, Egypt                                        Full Time   3 - 6 Yrs of Exp
14           https://wuzzuf.net/jobs/p/q6WxlzYhj8VX-Full-Stack-Web-Developer--E-Commerce-Technical-Manager-Cairo-Egypt?o=15&l=sp&t=sj&a=web|search-v3|hpb  Full Stack Web Developer -E Commerce Technical Manager                         Confidential         Nasr City, Cairo, Egypt                                        Full Time   5 - 8 Yrs of Exp
15                             https://wuzzuf.net/jobs/p/ezagCZAHRsEB-Senior-Web-Developer-Madar-Soft-Alexandria-Egypt?o=16&l=sp&t=sj&a=web|search-v3|hpb                                    Senior Web Developer                           Madar Soft      Fleming, Alexandria, Egypt                                        Full Time      3+ Yrs of Exp
16                               https://wuzzuf.net/jobs/p/FZfxQVvPd354-Web-Developer-Qualify-For-Training-Cairo-Egypt?o=17&l=sp&t=sj&a=web|search-v3|hpb                                           Web Developer                 Qualify For Training                    Cairo, Egypt   Part Time, Freelance / Project, Work From Home   3 - 7 Yrs of Exp
17                                                 https://wuzzuf.net/jobs/p/j2FQjOw3JOyN-PHP-Web-Developer-Giza-Egypt?o=18&l=sp&t=sj&a=web|search-v3|hpb                                       PHP Web Developer                         Confidential              Dokki, Giza, Egypt                                        Full Time      2+ Yrs of Exp
18                                   https://wuzzuf.net/jobs/p/dO4oaKV5oTNn-Senior-Web-Developer-Naba-Soft-Cairo-Egypt?o=19&l=sp&t=sj&a=web|search-v3|hpb                                    Senior Web Developer                            Naba Soft         Nasr City, Cairo, Egypt                                        Full Time  5 - 20 Yrs of Exp
19                  https://wuzzuf.net/jobs/p/OGbcaevuFODl-Web-Developer-Akhnaton-for-Trading-Distributing-Cairo-Egypt?o=20&l=sp&t=sj&a=web|search-v3|hpb                                           Web Developer  Akhnaton for Trading & Distributing          Downtown, Cairo, Egypt                                        Full Time      3+ Yrs of Exp
20    https://wuzzuf.net/jobs/p/mRehCpOufJWl-Junior-ASPNET-Web-Developer-Dimensions-Information-Technology-Cairo-Egypt?o=21&l=sp&t=sj&a=web|search-v3|hpb                            Junior ASP.NET Web Developer    Dimensions Information Technology                    Cairo, Egypt                                        Full Time   4 - 6 Yrs of Exp
21   https://wuzzuf.net/jobs/p/fgFp51ORpe21-Web-Designer-Front-End-Developer---WordPress-INTERNET-SOLUTIONS-Giza-Egypt?o=22&l=sp&t=sj&a=web|search-v3|hpb          Web Designer & Front End Developer - WordPress                   INTERNET SOLUTIONS       Sheikh Zayed, Giza, Egypt                                        Full Time   2 - 5 Yrs of Exp
22                      https://wuzzuf.net/jobs/p/HmjaKL0NYT8c-Web-Developer-Part-time-Fixed-term-contract-Cairo-Egypt?o=23&l=sp&t=sj&a=web|search-v3|hpb         Web Developer ( Part-time, Fixed term contract)                         Confidential        Heliopolis, Cairo, Egypt                        Part Time, Work From Home   5 - 7 Yrs of Exp
23                                          https://wuzzuf.net/jobs/p/hPvrjx9oe5CX-Web-Developer-justagain-Cairo-Egypt?o=24&l=sp&t=sj&a=web|search-v3|hpb                                           Web Developer                            justagain         New Cairo, Cairo, Egypt                                        Full Time   1 - 3 Yrs of Exp
24                                      https://wuzzuf.net/jobs/p/4CDCKYnVUNHt-Web-Development-Team-Leader-Cairo-Egypt?o=25&l=sp&t=sj&a=web|search-v3|hpb                             Web Development Team Leader                         Confidential                    Cairo, Egypt                        Full Time, Work From Home      7+ Yrs of Exp
25           https://wuzzuf.net/jobs/p/1b6C9mKaEYw5-Digital-Chat-Representative-Web-Chat-Seoudi-Supermarket-Giza-Egypt?o=26&l=sp&t=sj&a=web|search-v3|hpb                  Digital Chat Representative (Web Chat)                   Seoudi Supermarket       Sheikh Zayed, Giza, Egypt                                        Full Time   1 - 3 Yrs of Exp
26                     https://wuzzuf.net/jobs/p/BfFChnjzswPX-Web-Developer-PHP---Open-Source-Flojics-Alexandria-Egypt?o=27&l=sp&t=sj&a=web|search-v3|hpb                       Web Developer (PHP - Open-Source)                              Flojics               Alexandria, Egypt                        Full Time, Work From Home      5+ Yrs of Exp
27                                     https://wuzzuf.net/jobs/p/eVAPCCiIKpxZ-Senior-Web-Developer-Flojics-Cairo-Egypt?o=28&l=sp&t=sj&a=web|search-v3|hpb                                    Senior Web Developer                              Flojics                    Cairo, Egypt                        Full Time, Work From Home  5 - 20 Yrs of Exp
28                         https://wuzzuf.net/jobs/p/7aFp3iCHa5kz-Talented-Web-Designer-Egypt-Yellow-Pages-Cairo-Egypt?o=29&l=sp&t=sj&a=web|search-v3|hpb                                   Talented Web Designer                   Egypt Yellow Pages             Maadi, Cairo, Egypt                                        Full Time   2 - 3 Yrs of Exp
29                                                 https://wuzzuf.net/jobs/p/VKHX8zMmaWEm-NET-Web-Developer-Giza-Egypt?o=30&l=sp&t=sj&a=web|search-v3|hpb                                      .NET Web Developer                         Confidential     6th of October, Giza, Egypt                                        Full Time   2 - 3 Yrs of Exp
30                  https://wuzzuf.net/jobs/p/AcqIdMveUs71-Creative-Web-Developer-ParamInfo-Dubai-United-Arab-Emirates?o=31&l=sp&t=sj&a=web|search-v3|hpb                                  Creative Web Developer                            ParamInfo     Dubai, United Arab Emirates                                        Full Time  2 - 10 Yrs of Exp
31       https://wuzzuf.net/jobs/p/EM8J2Kkh0ErO-Senior-PHP-Web-Developer-Arabia-for-Information-Technology-Cairo-Egypt?o=32&l=sp&t=sj&a=web|search-v3|hpb                                Senior PHP Web Developer    Arabia for Information Technology                    Cairo, Egypt                                        Full Time      5+ Yrs of Exp
32                                  https://wuzzuf.net/jobs/p/2kBsyMmlCKTU-Front-End-Web-Developer-Peerless-Giza-Egypt?o=33&l=sp&t=sj&a=web|search-v3|hpb                                 Front End Web Developer                             Peerless        Mohandessin, Giza, Egypt                                        Full Time   2 - 4 Yrs of Exp
33                               https://wuzzuf.net/jobs/p/M0fjpZUveVl8-Senior-Java-Developer-AllegianceMD-Cairo-Egypt?o=34&l=sp&t=sj&a=web|search-v3|hpb                                   Senior Java Developer                         AllegianceMD          Mokattam, Cairo, Egypt                                        Full Time   3 - 5 Yrs of Exp
34                                   https://wuzzuf.net/jobs/p/fILb5T14SNpB-Software-Engineer---Kuwait-RDI-Cairo-Egypt?o=35&l=sp&t=sj&a=web|search-v3|hpb                              Software Engineer - Kuwait                                  RDI                    Cairo, Egypt                                        Full Time      5+ Yrs of Exp
35                               https://wuzzuf.net/jobs/p/wGTJHY1ohsUC-Senior-Front-End-Developer-Ejadtech-Giza-Egypt?o=36&l=sp&t=sj&a=web|search-v3|hpb                              Senior Front End Developer                             Ejadtech              Dokki, Giza, Egypt                                        Full Time      4+ Yrs of Exp
36   https://wuzzuf.net/jobs/p/PBZM2s2pFVKn-Senior-NET-Developer-ASPNET-WebForms---Hybrid-GET-Group--Egypt-Cairo-Egypt?o=37&l=sp&t=sj&a=web|search-v3|hpb     Senior .NET Developer ( ASP.NET WebForms ) - Hybrid                     GET Group- Egypt        Heliopolis, Cairo, Egypt                                        Full Time      3+ Yrs of Exp
37                                     https://wuzzuf.net/jobs/p/UP8Zyh8D83Py-Senior-PHP-Developer-Riyadh-Saudi-Arabia?o=38&l=sp&t=sj&a=web|search-v3|hpb                                    Senior PHP Developer                         Confidential            Riyadh, Saudi Arabia                                        Full Time   2 - 8 Yrs of Exp
38                                       https://wuzzuf.net/jobs/p/fBOHU9BwWhJG-Java-Team-Lead-Ejada-Cairo-Cairo-Egypt?o=39&l=sp&t=sj&a=web|search-v3|hpb                                          Java Team Lead                        Ejada (Cairo)                    Cairo, Egypt                                        Full Time      8+ Yrs of Exp
39                                https://wuzzuf.net/jobs/p/a17VcCnAyamF-Senior-Java-Developer-Ejada-Cairo-Cairo-Egypt?o=40&l=sp&t=sj&a=web|search-v3|hpb                                   Senior Java Developer                        Ejada (Cairo)        Heliopolis, Cairo, Egypt                                        Full Time   4 - 6 Yrs of Exp
40                          https://wuzzuf.net/jobs/p/VLWdhbaQDPJ0-Development-Team-Leader-egabi-solutions-Cairo-Egypt?o=41&l=sp&t=sj&a=web|search-v3|hpb                                 Development Team Leader                      egabi solutions                    Cairo, Egypt                                        Full Time  6 - 10 Yrs of Exp
41                             https://wuzzuf.net/jobs/p/vYqyoutFh9m4-Senior-Graphic-Designer---UIUXWebsite-Giza-Egypt?o=42&l=sp&t=sj&a=web|search-v3|hpb                 Senior Graphic Designer - UI/UX/Website                         Confidential              Dokki, Giza, Egypt                                        Full Time  3 - 20 Yrs of Exp
42                  https://wuzzuf.net/jobs/p/BZ6A5Pej3BJr-Front-End-Developer-Tam-Development-LLC-Riyadh-Saudi-Arabia?o=43&l=sp&t=sj&a=web|search-v3|hpb                                     Front End Developer                  Tam Development LLC            Riyadh, Saudi Arabia                        Full Time, Work From Home      3+ Yrs of Exp
43                               https://wuzzuf.net/jobs/p/Ganw296LvwQS-Senior-Front-End-Developer-Dafater-Cairo-Egypt?o=44&l=sp&t=sj&a=web|search-v3|hpb                              Senior Front End Developer                              Dafater         Nasr City, Cairo, Egypt                        Full Time, Work From Home      3+ Yrs of Exp
44                                    https://wuzzuf.net/jobs/p/CsHvaWU326dE-Senior-Odoo-Developer-aliaict-Cairo-Egypt?o=45&l=sp&t=sj&a=web|search-v3|hpb                                   Senior Odoo Developer                              aliaict         Nasr City, Cairo, Egypt                                        Full Time   3 - 7 Yrs of Exp
userXeFr0
Related