how to scrape the other span with same class

Question

im using beautifulSoup4, this is my code:

def extract(page):
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
r = requests.get(url)
soup = BeautifulSoup(r.content,'html.parser')
return soup

def transform(soup):
divs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
for items in divs:
location = items.find('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()

salary = items.find_next_sibling('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()

both span have the same class, but when i scrapped, both results were same.

user_15430200 · Answer

get the all spans by find_all and then remove the items that they have less than 2 spans. then get location and salary from the spans list

import requests
from bs4 import BeautifulSoup

def extract(page):
    url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
    r = requests.get(url)
    soup = BeautifulSoup(r.content,'html.parser')
    # transform
    jobs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
    data = []
    for job in jobs[:100]:
        items = job.find_all('span', attrs = {'class' : 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'})
        if len(items) < 2:
            continue
        location = items[0].text.strip()
        salary = items[1].text.strip()

        data.append([location, salary])
    
    return data
        
data = extract(1)
print(data)

user_14460824 · Answer

Try to avoid selecting by dynamic parts in HTML such as classes - While you still figured out the sibling relation you could go with find_next_sibling('span').

Following will select only by tags and also checks if there is a salary or not to avoid errors but scrape the whole job, while display salary None:

for item in soup.select('article'):
    data.append({
        'title': item.h1.text,
        'location': item.h1.parent.next_sibling.text,
        'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
    })

Example

import requests
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/1/'
r = requests.get(url)
soup = BeautifulSoup(r.content, 'html.parser')
data=[]
for item in soup.select('article'):
    data.append({
        'title': item.h1.text,
        'location': item.h1.parent.next_sibling.text,
        'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
    })
data

user_6593287 · Answer

Using dynamic classes is not the best idea. The site uses api for job search as well as graphql.

import requests
import json
import pandas as pd


def foo(page, keyword):
    url = "https://xapi.supercharge-srp.co/job-search/graphql?country=my&isSmartSearch=true"
    payload = json.dumps({
      "query": "query getJobs($country: String, $locale: String, $keyword: String, $createdAt: String, $jobFunctions: [Int], $categories: [String], $locations: [Int], $careerLevels: [Int], $minSalary: Int, $maxSalary: Int, $salaryType: Int, $candidateSalary: Int, $candidateSalaryCurrency: String, $datePosted: Int, $jobTypes: [Int], $workTypes: [String], $industries: [Int], $page: Int, $pageSize: Int, $companyId: String, $advertiserId: String, $userAgent: String, $accNums: Int, $subAccount: Int, $minEdu: Int, $maxEdu: Int, $edus: [Int], $minExp: Int, $maxExp: Int, $seo: String, $searchFields: String, $candidateId: ID, $isDesktop: Boolean, $isCompanySearch: Boolean, $sort: String, $sVi: String, $duplicates: String, $flight: String, $solVisitorId: String) {
  jobs(
    country: $country
    locale: $locale
    keyword: $keyword
    createdAt: $createdAt
    jobFunctions: $jobFunctions
    categories: $categories
    locations: $locations
    careerLevels: $careerLevels
    minSalary: $minSalary
    maxSalary: $maxSalary
    salaryType: $salaryType
    candidateSalary: $candidateSalary
    candidateSalaryCurrency: $candidateSalaryCurrency
    datePosted: $datePosted
    jobTypes: $jobTypes
    workTypes: $workTypes
    industries: $industries
    page: $page
    pageSize: $pageSize
    companyId: $companyId
    advertiserId: $advertiserId
    userAgent: $userAgent
    accNums: $accNums
    subAccount: $subAccount
    minEdu: $minEdu
    edus: $edus
    maxEdu: $maxEdu
    minExp: $minExp
    maxExp: $maxExp
    seo: $seo
    searchFields: $searchFields
    candidateId: $candidateId
    isDesktop: $isDesktop
    isCompanySearch: $isCompanySearch
    sort: $sort
    sVi: $sVi
    duplicates: $duplicates
    flight: $flight
    solVisitorId: $solVisitorId
  ) {
    total
    totalJobs
    relatedSearchKeywords {
      keywords
      type
      totalJobs
    }
    solMetadata
    suggestedEmployer {
      name
      totalJobs
    }
    queryParameters {
      key
      searchFields
      pageSize
    }
    experiments {
      flight
    }
    jobs {
      id
      adType
      sourceCountryCode
      isStandout
      companyMeta {
        id
        advertiserId
        isPrivate
        name
        logoUrl
        slug
      }
      jobTitle
      jobUrl
      jobTitleSlug
      description
      employmentTypes {
        code
        name
      }
      sellingPoints
      locations {
        code
        name
        slug
        children {
          code
          name
          slug
        }
      }
      categories {
        code
        name
        children {
          code
          name
        }
      }
      postingDuration
      postedAt
      salaryRange {
        currency
        max
        min
        period
        term
      }
      salaryVisible
      bannerUrl
      isClassified
      solMetadata
    }
  }
}
",
      "variables": {
        "keyword": keyword,
        "jobFunctions": [],
        "locations": [],
        "salaryType": 1,
        "jobTypes": [],
        "createdAt": None,
        "careerLevels": [],
        "page": page,
        "country": "my",
        "categories": [],
        "workTypes": [],
        "userAgent": "Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36",
        "industries": [],
        "locale": "en"
      }
    })
    headers = {
        'accept': '*/*',
        'content-type': 'application/json'
    }

    response = requests.request("POST", url, headers=headers, data=payload)
    salary = "Not specified"
    result = []
    for job in (response.json()['data']['jobs']['jobs']):
        locations = []
        if job['salaryVisible']:
            salary = f"{job['salaryRange']['min']}-{job['salaryRange']['max']} {job['salaryRange']['currency']}"
        for location in job['locations']:
            locations.append(location['name'])
        result.append([job['jobTitle'], job['description'], job['jobUrl'], salary, ", ".join(locations)])
    df = pd.DataFrame(result)
    df.columns = ['Title', 'Description', 'Link', 'Salary', 'Location']
    print(df.to_string())


foo(1, "personal assistant")

OUTPUT:

                                                                         Title                                                                                                                                                Description                                                                                                                                                                                                                                                   Link           Salary                Location
0                                                Personal Assistant/ Secretary        Job Description:GETO GLOBAL CONSTRUCTION TECH MALAYSIA SDN. BHD. is a subsidiary of a public listed company from China, specializing in Aluminum...                                                                                     https://www.jobstreet.com.my/en/job/personal-assistant-secretary-4986870?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=1&jobId=jobstreet-my-job-4986870    3000-6000 MYR        Shah Alam/Subang
1                                                          Executive Assistant      Responsibilities:  Responsible in managing day-to-day office related and administrative tasks. Organizing and attending meetings and ensuring well...                                                                                              https://www.jobstreet.com.my/en/job/executive-assistant-4986449?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=2&jobId=jobstreet-my-job-4986449    2000-3000 MYR          Johor - Others
2                                                                    Secretary        A good write-up Secretary's resume is a Must with experience and can handle Managing Director company matters only. Fresh grads or many years in...                                                                                                        https://www.jobstreet.com.my/en/job/secretary-4986714?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=3&jobId=jobstreet-my-job-4986714    2000-3000 MYR           Petaling Jaya
3                                                 Secretary/Personal assistant  We are looking for Secretary/Personal assistant in our company for: Organising and maintaining office systems Screening phone calls, emails, enquiries...                                                                                     https://www.jobstreet.com.my/en/job/secretary-personal-assistant-4983131?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=4&jobId=jobstreet-my-job-4983131    2000-3000 MYR            Kuala Lumpur
4                                         Personal Assistant to Country Leader    Do you want to make a positive impact on the world? Sustainable development is at the heart of everything we do.Seventy-five years ago, we created a...                                                                             https://www.jobstreet.com.my/en/job/personal-assistant-to-country-leader-4986906?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=5&jobId=jobstreet-my-job-4986906    2000-3000 MYR           Petaling Jaya
5                                        Personal Assistant to Senior Director  Requirement: 5 – 8 years in handling PA end to end tasks for a Director/CEO/Managing Director Highly proficient in MS Office software Strong knowledge...                                                                            https://www.jobstreet.com.my/en/job/personal-assistant-to-senior-director-4986792?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=6&jobId=jobstreet-my-job-4986792    2000-3000 MYR        Shah Alam/Subang
6                                                           Personal Assistant     Report to top management; Attend to all aspects of PA and corporate work; Draft and manage correspondences, email, and memos; Schedule and organize...                                                                                               https://www.jobstreet.com.my/en/job/personal-assistant-4987086?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=7&jobId=jobstreet-my-job-4987086    2000-4000 MYR                 Puchong
7                                   Personal Assistant (Secretary) to Director       Responsibilities:  Provide support to Director and act as the primary liaison with various divisions/department or any newly set-up branch within...                                                                         https://www.jobstreet.com.my/en/job/personal-assistant-secretary-to-director-4980388?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=8&jobId=jobstreet-my-job-4980388    2000-4000 MYR  Kuala Lumpur, Selangor
8                                                           Personal Assistant          Personal Assistants perform a variety of administrative tasks for an individual.Here are some common responsibilities of a Personal Assistant:...                                                                                               https://www.jobstreet.com.my/en/job/personal-assistant-4986053?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=9&jobId=jobstreet-my-job-4986053    2000-4000 MYR             Subang Jaya
9                          Special Officer to CEO - Public Listed Conglomerate  Reporting and supporting the CEO in his daily work in wide variety of area - including preparation for meetings, development of presentation materials...                                                               https://www.jobstreet.com.my/en/job/special-officer-to-ceo-public-listed-conglomerate-4986807?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24§ionRank=10&jobId=jobstreet-my-job-4986807  15000-20000 MYR            Kuala Lumpur
...

how to scrape the other span with same class

3 Answers

Example