Can't get data from each of the classes named 'heading' in a page using selenium

Question

Hi I am new in data scraping. Here I am trying to scrape data from all classes which has the 'heading' attribute. But in my code it's only printing the first element even though I am using for loop for iterating.

Expected Output - Scraping data from all pages classes which has attribute 'heading'

Actual Output - Scraping data only from first element having class name as 'heading' and not even clicking on next button.

The site I am using for testing is here

from selenium import webdriver
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import pandas as pd
from openpyxl.workbook import Workbook


DRIVER_PATH = 'C:/Users/Aishwary/Downloads/chromedriver_win32/chromedriver'

driver = webdriver.Chrome(executable_path=DRIVER_PATH)

driver.get('https://www.fundoodata.com/citiesindustry/19/2/list-of-information-technology-(it)-companies-in-noida')

# get all classes which has heading as a class name 
company_names = driver.find_elements_by_class_name('heading')

# to store all companies names from heading class name
names_list = []

while True:

    try:
        for name in company_names: # iterate each name in all div classes named as heading
            text = name.text    # get text data from those elements
            names_list.append(text)
            print(text)
            # Click on next button to get data from next pages as well
            driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="main-container"]/div[2]/div[4]/div[2]/div[44]/div[1]/ul/li[7]/a'))))
            driver.find_element_by_xpath('//*[@id="main-container"]/div[2]/div[4]/div[2]/div[44]/div[1]/ul/li[7]/a').click()

    except (TimeoutException, WebDriverException) as e:
        print("Last page reached")
        break


driver.quit()

# Store those data in excel sheet
df = pd.DataFrame(names_list)
writer = pd.ExcelWriter('companies_names.xlsx', engine='xlsxwriter')
df.to_excel(writer, sheet_name='List')
writer.save()

user_10035985 · Answer

This script will get all business names from the page:

import requests
import pandas as pd
from bs4 import BeautifulSoup


url = 'https://www.fundoodata.com/citiesindustry/19/2/list-of-information-technology-(it)-companies-in-noida'

all_data = []
while True:
    print(url)

    soup = BeautifulSoup( requests.get(url).content, 'html.parser' )
    for h in soup.select('div.heading'):
        all_data.append({'Name' : h.text})
        print(h.text)

    next_page = soup.select_one('a:contains("Next")')
    if not next_page:
        break

    url = 'https://www.fundoodata.com' + next_page['href']

df = pd.DataFrame(all_data)
print(df)

df.to_csv('data.csv')

Prints:

                              Name
0                   BirlaSoft Ltd
1             HCL Infosystems Ltd
2            HCL Technologies Ltd
3           NIIT Technologies Ltd
4          3Pillar Global Pvt Ltd
..                             ...
481  Innovaccer Analytics Pvt Ltd
482         Kratikal Tech Pvt Ltd
483          Sofocle Technologies
484    SquadRun Solutions Pvt Ltd
485   Zaptas Technologies Pvt Ltd

[486 rows x 1 columns]

And saves data.csv (screenshot from LibreOffice):

Can't get data from each of the classes named 'heading' in a page using selenium

1 Answers