how to make web scraping dataframe in python?

Question

I am scraping data from a website with web scraping technique. I got the html link of each page. I also got the url links of the postings on each page. I also took the data of the table containing the information about the residence.

My problem is that I wrote a for loop. I want to take the information of each product and make a dataframe. But for only one post's information is coming.

A second problem is that not every posting has an equal number of column names. That's why I couldn't create a dataframe.

The sample code I wrote is as follows:

#import library
from selenium import webdriver
import requests
from bs4 import BeautifulSoup
from time import sleep

# Data manipulation
import pandas as pd

# Visualization
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

# function
def get_soup(TARGET_URL):
    page = requests.get(TARGET_URL, headers = headers_param)
    jobs = page.content
    soup = BeautifulSoup(jobs, 'html.parser')
    return soup

def clean_scraped(data):
    return data.replace(' ', '').replace('
', '')

headers_param = {"User-Agent": ""}


#internet sitesine ulaşma
browser = webdriver.Chrome("...")
sahibinden_arama = "istanbul kiralık konut"

sahibinden = browser.get("https://www.sahibinden.com/")
sleep(0.5)

#arama
browser.find_element("xpath",'//*[@id="searchText"]')\
    .send_keys(sahibinden_arama)
browser.find_element("xpath",'//*[@id="searchSuggestionForm"]/button')\
    .click()
sleep(2)

browser.execute_script("arguments[0].click();", WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="searchResultsSearchForm"]/div/div[3]/div[3]/div[2]/ul/li[2]/a'))))

sleep(2)

page = 0
page_urls = []

while page != 1000:
    url = f"https://www.sahibinden.com/emlak-konut/istanbul?pagingOffset={page}&pagingSize=50&query_text_mf=istanbul+kiral%C4%B1k+konut&query_text=kiral%C4%B1k"
    page_urls.append(url)
    print(url)
    page = page + 50


product_urls = []

for i in page_urls:
    page_soup = get_soup(i)
    
    for j in page_soup.find_all("a", attrs={'class':'classifiedTitle'}):
        product_urls.append("https://www.sahibinden.com" + (j["href"]))
        sleep(0.5)
        print(product_urls)


for i in product_urls:
    product_soup = get_soup(i)
    sleep(0.3)
    all_data = []
    for i in product_soup.select("ul.classifiedInfoList"):
        adres_name = i.find_previous("h2").get_text(strip=True)
        info = [li.get_text(strip=True) for li in i.select("span")]
        all_data.append([*info])
    print(all_data)
    df = pd.DataFrame(all_data)
df

The problem I'm talking about is in this part:

for i in product_urls:
    product_soup = get_soup(i)
    sleep(0.3)
    all_data = []
    for i in product_soup.select("ul.classifiedInfoList"):
        adres_name = i.find_previous("h2").get_text(strip=True)
        info = [li.get_text(strip=True) for li in i.select("span")]
        all_data.append([*info])
    print(all_data)
    df = pd.DataFrame(all_data)
df

Thank you for every help.

how to make web scraping dataframe in python?

0 Answers