Im having problems scrapping using beautifullsoup

Viewed 27

im tryng to get son data from the web page that is in the code. The code works but I cant get any results. I think the problem is here results = soup.find_all('div', {'class': 'ui-search-layout__item' }) that's why i try to get the classes from the website, but i still cant gent any result.

I'm a bit lost here and I cant find the problem.

import requests
from bs4 import BeautifulSoup
import pandas as pd
from time import sleep

headers = {
    'user-agent' :'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
}


search_query = 'venta inmuebles'.replace(' ', '+')
base_url = 'https://listado.mercadolibre.com.uy/{0}'.format(search_query)

###
# class list set
class_list = set()
  
# Page content 
page = requests.get(base_url)
  
# parse html content
soup = BeautifulSoup( page.content , 'html.parser')
  
# get all tags
tags = {tag.name for tag in soup.find_all()}
  
# iterate all tags
for tag in tags:
  
    # find all element of tag
    for i in soup.find_all( tag ):
  
        # if tag has attribute of class
        if i.has_attr( "class" ):
  
            if len( i['class'] ) != 0:
                class_list.add(" ".join( i['class']))
print( class_list )
##
items = []    
for i in range(1, 2):
    print('Processing {0}...'.format(base_url + '&page={0}'.format(i)))
    response = requests.get(base_url + '&page={0}'.format(i), headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')
    
    results = soup.find_all('div', {'class': 'ui-search-layout__item' })
    
    for result in results:
        product_title = result.h2.text
        
        try: 
            value = result.find('span', {'class': 'price-tag-fraction'}).text
            currency = result.find('span', {'class': 'price-tag-symbol'}).text
            surface = result.find('li', {'class': 'ui-search-card-attributes__attribute'}).text
            dorms = result.find('li', {'class': 'ui-search-card-attributes__attribute'}).text
            url = 'https://listado.mercadolibre.com.uy' + result.h2.a['href']
            items.append([value, currency, surface, dorms, url])
            
        except AttributeError:
            continue
        sleep(1.5)


df = pd.DataFrame(items, columns=['value', 'currency', 'surface', 'dorms', 'url'])
0 Answers
Related