im tryng to get son data from the web page that is in the code. The code works but I cant get any results. I think the problem is here results = soup.find_all('div', {'class': 'ui-search-layout__item' }) that's why i try to get the classes from the website, but i still cant gent any result.
I'm a bit lost here and I cant find the problem.
import requests
from bs4 import BeautifulSoup
import pandas as pd
from time import sleep
headers = {
'user-agent' :'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
}
search_query = 'venta inmuebles'.replace(' ', '+')
base_url = 'https://listado.mercadolibre.com.uy/{0}'.format(search_query)
###
# class list set
class_list = set()
# Page content
page = requests.get(base_url)
# parse html content
soup = BeautifulSoup( page.content , 'html.parser')
# get all tags
tags = {tag.name for tag in soup.find_all()}
# iterate all tags
for tag in tags:
# find all element of tag
for i in soup.find_all( tag ):
# if tag has attribute of class
if i.has_attr( "class" ):
if len( i['class'] ) != 0:
class_list.add(" ".join( i['class']))
print( class_list )
##
items = []
for i in range(1, 2):
print('Processing {0}...'.format(base_url + '&page={0}'.format(i)))
response = requests.get(base_url + '&page={0}'.format(i), headers=headers)
soup = BeautifulSoup(response.content, 'html.parser')
results = soup.find_all('div', {'class': 'ui-search-layout__item' })
for result in results:
product_title = result.h2.text
try:
value = result.find('span', {'class': 'price-tag-fraction'}).text
currency = result.find('span', {'class': 'price-tag-symbol'}).text
surface = result.find('li', {'class': 'ui-search-card-attributes__attribute'}).text
dorms = result.find('li', {'class': 'ui-search-card-attributes__attribute'}).text
url = 'https://listado.mercadolibre.com.uy' + result.h2.a['href']
items.append([value, currency, surface, dorms, url])
except AttributeError:
continue
sleep(1.5)
df = pd.DataFrame(items, columns=['value', 'currency', 'surface', 'dorms', 'url'])