Scraping the scores (Location, Cleanliness etc.) from Airbnb

Question

I am trying to scrape the Location Score, Cleaningness Score etc. from the airbnb listings.

My problem is that when I am trying to scrape them from one page I scrape them normally with multiple ways.

But when I add this code in my loop to scrape them from multiple page either I get an error or I get back an empty column/list.

Let me show you what I mean.

This is my imports:

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import pandas as pd
import time
import re
import requests
from datetime import date


driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.maximize_window()

Some ways that I can get what Scores:

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.maximize_window()
wait = WebDriverWait(driver,20)
test_url = 'https://www.airbnb.com/rooms/51833563?adults=1&children=0&infants=0&check_in=2022-09-18&check_out=2022-09-23&federated_search_id=96d13170-4935-492e-ad71-ffe0c39bc006&source_impression_id=p3_1663079036_%2FKJDqmaFlIk3h8a6'
driver.get(test_url)
time.sleep(3)
soup2 = BeautifulSoup(driver.page_source, 'lxml')
scores = []
Cleanliness=  driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[0].text
Accuracy= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[1].text
Communication=  driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[2].text
Location= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[3].text
Check_in= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[4].text
Value= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[5].text

When I add this in my loop I get an error that the list is out of index.

This is another way to scrape the information that i want:

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.maximize_window()
wait = WebDriverWait(driver,20)
test_url = 'https://www.airbnb.com/rooms/51833563?adults=1&children=0&infants=0&check_in=2022-09-18&check_out=2022-09-23&federated_search_id=96d13170-4935-492e-ad71-ffe0c39bc006&source_impression_id=p3_1663079036_%2FKJDqmaFlIk3h8a6'
driver.get(test_url)
time.sleep(3)
soup2 = BeautifulSoup(driver.page_source, 'lxml')
scores = []
Cleanliness=  driver.find_elements(By.CLASS_NAME,"_4oybiu")
for i in Cleanliness:
    scores.append(i.text)

When i add this in my loop I get back an empty list.

And this is the code inside the loop:

url = 'https://www.airbnb.com/s/Thessaloniki--Greece/homes?tab_id=home_tab&flexible_trip_lengths%5B%5D=one_week&refinement_paths%5B%5D=%2Fhomes&place_id=ChIJ7eAoFPQ4qBQRqXTVuBXnugk&query=Thessaloniki%2C%20Greece&date_picker_type=calendar&search_type=user_map_move&price_filter_input_type=0&ne_lat=40.66256734970964&ne_lng=23.003752862853986&sw_lat=40.59051931897441&sw_lng=22.892087137145978&zoom=13&search_by_map=true&federated_search_session_id=1ed21e1c-0c5e-4529-ab84-267361eac02b&pagination_search=true&items_offset={offset}§ion_offset=2'

p_lat = re.compile(r'"lat":([-0-9.]+),')
p_lng = re.compile(r'"lng":([-0-9.]+),')

data = []
calendar = []





for offset in range(0, 20, 20):
    print('offset:', offset)
    
    driver.get(url.format(offset=offset))
    time.sleep(2)
    
    soup = BeautifulSoup(driver.page_source, 'lxml')

    detailed_pages = []
    
    for card in soup.select('div[class="c4mnd7m dir dir-ltr"] a[class="ln2bl2p dir dir-ltr"]'):
        link = 'https://www.airbnb.com' + card['href']
        detailed_pages.append(link)
        
    print('len(detailed_pages):', len(detailed_pages))
    
    for number, page in enumerate(detailed_pages, 1):
        print(number, 'page:', page)

        driver.get(page)
        time.sleep(5)
        
        soup2 = BeautifulSoup(driver.page_source, 'lxml')
        Cleanliness=  driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[0].text
        Accuracy= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[1].text
        Communication=  driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[2].text
        Location= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[3].text
        Check_in= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[4].text
        Value= driver.find_elements(By.XPATH,"//span[@class='_4oybiu'][1]")[5].tex

KunduK · Answer

To investigate the element, I found the element you are after is inside scripts tag. You need the get the text from script tag and then covert into json object and then traverse the tree to get the ratings.

driver.get('https://www.airbnb.com/rooms/51833563?adults=1&children=0&infants=0&check_in=2022-09-18&check_out=2022-09-23&federated_search_id=96d13170-4935-492e-ad71-ffe0c39bc006&source_impression_id=p3_1663079036_%2FKJDqmaFlIk3h8a6')
element=driver.find_element(By.CSS_SELECTOR, "script#data-deferred-state")
elementHtml=element.get_attribute("innerHTML")
json_object = json.loads(elementHtml) # convert string to json object
parent=json_object['niobeMinimalClientData'][0]
child=parent[1]
eventlogging=child['data']['presentation']['stayProductDetailPage']['sections']['metadata']['loggingContext']['eventDataLogging']

print("Accuracy Ratings : " + str(eventlogging['accuracyRating']))
print("Cleanliness Ratings : " + str(eventlogging['cleanlinessRating']))
print("Communication Ratings : " +str(eventlogging['communicationRating']))
print("Location Ratings : " +str(eventlogging['locationRating']))
print("Checking Ratings : " +str(eventlogging['checkinRating']))

You need to import below library

import json

Output:

Accuracy Ratings : 5
Cleanliness Ratings : 4.9
Communication Ratings : 5
Location Ratings : 4.8
Checking Ratings : 5

Scraping the scores (Location, Cleanliness etc.) from Airbnb

1 Answers