Scrape all amenities and Calendar data from AirBnB

There are few problems:

• click() works only with Selenium (driver.find_element()) but not with BeautifulSoup (soup2.select_one()) - so first you have to use different function

• for some reasons it can't find 'div.b6xigss.dir.dir-ltr' but it finds 'div.b6xigss button' (To make sure I search button because div can be "unclickable")

• there is message about cookies and it hides this element and selenium can't click. It would need to close this message (accept cookies), or it would need to scroll page to move button in visible place, or it needs to use JavaScript (driver.execute_script()) to click it.

This works for me

button = driver.find_element(By.CSS_SELECTOR, 'div.b6xigss button')

driver.execute_script('arguments[0].click()', button)

Miniamal working code:

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import pandas as pd
import time
import re

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
driver.maximize_window()

url = 'https://www.airbnb.com/s/Thessaloniki--Greece/homes?tab_id=home_tab&flexible_trip_lengths%5B%5D=one_week&refinement_paths%5B%5D=%2Fhomes&place_id=ChIJ7eAoFPQ4qBQRqXTVuBXnugk&query=Thessaloniki%2C%20Greece&date_picker_type=calendar&search_type=user_map_move&price_filter_input_type=0&ne_lat=40.66256734970964&ne_lng=23.003752862853986&sw_lat=40.59051931897441&sw_lng=22.892087137145978&zoom=13&search_by_map=true&federated_search_session_id=1ed21e1c-0c5e-4529-ab84-267361eac02b&pagination_search=true&items_offset={offset}&section_offset=2'

p_lat = re.compile(r'"lat":([-0-9.]+),')
p_lng = re.compile(r'"lng":([-0-9.]+),')

data = []

for offset in range(0, 40, 20):
    print('offset:', offset)
    
    driver.get(url.format(offset=offset))
    time.sleep(2)
    
    soup = BeautifulSoup(driver.page_source, 'lxml')

    detailed_pages = []
    
    for card in soup.select('div[class="c4mnd7m dir dir-ltr"] a[class="ln2bl2p dir dir-ltr"]'):
        link = 'https://www.airbnb.com' + card['href']
        detailed_pages.append(link)
        
    print('len(detailed_pages):', len(detailed_pages))
    
    for number, page in enumerate(detailed_pages, 1):
        print(number, 'page:', page)

        driver.get(page)
        time.sleep(5)
        
        soup2 = BeautifulSoup(driver.page_source, 'lxml')
        
        room_type = soup2.select_one('div._tqmy57')
        room_type = room_type.text if room_type else None
        
        #r= requests.get(page).text
        r = driver.page_source
        
        lat = p_lat.findall(r)[0]
        lng = p_lng.findall(r)[0]
        
        room_id = page[29: link.index("?")]
        
        titles = soup2.select_one('span._1n81at5')
        titles = titles.text if titles else None
        
        price = soup2.select_one('span._tyxjp1')
        price = price.text if price else None
        
        rating= soup2.select_one('span._12si43g')
        rating = rating.text if rating else None
        
        bedroom_area = soup2.select_one('div[class="_1a5glfg"]')
        bedroom_area = bedroom_area.text if bedroom_area else None
        
        place_offers= ', '.join([x.get_text(strip=True) for x in soup2.select('[class="sewcpu6 dir dir-ltr"]+div:nth-of-type(3) > div')])
        
        try:
            button = driver.find_element(By.CSS_SELECTOR, 'div.b6xigss button')
            driver.execute_script('arguments[0].click()', button)
        except Exception as ex:
            print('Exception:', ex)
                                              
        data.append({
            'Room_ID': room_id,
            'titles': titles,
            'place_offers': place_offers,
            'price': price,
            'rating': rating,
            'Bedroom_area': bedroom_area,
            'Room_Type': room_type,
            'Latitude': lat,
            'Longitude': lng
        })

df = pd.DataFrame(data)
df.to_csv('output.csv')
print(df)

EDIT:

As for calendar: every date has aria-disabled=True or aria-disabled=False and you can use aria-disabled to detect dates in calendar and later you can get value from aria-disabled like from any other attribute - item["aria-disabled"]

EDIT:

This works for me

for number, page in enumerate(detailed_pages, 1):

    print(number, 'page:', page)

    driver.get(page)
    time.sleep(5)

    # ... other code ...

    xpath = '//div[@aria-label="Calendar"]//div[@data-testid]'
    
    for item in driver.find_elements(By.XPATH, xpath):
        date    = item.get_attribute("data-testid")
        blocked = item.get_attribute("data-is-day-blocked")
        print(blocked, '|', date)

Result like this:

true | calendar-day-09/18/2022
true | calendar-day-09/19/2022
true | calendar-day-09/20/2022
false | calendar-day-09/21/2022
false | calendar-day-09/22/2022
false | calendar-day-09/23/2022