Scraping an Infinite Scroll Page

What you have to do is:

1. find the house button you need
2. scroll down to this button
3. click the button, switch tab, get the data, go back to the main tab
4. go to step 1.

Here's the code to do this (except for extracting the data):

    from selenium import webdriver
    import time
    import numpy as np


    url = 'https://www.quintoandar.com.br/alugar/imovel/sao-paulo-sp-brasil'
    xpath_house_buttons = "//div[@class='sc-1qwl1yl-0 igVsBW']"
    x_path_title = "//h1[@class='sc-1q9n36n-0 ghXeyc sc-bdVaJa hgGleC']"
    x_path_address = "//p[@data-testid='listing-address-subtitle']"
    num_houses = 40
    houses = []


    def scroll_to_house_button(driver, num_btn) -> bool:
        """
        returns true if it could scroll to the house button
        """
        try:
            house_buttons = driver.find_elements_by_xpath(xpath_house_buttons)
            driver.execute_script("arguments[0].scrollIntoView();", house_buttons[num_btn])
            return True
        except:
            return False
    def switch_to_house_tab(driver) -> bool:
        """
        returns true if switching tab was successful
        """
        try:
            driver.switch_to.window(driver.window_handles[1])
            return True
        except:
            return False
    def switch_to_main_tab(driver) -> bool:
        """
        returns true if switching tab was successful
        """
        try:
            driver.switch_to.window(driver.window_handles[0])
            return True
        except:
            return False
    def get_house_button_index(house_buttons, houses_scraped_text, index):
        """
        returns house_button's index in house_buttons
        """
        # at the beginning, the house to scrape is given by its index
        if len(houses_scraped_text) < 5:
            return index
        # afterwards, we try to find by comparing the buttons' content
        else:
            for i in reversed(range(len(house_buttons))):
                if (house_buttons[i].text == houses_scraped_text[-1]) and (house_buttons[i - 1].text == houses_scraped_text[-2]):
                    return i + 1


    # Initializing the webdriver
    driver = webdriver.Chrome()
    driver.set_window_size(1600, 1024)
    driver.get(url)


    # get data
    i = 0
    houses_scraped_text = []
    while len(houses) < num_houses:
        house_buttons = driver.find_elements_by_xpath(xpath_house_buttons)

        # as house_buttons never exceeds a length of 30,
        # we need a smart way of getting the next one
        index_btn = get_house_button_index(house_buttons, houses_scraped_text, i)
        house_button = house_buttons[index_btn]
        houses_scraped_text.append(house_button.text)

        # can't scroll to house button => wait 1 sec
        while not scroll_to_house_button(driver, num_btn=index_btn):
            time.sleep(1)
        print("scroll to house -- house", i + 1)

        # filter houses to be scraped
        if not ((not 'Sem tempo pra procurar' in house_button.text) and (not 'Ainda não encontrou seu lar' in house_button.text)):
            print("house filtered -- house", i + 1, "\n")
            i += 1 # you have to increment here to not loop over the same house forever
            continue

        # new house tab not open yet => wait 1 sec
        while len(driver.window_handles) != 2: # check number of open tabs
            house_button.click()
            time.sleep(1)
        print("new house tab opened -- house", i + 1)

        # can't switch yet => wait 1 sec
        while not switch_to_house_tab(driver):
            time.sleep(1)
        print("switched to new house tab -- house", i + 1)

        ##################
        # LOAD DATA HERE #
        ##################
        print("data loaded -- house", i + 1)

        # close tab
        driver.close()
        # can't switch back to main => wait 1 sec
        while not switch_to_main_tab(driver):
            time.sleep(1)
        print("house tab closed & switched back to main -- house", i + 1)
        
        print(len(houses), "house scraped\n")
        i += 1