Web Scrapping from TripAdvisor with Scrapy don't extract

Viewed 28

I'm trying to extract the all reviews from people who reviewed hotels in Bali, this should be the pathway that i'm following

  1. First define the star url https://www.tripadvisor.com.pe/Hotels-g294226-Bali-Hotels.html
  2. Scrapy the pages from all the hotels related to first link https://www.tripadvisor.com.pe/Hotels-g294226-oa30-Bali-Hotels.html (every page change -oaXX-)
  3. Obtain the reviews from each hotel in each page from the hotel https://www.tripadvisor.com.pe/Hotel_Review-g3404940-d1532277-Reviews-Munduk_Moding_Plantation_Nature_Resort_Spa-Gobleg_Banjar_Buleleng_Regency_Bali.html
  4. Scrapy on the pages from the opinions from each hotel https://www.tripadvisor.com.pe/Hotel_Review-g3404940-d1532277-Reviews-or5-Munduk_Moding_Plantation_Nature_Resort_Spa-Gobleg_Banjar_Buleleng_Regency_Bali.html#REVIEWS (every page change -orX-) 5.Access to the profile of each user who reviewed and extract all the opinions by the user https://www.tripadvisor.com.pe/Profile/AntonioPozo89?fid=6b049a76-cdf0-4534-a276-3b124769df03
from scrapy.item import Field
from scrapy.item import Item
from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from scrapy.loader.processors import MapCompose
from scrapy.linkextractors import LinkExtractor
from scrapy.loader import ItemLoader

class Opinion(Item):
    titulo = Field()
    calificacion = Field()
    contenido = Field()
    autor = Field()

class TripAdvisor(CrawlSpider):
    name = "OpinionesTripAdvisor"
    custom_settings = {
        'USER_AGENT': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
        'CLOSESPIDER_PAGECOUNT':100
    }
    download_delay = 1
    allowed_domains = ['tripadvisor.com']
    start_urls = ["https://www.tripadvisor.com.pe/Hotels-g294226-Bali-Hotels.html"]

    rules = (
        #Page by hotels Horixontal
        Rule(
            LinkExtractor(
                allow=r'-oa\d+-'
            ), follow=True
        ),

        #Hotel Details Vertical, restric only the links that are in the names
        Rule(
            LinkExtractor(
                allow=r'/Hotel_Review-',
                restrict_xpaths=['//div[@id="taplc_hsx_hotel_list_lite_dusty_hotels_combined_sponsored_ad_density_control_0"]//a[data-clicksource="HotelName"]']
            ), follow=True
        ),

        #Page by opinions in the below part
        Rule(
            LinkExtractor(
                allow=r'/-or\d+-/'
            ), follow=True

        ),

        #Detail by profile
        Rule(
            LinkExtractor(
                allow=r'/profile/',
                #restrict_xpaths=['//div[@data-test-target="HR_CC_CARD"]//a[contains(@class, "ui_header")'] #a[@class="ui_header_link uyyBf"]
            ), follow=True, callback='parse_opinion'
        )
    )

    #obtain review from tag ui_bubble_rating bubble_50
    def obtenerCalif(self, texto):
        calificacion = texto.split("_")[-1]
        return calificacion

    #parsing the callback, iteration the review
    def parse_opinion(self, response):
        sel = Selector(response)
        opiniones = sel.xpath('//div[@id="content"]/div/div')
        autor = sel.xpath('//h1/text()').get() #get the user by the last page


        for opinion in opiniones:
            item = ItemLoader(Opinion(), opinion)
            item.add_value('autor',autor)
            item.add_xpath('titulo','//div[class="AzIrY b _a VrCoN"]/text()')
            item.add_xpath('contenido', './/q/text()')
            item.add_xpath('calificacion', './/div[@class="muQub VrCoN"]/span/@class',
                           MapCompose(self.obtenerCalif))
            yield item.load_item()

the script runs but the file is empty, I'm not sure what is wrong

1 Answers

I did a mistake in the allowed domain, and also I add a new item to get the profile url and check if the scraped information its correct

from scrapy.item import Field
from scrapy.item import Item
from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from scrapy.loader.processors import MapCompose
from scrapy.linkextractors import LinkExtractor
from scrapy.loader import ItemLoader
from scrapy.crawler import CrawlerProcess

class Opinion(Item):
    titulo = Field()
    calificacion = Field()
    contenido = Field()
    autor = Field()
    profilel = Field()

class TripAdvisor(CrawlSpider):
    name = "OpinionesTripAdvisor"
    custom_settings = {
        'USER_AGENT': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
        'CLOSESPIDER_PAGECOUNT':35
    }
    download_delay = 1
    allowed_domains = ['tripadvisor.com.pe'] #its necesary set .pe
    start_urls = ["https://www.tripadvisor.com.pe/Hotels-g294226-Bali-Hotels.html"]
    rules = (
        #Page by hotels Horizontal
        Rule(
            LinkExtractor(
                allow=r'-oa\d+'
            ), follow=True
        ),

        #Hotel Details Vertical, restric only the links that are in the names
        Rule(
            LinkExtractor(
                allow=r'/Hotel_Review-',
                # restrict_xpaths=['//div[@id="taplc_hsx_hotel_list_lite_dusty_hotels_combined_sponsored_ad_density_control_0"]//a[data-clicksource="HotelName"]']
            ), follow=True
        ),
        #Page by opinions in the below part
        Rule(
            LinkExtractor(
                allow=r'/-or\d+-/'
            ), follow=True

        ),

        #Detail by profile
        Rule(
            LinkExtractor(
                allow=r'/Profile/',
                #restrict_xpaths=['//div[@data-test-target="HR_CC_CARD"]//a[contains(@class, "ui_header")'] #a[@class="ui_header_link uyyBf"]
            ), follow=True, callback='parse_item'
        )
    )

    #obtain review from tag ui_bubble_rating bubble_50
    def obtenerCalif(self, texto):
        calificacion = texto.split("_")[-1]
        return calificacion

    #parsing the callback, iteration the review
    def parse_item(self, response):
        sel = Selector(response)
        opiniones = sel.xpath('//div[@id="content"]/div/div')
        autor = sel.xpath('//h1/span/text()').get() #get the user by the last page


        for opinion in opiniones:
            item = ItemLoader(Opinion(), opinion)
            item.add_value('autor',autor)
            item.add_xpath('titulo','//div[@class="AzIrY b _a VrCoN"]/text()')
            item.add_xpath('contenido', './/q/text()')
            item.add_xpath('calificacion', './/div[@class="muQub VrCoN"]/span/@class',
                           MapCompose(self.obtenerCalif))
            item.add_xpath('profilel','//link[@rel="canonical"]/@href')

            yield item.load_item()



Related