I'm trying to extract the all reviews from people who reviewed hotels in Bali, this should be the pathway that i'm following
- First define the star url https://www.tripadvisor.com.pe/Hotels-g294226-Bali-Hotels.html
- Scrapy the pages from all the hotels related to first link https://www.tripadvisor.com.pe/Hotels-g294226-oa30-Bali-Hotels.html (every page change -oaXX-)
- Obtain the reviews from each hotel in each page from the hotel https://www.tripadvisor.com.pe/Hotel_Review-g3404940-d1532277-Reviews-Munduk_Moding_Plantation_Nature_Resort_Spa-Gobleg_Banjar_Buleleng_Regency_Bali.html
- Scrapy on the pages from the opinions from each hotel https://www.tripadvisor.com.pe/Hotel_Review-g3404940-d1532277-Reviews-or5-Munduk_Moding_Plantation_Nature_Resort_Spa-Gobleg_Banjar_Buleleng_Regency_Bali.html#REVIEWS (every page change -orX-) 5.Access to the profile of each user who reviewed and extract all the opinions by the user https://www.tripadvisor.com.pe/Profile/AntonioPozo89?fid=6b049a76-cdf0-4534-a276-3b124769df03
from scrapy.item import Field
from scrapy.item import Item
from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from scrapy.loader.processors import MapCompose
from scrapy.linkextractors import LinkExtractor
from scrapy.loader import ItemLoader
class Opinion(Item):
titulo = Field()
calificacion = Field()
contenido = Field()
autor = Field()
class TripAdvisor(CrawlSpider):
name = "OpinionesTripAdvisor"
custom_settings = {
'USER_AGENT': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
'CLOSESPIDER_PAGECOUNT':100
}
download_delay = 1
allowed_domains = ['tripadvisor.com']
start_urls = ["https://www.tripadvisor.com.pe/Hotels-g294226-Bali-Hotels.html"]
rules = (
#Page by hotels Horixontal
Rule(
LinkExtractor(
allow=r'-oa\d+-'
), follow=True
),
#Hotel Details Vertical, restric only the links that are in the names
Rule(
LinkExtractor(
allow=r'/Hotel_Review-',
restrict_xpaths=['//div[@id="taplc_hsx_hotel_list_lite_dusty_hotels_combined_sponsored_ad_density_control_0"]//a[data-clicksource="HotelName"]']
), follow=True
),
#Page by opinions in the below part
Rule(
LinkExtractor(
allow=r'/-or\d+-/'
), follow=True
),
#Detail by profile
Rule(
LinkExtractor(
allow=r'/profile/',
#restrict_xpaths=['//div[@data-test-target="HR_CC_CARD"]//a[contains(@class, "ui_header")'] #a[@class="ui_header_link uyyBf"]
), follow=True, callback='parse_opinion'
)
)
#obtain review from tag ui_bubble_rating bubble_50
def obtenerCalif(self, texto):
calificacion = texto.split("_")[-1]
return calificacion
#parsing the callback, iteration the review
def parse_opinion(self, response):
sel = Selector(response)
opiniones = sel.xpath('//div[@id="content"]/div/div')
autor = sel.xpath('//h1/text()').get() #get the user by the last page
for opinion in opiniones:
item = ItemLoader(Opinion(), opinion)
item.add_value('autor',autor)
item.add_xpath('titulo','//div[class="AzIrY b _a VrCoN"]/text()')
item.add_xpath('contenido', './/q/text()')
item.add_xpath('calificacion', './/div[@class="muQub VrCoN"]/span/@class',
MapCompose(self.obtenerCalif))
yield item.load_item()
the script runs but the file is empty, I'm not sure what is wrong