I have a list of instagram in a .txt file.
This is the URL which i have to scrape: https://brandfollowers.io/kol/all-post?uid=$INSTAGRAM$&page_num=$PAGENUMBER$(note i put $INSTAGRAM$ and $PAGENUMBER$ where i need to change variables)
For example, in this URL https://brandfollowers.io/kol/all-post?uid=philipppleinofficial&page_num=1
Im very new at this but actually i managed to work on take all items in page 1 for every instagram in list. However, i cant iterate over all pages from every instagram.
Could you please give me some tip? Im very new in this topic.
This is what i got right know:
# -*- coding: utf-8 -*-
import scrapy
import json
class ContenidoSpider(scrapy.Spider):
name = 'BACKUP_contenido'
allowed_domains = ['brandfollowers.io']
start_urls = ['http://brandfollowers.io/']
base_url = 'http://brandfollowers.io/kol/all-post?uid='
def parse(self, response):
FILE = open('list.txt', 'r').readlines()
instagrams = []
for lines in FILE:
new_line = lines.replace('https://www.instagram.com/', '')
instagrams.append(new_line)
for instagram in instagrams:
posts_url = self.base_url + instagram
yield scrapy.Request(posts_url, callback=self.parse_json)
def parse_json(self, response):
current_page = 0
pagesize = 6
json_response = json.loads(response.text)
path = json_response["data"]["models"]
while current_page < pagesize:
brand = path[current_page]["author"]["platform_unique_id"]
date = path[current_page]["platform_create_time"]
comments = path[current_page]["comment_count"]
likes = path[current_page]["like_count"]
engagement_rate = path[current_page]["share_count"]
description = path[current_page]["description"]
url_post = path[current_page]["post_url"]
picture_link = path[current_page]["picture_link"]
yield {
'BRAND': brand,
'DATE': date,
'COMMENTS': comments,
'LIKES': likes,
'ENGAGEMENT RATE': engagement_rate,
'DESCRIPTION': description,
'URL': url_post,
'PICTURE LINK': picture_link,
}
current_page += 1