Instaloader JSON files: Convert 200 JSON files into a Single CSV (Python 3.7)

Question

I want to automatically download pictures (or videos) along with their captions and other data from a specific Instagram Hashtag (e.g. #moodoftheday) using Instaloader. Instaloader returns JSON files including posts metadata.

The following code worked with just a single @user_profile metadata. I want to do the same, but for a #hashtag not a specific @user.

The ultimate goal is to have all of the JSON files (e.g. 200) into a csv file. How can I process my downloaded data in a clean excel/CSV file?

Here is my code:

# Install Instaloader
import instaloader


def get_instagram_posts(username, startdate, enddate):
   # Create an instaloader object with parameters
   L = instaloader.Instaloader(download_pictures = False, download_videos = False, download_comments= False, compress_json = False)
   
   # Log in with the instaloader object
   L.login("username" , "password")
   # Search the instagram profile
   profile = instaloader.Profile.from_username(L.context, username) 
   # Scrape the posts
   posts = profile.get_posts()
   for post in takewhile(lambda p: p.date > startdate, dropwhile(lambda p : p.date > enddate, posts)): 
   print(post.date)
   L.download_post(post, target = profile.username)

'''
This function will now save all instagram posts and related data to a folder in you current working directory.
Let’s call this function on the instagram account of “moodoftheday”. let the script do its magic.
This might take a while so be patient. 
'''
  
import os
import datetime
# instagram username
username = "realdonaldtrump"
# daterange of scraping
startdate = datetime(2020, 9, 1)
enddate = datetime(2020, 10, 1)
# get your current working directory
current_wkdir = os.get_cwd()
# Call the function. This will automatically store all the scrape data in a folder in your current working directory
get_instagram_posts(username, startdate, enddate)


'''
You notice that this data is NOT yet in the right format since each post has a separate json file. 
You will need to process all these json files to a consolidated excel file in order to perform analyses on the data.
'''


def parse_instafiles(username, path):
    """ 
    This function loads in all the json files generated by the instaloader package and parses it into a csv file.
    """
    #print('Entering provided directory...')
    os.chdir(os.path.join(path, username))
    
    columns = ['filename', 'datetime', 'type', 'locations_id', 'locations_name', 'mentions', 'hashtags', 'video_duration']
    
    dataframe = pd.DataFrame(columns=[])
    
    #print('Traversing file tree...')
    
    glob('*UTC.json')
    
    for file in glob('*UTC.json'):
        with open(file, 'r') as filecontent:
            filename = filecontent.name
            #print('Found JSON file: ' + filename + '. Loading...')
            
            try:
                metadata = orjson.loads(filecontent.read())
            
            except IOError as e:
                #print("I/O Error. Couldn't load file. Trying the next one...")
                continue
            else:
                pass
            #print('Collecting relevant metadata...')
            time = datetime.fromtimestamp(int(metadata['node']['taken_at_timestamp']))
            type_ = metadata['node']['__typename']
            likes = metadata['node']['edge_media_preview_like']['count']     
            comments = metadata['node']['edge_media_to_comment']['count']
            username = metadata['node']['owner']['username']
            followers = metadata['node']['owner']['edge_followed_by']['count']
            try:
                text = metadata['node']['edge_media_to_caption']['edges'][0]['node']['text']
            except:
                text = ""
            try:
                post_id = metadata['node']['id']
            except:
                post_id = ""
            minedata = {'filename': filename, 'time': time, 'text': text,
                    'likes': likes, 'comments' : comments, 'username' : username,  'followers' : followers, 'post_id' : post_id}
            #print('Writing to dataframe...')
            dataframe = dataframe.append(minedata, ignore_index=True)
            #print('Closing file...')
            del metadata
            filecontent.close()
    #print('Storing dataframe to CSV file...')
    #print('Done.')
    dataframe['source'] = 'Instagram'
    return dataframe

'''
You can then use this function to process the "moodoftheday" Instagram data.
'''  

df_instagram = parse_instafiles(username, os.getcwd() )
df_instagram.to_excel("moodoftheday.csv")

I am very new to Python and programming overall, therefore any help is very much appreciated!! Thank you in advance! Sofia

user_5022529 · Answer

I made some changes it's not showing error but still needs some professional works:

import instaloader
from datetime import datetime
import datetime
from itertools import takewhile
from itertools import dropwhile
import os
import glob as glob
import json
import pandas as pd
import csv

lusername = ''
lpassword = ''

def get_instagram_posts(username, startdate, enddate):
   # Create an instaloader object with parameters
   L = instaloader.Instaloader(download_pictures = False, download_videos = False, download_comments= False, compress_json = False)
   
   # Log in with the instaloader object
   L.login("lusername" , "lpassword")

   # Search the instagram profile
   profile = instaloader.Profile.from_username(L.context, username) 
   # Scrape the posts
   posts = profile.get_posts()
   for post in takewhile(lambda p: p.date > startdate, dropwhile(lambda p : p.date > enddate, posts)): 
    print(post.date)
   L.download_post(post, target = profile.username)


# instagram username
username = "realdonaldtrump"
# daterange of scraping
startdate = datetime.datetime(2020, 9, 1,0,0)
enddate = datetime.datetime(2022, 2, 1,0,0)
# get your current working directory
current_wkdir = os.getcwd()
# Call the function. This will automatically store all the scrape data in a folder in your current working directory
get_instagram_posts(username, startdate, enddate)


def parse_instafiles(username, path):
    #print('Entering provided directory...')
    os.chdir(os.path.join(path, username))
    
    columns = ['filename', 'datetime', 'type', 'locations_id', 'locations_name', 'mentions', 'hashtags', 'video_duration']
    
    dataframe = pd.DataFrame(columns=[])
    
    #print('Traversing file tree...')
    
#    glob('*UTC.json')
    
    for file in glob.glob('*UTC.json'):
        with open(file, 'r') as filecontent:
            filename = filecontent.name
            #print('Found JSON file: ' + filename + '. Loading...')
            
            try:
                metadata = json.load(filecontent)
            
            except IOError as e:
                #print("I/O Error. Couldn't load file. Trying the next one...")
                continue
            else:
                pass
            #print('Collecting relevant metadata...')
            time = datetime.datetime.fromtimestamp(int(metadata['node']['taken_at_timestamp']))
            type_ = metadata['node']['__typename']
            likes = metadata['node']['edge_media_preview_like']['count']     
            comments = metadata['node']['edge_media_to_comment']['count']
            username = metadata['node']['owner']['username']
            followers = metadata['node']['owner']['edge_followed_by']['count']
            try:
                text = metadata['node']['edge_media_to_caption']['edges'][0]['node']['text']
            except:
                text = ""
            try:
                post_id = metadata['node']['id']
            except:
                post_id = ""
            minedata = {'filename': filename, 'time': time, 'text': text,
                    'likes': likes, 'comments' : comments, 'username' : username,  'followers' : followers, 'post_id' : post_id}
            #print('Writing to dataframe...')
            dataframe = dataframe.append(minedata, ignore_index=True)
            #print('Closing file...')
            del metadata
            filecontent.close()
    #print('Storing dataframe to CSV file...')
    #print('Done.')
    dataframe['source'] = 'Instagram'
    return dataframe

'''
You can then use this function to process the "moodoftheday" Instagram data.
'''  

df_instagram = parse_instafiles(username, os.getcwd() )
df_instagram.to_csv("moodoftheday.csv")

user_15694555 · Answer

Instaloader has an example of hashtag search on its documentation, here's the code:

from datetime import datetime
import instaloader

L = instaloader.Instaloader()

posts = instaloader.Hashtag.from_name(L.context, "urbanphotography").get_posts()

SINCE = datetime(2020, 5, 10)  # further from today, inclusive
UNTIL = datetime(2020, 5, 11)  # closer to today, not inclusive

k = 0  # initiate k
#k_list = []  # uncomment this to tune k

for post in posts:
    postdate = post.date

    if postdate > UNTIL:
        continue
    elif postdate <= SINCE:
        k += 1
        if k == 50:
            break
        else:
            continue
    else:
        L.download_post(post, "#urbanphotography")
        # if you want to tune k, uncomment below to get your k max
        #k_list.append(k)
        k = 0  # set k to 0

#max(k_list)

here's the link for more info:

https://instaloader.github.io/codesnippets.html

I'm trying do to something similar, but I'm still very new to programming, so I'm sorry if I can't offer much help

Instaloader JSON files: Convert 200 JSON files into a Single CSV (Python 3.7)

2 Answers