I am attempting to learn python in my spare time and I was trying to scrape Amazon for product names and links using this code I found online with some input by me. I am able to get the names and links from a search query on Amazon, example: Playstation. I get multiple results and when I run the code, it prints the multiple product names as well as their links.
So, I thought it would be cool to write some more code to Tweet this information on Twitter. However, I would like to send one tweet with only one product name and link at a time. The way I wrote my code now attempts to post every product with every link in just one tweet and that is too many characters to post to Twitter. Below is my code, it may be formatted incorrectly here, but I have it formatted correctly on my desktop.
Is there a way I can break up the list of links and post 1 product with 1 link in only 1 tweet and then write the code so that I can post the 1st product, 2nd, and so on after setting a time delay between tweets?
from bs4 import BeautifulSoup
import requests
from urllib.request import urlopen
import tweepy
import config
import time
#Twitter User Authentication Information
client = tweepy.Client(consumer_key=config.API_Key,
consumer_secret=config.API_Secret,
access_token=config.Access_Token,
access_token_secret=config.Access_Token_Secret)
# Headers for request
HEADERS = ({'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
'Accept-Language': 'en-US'})
# The webpage URL
URL = "https://www.amazon.com/s?k=playstation&crid=3B4KWPIKP87G0&sprefix=playstation%2Caps%2C111&ref=nb_sb_noss_1"
# HTTP Request
webpage = requests.get(URL, headers=HEADERS)
# Soup Object containing all data
soup = BeautifulSoup(webpage.content, "lxml")
# Function to extract Product Title
def get_title(soup):
try:
# Outer Tag Object
title = soup.find("span", attrs={"id":'productTitle'})
# Inner NavigatableString Object
title_value = title.string
# Title as a string value
title_string = title_value.strip()
except AttributeError:
title_string = ""
# Function to extract Product Price
def get_price(soup):
try:
price = soup.find("span", attrs={'id':'a-offscreen'}).string.strip()
except AttributeError:
try:
# If there is some deal price
price = soup.find("span", attrs={'id':'priceblock_dealprice'}).string.strip()
except:
price = ""
return price
return title_string
# Game System listed underneath Title
def system(soup):
system_name = soup.find("a", attrs={"class":'a-size-base a-link-normal s-underline text s-underline-link-text s-link-style a-text-bold'})
title_text = system_name.text
return title_text
# Fetch links as List of Tag Objects
links = soup.find_all("a", attrs={'class':'a-link-normal s-no-outline'})
# Store the links
links_list = []
# Loop for extracting links from Tag Objects
for link in links:
links_list.append(link.get('href'))
# Loop for extracting product details from each link
for link in links_list:
new_webpage = requests.get("https://www.amazon.com" + link, headers=HEADERS)
new_soup = BeautifulSoup(new_webpage.content, "lxml")
if ('Playstation' in system(soup)):
post_tweet()
def post_tweet():
tweet = client.create_tweet(text = get_title(new_soup) + " " + "is" + get_price(new_soup) + " " + "https://www.amazon.com" + link)
return tweet