I am trying to scrape news from a website to twitter. But it is posting strange characters on twitter.
# -*- coding: utf-8 -*-
import tweepy
import requests
import logging
from config import *
from time import sleep
from bs4 import BeautifulSoup
logging.basicConfig(format='[%(levelname) 5s/%(asctime)s] %(name)s: %(message)s',
level=logging.INFO)
headers = {'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0'}
url = "https://www.todayonchain.com"
auth = tweepy.OAuth1UserHandler(
consumer_key, consumer_secret, access_token, access_token_secret
)
api = tweepy.API(auth)
old_link = ""
while True:
r = requests.get(url=url,headers=headers)
logging.info("Getting website date")
soup = BeautifulSoup(r.text,"html.parser")
title = soup.find("div",class_="api_article_title_sm")
title_text = title.text
link_container = soup.find("div",class_="api_article_include")
link = link_container.find("a",href=True)
link = link["href"]
soup = BeautifulSoup(requests.get(link,headers=headers).text,"html.parser")
new_title = (soup.title.get_text())
logging.info("Title: %s",new_title)
logging.info("Checking if For New Post")
if not link==old_link:
logging.info("New Post Detected on Website")
old_link=link
try:
logging.info("Posting to twitter")
api.update_status(f"""{title_text}\n{link}""")
logging.info("Posted on twitter")
except:
logging.info("Failed to post on twitter")
logging.info("Sleeping For 1 Minute")
sleep(10)