I'm currently writing code to download all words from a conference but I currently can't get it because of this codec issue. How do I fix it?
import requests
import PyPDF2
import io
from bs4 import BeautifulSoup
import urllib.request
from PyPDF2._reader import PdfFileReader
# Target URL and go get
URL = 'http://proceedings.mlr.press/v70/'
response = requests.get(URL)
# parse content
content = BeautifulSoup(response.text, 'lxml')
all_content_links = content.find_all('a')
count = 0
PDF_links = []
for link in all_content_links:
urllink = str(link.encode('utf-8'))
# picking up URLs that have a PDF
if ".pdf" in urllink:
url_pdf = str(link['href'].encode('utf-8'))
url_pdf = url_pdf[2:] # Remove first two characters
url_pdf = url_pdf[:-1] # Remove last characters
#print(url_pdf)
PDF_links.append(url_pdf)
count = count + 1
url = PDF_links[0]
r = requests.get(url)
f = io.BytesIO(r.content)
reader = PdfFileReader(f)
contents = reader.getPage(0).extractText().split("\n")
print(contents)