Unable to download pdf files from a website

user_13548040 0
Viewed 74

Hi I have following code which i want to download the pdf from a website "https://www.journal-officiel.gouv.fr/balo/recherche/resultats?parutionDateStart=2021-05-17&parutionDateEnd=2021-05-17&_token=0oP3_cJ2xZ10SbEEGoNdP6vUpAIv5nBkrTZptI0Nzd8"

Here is the script that i am using to download the files but none of pdf are getting downloaded .

Though it is not giving any error it is creating a blank folder every time

from selenium import webdriver
from selenium.webdriver.support.ui import Select
driver = webdriver.Chrome(executable_path='C:\\Users\\u6080267\\Documents\\chromedriver.exe')
driver.get("https://www.journal-officiel.gouv.fr/balo/recherche/")
link = driver.find_element_by_xpath("//a[contains(@href,'token')]")
link.click()

url1 = driver.current_url
import urllib.request
from bs4 import BeautifulSoup
import os
import urllib
from datetime import datetime
import requests
HEADERS = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'}
def get_urls(url):
    
    url = url1    
    #url = https://www.journal-officiel.gouv.fr/balo/recherche/resultats?parutionDateStart=2021-05-03&parutionDateEnd=2021-05-03&_token=48BMi0HUW0CZJVdbccoO_wX9IzRJfglO8Uq-K0lfMNg
    req = urllib.request.Request(url, None, HEADERS)
    opener = urllib.request.build_opener()
    content = opener.open(req).read()  
    soup = BeautifulSoup(content, "html.parser")
    soup.prettify()      
    urls = {}
    for anchor in soup.findAll('a', href=True): #Going inside links        
        if "/balo/document" in anchor.get('href'):            
            name = anchor.get('href')[(anchor.get('href').rindex("=")+1):]
            url = "https://www.journal-officiel.gouv.fr/" + anchor.get('href')

            if name not in urls:
                urls[name]=url

    return urls
def download(urls, path):
    os.chdir(path)
    for name, url in urls.items():
        try:
            res = requests.get(url, allow_redirects=True)            
            # programmatic access requires a form to be submitted. On agreeing the consent, the pdfurl can be used
            soup = BeautifulSoup(res.content, "html.parser")
            soup.prettify()
            for pdfurl in soup.findAll(attrs={"name": "pdfURL"}):
                downloadurl = "https://www.journal-officiel.gouv.fr/" + pdfurl.get('value')
                res = requests.get(downloadurl)
                open(name + ".pdf", 'wb').write(res.content)
                print ("Downloaded", name + ".pdf")
            
        except Exception as e:
            print ("Failed to download", name, ", because of", e)   

def main():
    pathToStoreFiles = os.getcwd() + "\\" + datetime.today().strftime('%Y-%m-%d')
    os.makedirs(pathToStoreFiles)
    urls = get_urls('')
    download(urls, pathToStoreFiles)
if __name__ == "__main__":
    main()
web-scraping beautifulsoup python
1 Answers

Apparently, you're not getting the right pdf download links. Also, you're making the scraping a lot more complicated than it has to be, namely, there's no need for the heavy guns of selenium.

The token and the query url are easily reachable with bs4.

You can then use it to get the results HTML and parse it for pdf links.

Here's how to download the first 50 files.

import time

import requests
from bs4 import BeautifulSoup

main_url = "https://www.journal-officiel.gouv.fr"
search_path = "/balo/recherche/"


def wait_a_bit(wait_for: float = 1.5):
    time.sleep(wait_for)


with requests.Session() as connection:
    connection.headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36"

    search_url = (
        BeautifulSoup(connection.get(f"{main_url}{search_path}").text, "lxml")
        .find_all("a", class_="aide-link")[-1]["href"]
    )

    pdf_links = (
        f'{main_url}{link.find("a")["href"]}' for link in
        BeautifulSoup(
            connection.get(f"{main_url}{search_url}&limit=50").text, "lxml"
        )
        .select(".download-link")
    )

    for pdf_link in pdf_links:
        print(f"Fetching {pdf_link}")
        pdf_file = connection.get(pdf_link).content
        with open(f'{pdf_link.rsplit("/")[-1]}.pdf', "wb") as output:
            output.write(pdf_file)
        wait_a_bit()

Output:

Fetching https://www.journal-officiel.gouv.fr/balo/document/202105172101761-59
Fetching https://www.journal-officiel.gouv.fr/balo/document/202105172101784-59
Fetching https://www.journal-officiel.gouv.fr/balo/document/202105172101798-59
Fetching https://www.journal-officiel.gouv.fr/balo/document/202105172101801-59
Fetching https://www.journal-officiel.gouv.fr/balo/document/202105172101810-59

and more ...

All files are saved in the script's current directory as:

202105172101675-59.pdf
202105172101686-59.pdf
202105172101687-59.pdf
202105172101688-59.pdf
202105172101697-59.pdf
...
user_61067910
Related