Databricks web scraping

Viewed 716

I wrote some python code to automatically download a csv file from the internet. The code works when it runs on my local computer but not when I run it on DataBricks. The problem is that I don't know how to save it to my DBFS: folder = "/dbfs/mnt/fmi-import/DNB Scenariosets/". The code does execute but the file is nowhere to be found.

import requests
from bs4 import BeautifulSoup
import re

url_scenariosets_dnb = 'https://www.dnb.nl/voor-de-sector/open-boek-toezicht-sectoren/pensioenfondsen/haalbaarheidstoets/uitvoering-en-normen/scenarioset-haalbaarheidstoets-pensioenfondsen/'
folder = "/dbfs/mnt/fmi-import/DNB Scenariosets/"


class dataset_downloader:
   
    def __init__(self,url):
        self.url=url
    
    def scrape(self):
        reqs = requests.get(self.url, verify=False)
        soup = BeautifulSoup(reqs.text, 'html.parser')
        self.urls=[]
        for link in soup.find_all('a'):
            self.urls.append(link.get('href'))
        return self.urls
    
    def filter_scenarioset(self):
        # Search data based on regular expression in the list
        self.scenarioset_links=[]
        [self.scenarioset_links.append('https://www.dnb.nl'+val) for val in self.urls
                if re.search(r'hbt-scenarioset-10k', val)]
        return self.scenarioset_links
    
    def download_file(self, year, quarter):
        try:
            self.downloadlink=[]
            [self.downloadlink.append(val) for val in self.scenarioset_links
                if re.search(r'hbt-scenarioset-10k-{}q{}'.format(year,quarter),val)]
            filename='hbt-scenarioset-10k-{}q{}.xlsx'.format(year,quarter) 
            with requests.get(self.downloadlink[0]) as req:
                with open(filename, 'wb') as f:
                    for chunk in req.iter_content(chunk_size=8192):
                        if chunk:
                            f.write(chunk)
                return "/dbfs/mnt/fmi-import/DNB Scenariosets/"+filename
        except Exception as e:
            print(e)
            return None

#%% EXECUTE
download = dataset_downloader(url_scenariosets_dnb)
download.scrape()
download.filter_scenarioset()
download.download_file(2020,2)  # select year and quarter


Do you have any suggestion on how you can download a csv file with databricks and save it to a DBFS folder? Thank you in advance!

Vincent

1 Answers

Problem is that in code you have only filename without path to folder. It should be:

with open(folder + filename, 'wb')
Related