How to extract data from website into HDFS

Viewed 14

I'm trying to extract data from website into HDFS, i'm using this code but it's not working, there is no error but i can't find the file in HDFS!

import requests
from pathlib import Path
import os
from datetime import date
from hdfs import InsecureClient
date= date.today()
date

def downloadFile(link, destfolder):
    r = requests.get(link,stream=True)
    filename="datanew1"+ str(date)+".xls"
    downloaded_file = open(os.path.join(destfolder, filename), 'wb')
    client= InsecureClient('http://hdfs-namenode.default.svc.cluster.local:50070', user='hdfs')
    with client.download('/data/test.csv')
        for chunk in r.iter_content(chunk_size=256):
            if chunk:
                downloaded_file.write(chunk)
           
link="https://api.worldbank.org/v2/fr/indicator/FP.CPI.TOTL.ZG?downloadformat=excel"
Path('http://hdfs-namenode.default.svc.cluster.local:50070/data').mkdir(parents=True, exist_ok=True)
downloadFile(link, 'http://hdfs-namenode.default.svc.cluster.local:50070/data')
0 Answers
Related