how to download image from a website using beautiful soup

user_14157675 0
Viewed 55

I am using python and Beautiful soup to download & locally save the image embedded/uploaded in this post: https://gall.dcinside.com/mgallery/board/view/?id=irudagall&no=18886&page=2027

However, it seems the image file is not downloadable. This is what I have so far:

from bs4 import BeautifulSoup
from requests import get
from PIL import Image

headers = {
    "Connection" : "keep-alive",
    "Cache-Control" : "max-age=0",
    "sec-ch-ua-mobile" : "?0",
    "DNT" : "1",
    "Upgrade-Insecure-Requests" : "1",
    "User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36",
    "Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Sec-Fetch-Site" : "none",
    "Sec-Fetch-Mode" : "navigate",
    "Sec-Fetch-User" : "?1",
    "Sec-Fetch-Dest" : "document",
    "Accept-Encoding" : "gzip, deflate, br",
    "Accept-Language" : "ko-KR,ko;q=0.9"
    }


test_url = 'https://gall.dcinside.com/mgallery/board/view/?id=irudagall&no=18887&page=2027'
test_res = requests.get(test_url, headers=headers)
test_soup = BeautifulSoup(test_res.text, "lxml")

img_url = test_soup.find("ul", {"class": "appending_file"}).find("a")['href']
img = Image.open(requests.get(image_url, stream = True).raw)
img.save('image.jpg')
beautifulsoup python
2 Answers

Okay, this is a tricky one. If you inspect your Network calls in your browser (Click on F12 ) while the request is being fetched, you'll see how the request was made - what parameter/headers were requested.

If you then right-click on the Network call, you'll see a button to copy as Curl

enter image description here

You can then copy the code to curlconverter.com and it will give you the python code.

So, you can directly get the image, using the correct parameters:

import requests


headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Language': 'en-US,en;q=0.9',
    'Connection': 'keep-alive',
    # Requests sorts cookies= alphabetically
    # 'Cookie': 'PHPSESSID=3121e08010ef697de8f66ab58c5f0ec2; ci_c=128cd80d9f5fdfc51c7c61edb5645ec6; ck_lately_gall=9XC; csid=0029f829d7ed21adb420a27f300ddb9d1a0a7da0b6d5a363db86afe5b658ca4807eb780658d416; ck_l_f=l; __utma=118540316.2061578954.1656510647.1656510647.1656510647.1; __utmc=118540316; __utmz=118540316.1656510647.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _ga=GA1.2.2061578954.1656510647; _gid=GA1.2.1765121273.1656510647; _cc_id=4e3678109cd1c9313694988a7a67c933; panoramaId_expiry=1657115450856; panoramaId=a32107449d1dfe505a1342c855274945a702958a38612ff1e8d52232bab45444; alarm_new=1; ck_img_view_cnt=3; __utmt=1; __utmb=118540316.6.10.1656510647; last_alarm=1656511621; gallRecom=MjAyMi0wNi0yOSAyMzowNzo0My9jMzAwODJiNjRjYjQwZGRkMzZkZGZmNDUzODM3YjM3ZDFiMTNkNjg4OTlhNGFiYTM2YTdlY2IyM2ZkZGNiNDhk; service_code=21ac6d96ad152e8f15a05b7350a2475909d19bcedeba9d4face8115e9bc1fa4c1fb428016691909c29fb3e7c9f6390a488eb8106cf73377811f8e19d62b91109c8222c6181a534254c3cc6086a49a064ad17a08132571b4449d6bc0e10ec08afe5a9c3f3890340535a1bb0913e81814db61c3e9ff51bc2807ddc90ae309289dccde248ba6d7959046766f07235a7d51dd243176d34eccef23feebeaa5aa212e3785600f5573e2caede3b46d6b75496ce0605fe2cc763951c5632903a1dd0f4fa5b05e1a057f9e20a31',
    'Referer': 'https://gall.dcinside.com/',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-site',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
    'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="102", "Google Chrome";v="102"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
}

params = {
    'no': '24b0d769e1d32ca73cec85fa11d028318672c324147e3f6e5672e000c8e1092199c885c51d343f31d2f953fd08aa818e5ca9106941082978e5453bba01a55e89609815c87ddee88307576533477dc4',
    'f_no': '20210110_215524.jpg',
}

response = requests.get('https://image.dcinside.com/download.php', params=params, headers=headers)

with open('image.jpg', 'wb') as f:
    f.write(response.content)
user_123497340

First you need to find the number of the picture in order to transfer it for download. We use BeautifulSoup for this. Then we simply pass the found number to the download as a parameter

import requests
from bs4 import BeautifulSoup


url = 'https://gall.dcinside.com/mgallery/board/view/?id=irudagall&no=18887&page=2027'
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36",
    "Accept": "*/*",
    "Referer": "https://gall.dcinside.com/"
    }
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'lxml')
image_no = soup.find('div', class_='write_div').find('img').get('src').split('=')[1]
with open('image.jpg', 'wb') as img_file:
    response = requests.get('https://image.dcinside.com/download.php', {'no': image_no}, headers=headers)
    img_file.write(response.content)
user_65932870
Related