How to create one folder to store images from a website scrape

user_16460269 0
Viewed 291

I have written the following code to extract images for each product from a website scrape. I am very new to this and am unsure how to stop it creating a new folder for each product. Currently it creates a new folder called Whiteline Images inside the previous folder also titled whiteline images - easy enough to fix manually when its 5 products - not so much when I change it to 500+!! I know where in the code its doing this... just unsure how to fix it. Any help is appreciated!

import requests
from bs4 import BeautifulSoup
import os

def imagedown(url,folder):
try:
        os.mkdir(os.path.join(os.getcwd(), folder))
    except:
        pass    
    os.chdir(os.path.join(os.getcwd(), folder)) 
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'html.parser')

    images = soup.findAll('img',{"src":True})

    for index, image in enumerate(images, start=1):
        if(image.get('src').startswith('https://imageapi.partsdb.com.au/api/Image')):
            link = (image.get('src'))
            name = f'{soup.find("div", {"class": "head2BR"}).text} ({index})'

            with open(name + '.jpg','wb') as f:
                im = requests.get(link) 
                f.write(im.content)
                print('Writing:', name)

imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=KBR15', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=W13374', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=BMR98', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=W51210', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=W51211', 'whiteline_images')
web-scraping directory image python
1 Answers

Instead of changing directories, use os.path.join when writing the image to the directory:

import requests, os
from bs4 import BeautifulSoup
def imagedown(url, folder):
   if not os.path.isdir(folder): #cleaner to use os.path.isdir when checking for folder existence
      os.mkdir(folder)
   soup = BeautifulSoup(requests.get(url).text, 'html.parser')
   for index, image in enumerate(soup.findAll('img',{"src":True}), start=1): 
      if image.get('src').startswith('https://imageapi.partsdb.com.au/api/Image'):
          link = image.get('src')
          name = f'{soup.find("div", {"class": "head2BR"}).text} ({index})'
           with open(os.path.join(folder, name + '.jpg'), 'wb') as f: #join folder name to new image name
               im = requests.get(link) 
               f.write(im.content)

Edit: updated solution:

def imagedown(url, folder):
   if not os.path.isdir(folder): #cleaner to use os.path.isdir when checking for folder existence
      os.mkdir(folder)
   soup = BeautifulSoup(requests.get(url).text, 'html.parser')
   for i, a in enumerate(soup.select('img:is(.mainman, .thumbbot)'), 1):
        name = soup.select_one('div.head2BR').text+f'({i})'
        with open(os.path.join(folder, name + '.jpg'), 'wb') as f: #join folder name to new image name
           im = requests.get(a['src']) 
           f.write(im.content)

imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=KBR15', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=W13374', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=BMR98', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=W51210', 'whiteline_images')
imagedown('https://www.whiteline.com.au/product_detail4.php?part_number=W51211', 'whiteline_images')
user_73267380
Related