How to crawl images inside links of a page

Mike D Hovhannisyan 0
Viewed 40

I need the crawler to go to the links inside a website and scan images there. I've managed to get this far but I'm confused.

I'm trying to do something like this but I'm sure there's gonna be an easier way.

from bs4 import *
import requests as rq
import os
import sys
from urllib.parse import urlparse

page_url = sys.argv[1]
depth = int(sys.argv[2])
crawl = str(page_url)
r2 = rq.get('https://www.' + crawl + '' + '/')
soup2 = BeautifulSoup(r2.text, "html.parser")
links = []
images = []

link_urls = soup2.select('a')


def url_validator(link):
    try:
        result = urlparse(link)
        return all([result.scheme, result.netloc])
    except:
        return False


def crawl_images(link):
    requested_link = rq.get(link)
    images = BeautifulSoup(requested_link.text, "html.parser")
    image = images.select('img')
    for img in image:
        print(img['src'])
        return img['src']


for link_url in link_urls[:depth]:
    links.append(link_url['href'])
    for link in links:

        # print(link)

        if url_validator(link):
            crawl_images(link)

I try python3 new_crawler.py imdb.com 3 which should print sources of images crawled in 3 links inside imdb.com but it's not printing anything.

beautifulsoup python
2 Answers

You want to crawl through the images, correct? Try this:


from bs4 import BeautifulSoup
import requests as rq


URL = ""
source = rq.get(URL)
soup = BeautifulSoup(source.text, "html.parser")


image_links = soup.find_all("img")

for img in image_links:    
    print(img['src'])

Add the website's url to the constant URL that you are trying to scrap. The page's img tags should all be saved in the image_links variable.

Seraph0

This is what I ended up with. It's not working how it's supposed to but the time for the task is up and I decided to share anyway.

from bs4 import *
import requests as rq
import sys
from urllib.parse import urlparse
import json

page_url = sys.argv[1]
depth = int(sys.argv[2])
crawl = str(page_url)
r2 = rq.get('https://www.' + crawl + '' + '/')
soup2 = BeautifulSoup(r2.text, "html.parser")
link_urls = soup2.select('a')
links = []
images_sources = []


def url_validator(link):
    try:
        result = urlparse(link)
        return all([result.scheme, result.netloc])
    except:
        return False


def crawl_images(link):
    requested_link = rq.get(link)
    images = BeautifulSoup(requested_link.text, "html.parser")
    image = images.select('img')
    for img in image:
        images_sources.append(img['src'])
        results = {
            "imageUrl": img['src'],
            "sourceUrl": link,
            "depth": depth
        }
        json_object = json.dumps(results)
        with open("results.json", "w") as f:
            f.write(json_object)
            f.close()
            return results


for link_url in link_urls[:depth]:
    links.append(link_url['href'])
    for link in links:
        if url_validator(link):
            crawl_images(link)
Mike D Hovhannisyan0
Related