Print different Tags from different ResultSet at the same time in BeautifoulSoup

Question

Good afternoon everybody, I am trying to extract some data about companies from the Italian version of YellowPages (paginegialle.it).
Basically the website is composed by containers, every container represents a company, and my goal is to extract different tags in this container at the same time not iterating every ResultSet I get from BeautifoulSoup

Check the code below:

from bs4 import BeautifulSoup
from urllib.request import urlopen
import pandas as pd
import random
import re


def search(cat, cit):
    cat = cat.replace(" ", "%20")
    cit = cit.replace(" ", "%20")
    end = False
    name_lst = addr_lst = phone_lst = web_lst = []
    pag = 1
    url = "https://www.paginegialle.it/ricerca/" + cat.lower() + "/" + cit.lower()
    while not end:
        try:
            with urlopen(url + "/p-" + str(pag)) as response:
                soup = BeautifulSoup(response, 'html.parser')
                print("Search in url -> ", url + "/p-" + str(pag))
                for container in soup.find_all("div", {"class": "container"}):
                    names = container.findChildren("h2", {"class": "itemTitle"})
                    phones = container.findChildren("span", {"class": "tel"})
                    websites = container.findChildren("a", {"data-pag": ["www", "Sito Personalizzato"]})
                    for name in names:
                        print(name.text.strip())
                    for wrap_addr in container.findChildren("div", {"class": "street-address"}):
                        addresses = wrap_addr.find_all("span", recursive=False)
                        for i in range(len(addresses)):
                            print(addresses[i].get_text())
                    for phone in phones:
                        print(phone.get_text().strip())
                    for website in websites:
                        print(
                            re.search("(?Phttps?://[^\s]+)", str(website).replace('"', '')).group("url"))
                pag += 1
        except Exception as e:
            end = True
            print(e)

category = "tabelloni luminosi"
city = "roma"

search(category, city)

As you can see I am referring to the for loops I have in my code (for name in names, for phone in phones etc.), I cannot think a way to print the whole sequence and not just all the names, all the addresses then the phones then the websites.

Partial output:

Search in url ->  https://www.paginegialle.it/ricerca/tabelloni%20luminosi/roma/p-1
Ea Pubblicita' # First company name
Neon Angher
Polineon
Ars Pubblicita' Insegne Luminose
Luminor - Insegne Led # Last company name
Via Aurelia, 352/A # First part of the address
00165
Roma
(RM)
Circonvallazione Nomentana, 132
00162
Roma
(RM)
Via Portuense, 956
00148
Roma
(RM)
Via Di Casal Del Marmo, 770
00166
Roma
(RM) # Last part of the address
06 87757538 # First phone number
06 8600063
06 65192493
328 6842073
06 45476895 # Last phone number
http://www.neonangher.it # First website
https://www.insegneluminosepolineon.com
https://www.insegneluminosecastelliromani.it # Last website

Output I need:

Search in url ->  https://www.paginegialle.it/ricerca/tabelloni%20luminosi/roma/p-1
Ea Pubblicita' # First company data
Via Aurelia, 352/A
00165
Roma
(RM)
06 87757538
NO WEBSITE
Neon Angher # Second company data
Via Aurelia, 352/A
00165
Roma
(RM)
06 8600063
http://www.neonangher.it
Polineon # Third company data
Circonvallazione Nomentana, 132
00162
Roma
(RM)
06 65192493
https://www.insegneluminosepolineon.com

# ETC.

That's what you are seeing on https://www.paginegialle.it/ricerca/tabelloni%20luminosi/roma/p-1
Is my goal clear??? Thank you everyone.

user_10035985 · Answer

This script gets all company names, adresses, telephones and websites from the page:

import requests
from bs4 import BeautifulSoup


url = 'https://www.paginegialle.it/ricerca/tabelloni%20luminosi/roma/p-1'

soup = BeautifulSoup(requests.get(url).content, 'html.parser')

for section in soup.select('section.vcard.listElement'):
    for t in section.select('[style="display: none;"]'):
        t.extract()
    title = section.select_one('h2[itemprop="name"]').get_text(strip=True)
    address = section.select_one('.street-address, .itemAddress').get_text(strip=True, separator='
')
    address = address.replace('
-
', '')
    tel = section.select_one('.tel').get_text(strip=True)
    www = section.select_one('[title^="sito web"][href]')
    www = www['href'] if www else 'NO WEBSITE'

    print(title)
    print(address)
    print(tel)
    print(www)
    print('-' * 80)

Prints:

Ea Pubblicita'
Via Aurelia, 352/A00165
Roma
(RM)
06 87757538
NO WEBSITE
--------------------------------------------------------------------------------
Neon Angher
Circonvallazione Nomentana, 13200162
Roma
(RM)
06 8600063
http://www.neonangher.it
--------------------------------------------------------------------------------
Polineon
Via Portuense, 95600148
Roma
(RM)
06 65192493
https://www.insegneluminosepolineon.com
--------------------------------------------------------------------------------
Ars Pubblicita' InsegneLuminose
Sede a Albano Laziale (RM)
Attivo anche a Roma
328 6842073
https://www.insegneluminosecastelliromani.it
--------------------------------------------------------------------------------

... an so on.

Print different Tags from different ResultSet at the same time in BeautifoulSoup

1 Answers