How to scrape multiple hierarchy from a website's menu bar?

Rovsh 0
Viewed 39

I would like to scrape all the information present inside a website's menu/navigation bar while maintaining it's hierarchy. Example: https://www.trumpf.com/en_IN/ The image shows the levels of hierarchy in the contents of a the menu of a site

I have used beautiful soup to get the contents of the navigation bar however, I'm getting repeated information and hierarchical information is also lost.

from bs4 import BeautifulSoup as bs
import requests
 
url = 'https://www.trumpf.com/en_IN/'
html_text = requests.get(url).text
soup = bs(html_text, 'html.parser')
d = {}

for e in soup.select('nav > ul > li'):
    s = list(e.stripped_strings)
    d.update({s[0]: s[1:]})

print(d)

OUTPUT: enter image description here

Whereas I would like the output to have nested dictionaries and lists to preserve the hierarchy, for example;

{'Products':{'Machine & Systems': ['2D laser cutting machines', '3D laser cutting machines', 'Laser welding systems and the arc welding cell', 'Laser tube cutting machines', 'Marking systems', 'Additive production systems', 'Punching machines', 'Punch laser machines', 'Bending machines', 'Storage systems', 'Automation'], 'Lasers': [...]},'Solutions':{}...}
beautifulsoup web-scraping selenium python
1 Answers

Try:

import requests
from bs4 import BeautifulSoup


url = "https://www.trumpf.com/en_IN/"
soup = BeautifulSoup(requests.get(url).content, "html.parser")


def get_tree(soup, lvl=1):
    # are we at last level?
    if not soup.select_one(f".ux-iws-nav__lvl{lvl+1}-item"):
        return [
            a.get_text(strip=True)
            for a in soup.select(f".ux-iws-nav__lvl{lvl}-item a")[1:]
        ]

    out = {}
    for li in soup.select(f".ux-iws-nav__lvl{lvl}-item"):
        t = li.a.text
        if t.startswith("Overview"):
            continue
        out[t] = get_tree(li, lvl + 1)
    return out


print(get_tree(soup.select_one("nav")))

Prints:

{
    "Products": {
        "Machines & systems": [
            "2D laser cutting machines",
            "3D laser cutting machines",
            "Laser welding systems and the arc welding cell",
            "Laser tube cutting machines",
            "Marking systems",
            "Additive production systems",
            "Punching machines",
            "Punch laser machines",
            "Bending machines",
            "Storage systems",
            "Automation",
        ],
        "Lasers": [
            "Disk lasers",
            "Fiber laser",
            "Diode lasers",
            "Short and ultrashort pulse laser",
            "Marking lasers",
            "Pulsed lasers",
            "CO2 lasers",
            "EUV Drive Laser",
            "Sensor system",
            "Processing optics",
            "Technology packages",
            "Scientific lasers",
        ],
        "VCSEL solutions & photodiodes": [
            "Single & multimode VCSEL",
            "Datacom VCSELs & photodiodes",
            "Integrated VCSEL solutions",
            "VCSEL heating systems",
        ],
        "Real-time localization (RTLS)": [],
        "Power electronics": [
            "Plasma Excitation",
            "Induction generators",
            "Inverters",
            "TRUMPF Hüttinger Whitepaper",
        ],
        "Power tools": [
            "Battery machines",
            "Slitting shears",
            "Shear cutting",
            "Nibbler",
            "Profile nibbler",
            "Panel cutter",
            "Fiber Composite Nibbler",
            "Seam locker",
            "Power fastener",
            "Deburrers",
            "Beveler",
            "Slat cleaner",
        ],
        "Software": [],
        "Services": [],
    },
    "Solutions": {
        "Smart Factory": [
            "Smart Factory",
            "Starting out",
            "Step-by-step expansion",
            "Fully networked",
            "Smart Factory Consulting",
            "Our Smart Factories",
            "Lot size of 1",
            "Transparency with real-time locating systems",
            "Smart Material Flow",
        ],
        "Applications": [
            "Laser welding",
            "Arc welding",
            "Laser cutting",
            "EUV lithography",
            "Additive manufacturing",
            "Surface processing with the laser",
            "Microprocessing",
            "Laser marking",
            "Plasma technology",
            "Induction heating",
            "Cutting",
            "Joining",
            "Edge forming",
            "Punching and nibbling",
            "Bending",
            "Optical sensing",
        ],
        "Industries": [
            "Automotive",
            "Construction industry",
            "Sheet metal processing",
            "Dental",
            "Data communication",
            "Displays",
            "Electronics",
            "Air conditioning and energy technology",
            "Aviation and aerospace",
            "Machine and systems engineering",
            "Medical technology",
            "Commercial vehicles and transport",
            "Photovoltaics",
            "Watch and jewelry industry",
            "Tool and mold making",
            "Science",
        ],
        "Success stories": [],
        "Advantages of TRUMPF machines": [
            "Advantages of 2D laser cutting machines",
            "Advantages of bending machines",
            "Advantages of punching machines",
            "Advantages of punch laser machines",
            "Advantages of laser tube cutting machines",
            "Advantages of the TRUMPF VCSEL",
            "Advantages of additive production systems",
        ],
    },
    "Company": {
        "TRUMPF Group": [
            "Management Board",
            "Company profile",
            "Supervisory Board",
            "Locations",
            "Events and dates for your calendar",
            "Annual report",
            "Suppliers",
            "SYNCHRO",
            "Quality",
            "Company Principles",
            "Milestones in the history of TRUMPF",
            "Affiliated companies and other brands",
            "TRUMPF Venture",
            "Financial Services",
        ],
        "Responsibility": [
            "Culture",
            "Employees",
            "Education",
            "Society",
            "Products and supply chain",
            "Environment",
        ],
    },
    "Newsroom": [],
    "Careers": {
        "Vacancies": [],
        "TRUMPF as an employer": [
            "TRUMPF as an employer",
            "Benefits and opportunities",
            "Diversity",
            "Development opportunities",
            "International work",
        ],
        "Experienced professionals": [],
        "Graduates": [],
        "College students": [],
        "High school students": [],
        "How to apply": [],
        "People at TRUMPF": [],
    },
}
Andrej Kesely0
Related