webscraping bus stops with beautifulsoup

user_13134420 0
Viewed 62

I am trying to web scrape bus stop names for a given line, here is an example page for line 212 https://www.m2.rozkladzik.pl/warszawa/rozklad_jazdy.html?l=212. I want to have as an output two lists, one with bus stop names in one direction and the other list with another direction. (It's clearly seen on the web page). I managed to get all names in one list with

import requests
from bs4 import BeautifulSoup


def download_bus_schedule(bus_number):
    URL = "http://www.m2.rozkladzik.pl/warszawa/rozklad_jazdy.html?l=" + bus_number
    r = requests.get(URL)
    soup = BeautifulSoup(r.content,
                         'html5lib')
    print(soup.prettify())
    all_bus_stops = []
    table = soup.find_all('a')
    for element in table:
        if element.get_text() in all_bus_stops:
            continue
        else:
            all_bus_stops.append(element.get_text())
    return all_bus_stops

print(download_bus_schedule('212'))

I guess the solution would be to somehow divide the soup into two parts.

web-scraping divide beautifulsoup list python
4 Answers
import requests
from bs4 import BeautifulSoup


def download_bus_schedule(bus_number):
    URL = "http://www.m2.rozkladzik.pl/warszawa/rozklad_jazdy.html?l=" + bus_number
    r = requests.get(URL)
    soup = BeautifulSoup(r.content,
                         'html5lib')

    bus_stops_1 = []
    bus_stops_2 = []

    directions = soup.find_all("ul", {"class":"holo-list"})
    
    for stop in directions[0].find_all("a"):
        if stop not in bus_stops_1:
            bus_stops_1.append(stop.text.strip())

    for stop in directions[1].find_all("a"):
        if stop not in bus_stops_2:
            bus_stops_2.append(stop.text.strip())
    
    all_bus_stops = (bus_stops_1, bus_stops_2)

    return all_bus_stops

print(download_bus_schedule('212')[0])
print(download_bus_schedule('212')[1])
user_151306710

You can use the bs4.element.Tag.findAll method:

import requests
from bs4 import BeautifulSoup


def download_bus_schedule(bus_number):
    all_bus_stops = []
    URL = "http://www.m2.rozkladzik.pl/warszawa/rozklad_jazdy.html?l=" + bus_number
    r = requests.get(URL)
    soup = BeautifulSoup(r.content, 'html.parser')
    for s in soup.select(".holo-list"):
        bus_stops = []
        for f in s.findAll("li"):
            if f.text not in bus_stops:
                bus_stops.append(f.text)
        all_bus_stops.append(bus_stops)
    return all_bus_stops

print(download_bus_schedule('212'))

Output:

[['Pl.Hallera', 'Pl.Hallera', 'Darwina', 'Namysłowska', 'Rondo Żaba', 'Rogowska', 'Kołowa', 'Dks Targówek', 'Metro Targówek Mieszkaniowy', 'Myszkowska', 'Handlowa', 'Metro Trocka', 'Bieżuńska', 'Jórskiego', 'Łokietka', 'Samarytanka', 'Rolanda', 'Żuromińska', 'Targówek-Ratusz', 'Św.Wincentego', 'Malborska', 'Ch Targówek'], 
 ['Ch Targówek', 'Ch Targówek', 'Malborska', 'Św.Wincentego', 'Targówek-Ratusz', 'Żuromińska', 'Gilarska', 'Rolanda', 'Samarytanka', 'Łokietka', 'Jórskiego', 'Bieżuńska', 'Metro Trocka', 'Metro Trocka', 'Metro Trocka', 'Handlowa', 'Myszkowska', 'Metro Targówek Mieszkaniowy', 'Dks Targówek', 'Kołowa', 'Rogowska', 'Rondo Żaba', '11 Listopada', 'Bródnowska', 'Szymanowskiego', 'Pl.Hallera', 'Pl.Hallera']]
user_135524700

I may have misunderstood as I do not know Polish but see if this helps.

from bs4 import BeautifulSoup
import requests

url = 'https://www.m2.rozkladzik.pl/warszawa/rozklad_jazdy.html?l=212'

resp = requests.get(url)
soup = BeautifulSoup(resp.content, "html.parser")

d = {}
for h2 in soup.select('h2.holo-divider'):
    d[h2.text] = []
    ul = h2.next_sibling
    for li in ul.select('li'):
        if li.a.text not in d[h2.text]:
            d[h2.text].append(li.a.text)

from pprint import pprint

pprint(d)
Admin1

As all stops are encapsulated in the next un-ordered list, you could use the find_next function of bs4. e.g.

URL = f"http://www.m2.rozkladzik.pl/warszawa/rozklad_jazdy.html?l={bus_number}"
r = requests.get(URL)
soup = BeautifulSoup(r.content,
                  'html5lib')
directions = ["Ch Targówek","Pl.Hallera"]
result = {}
for direction in directions:
  header = soup.find(text=direction)
  list = header.find_next("ul")
  stops_names = [stop.get_text() for stop in list]
  result[direction] = stops_names

return result

Plus you might want to use f-string to format your strings as it improves reading and is less error prone.

user_98413370
Related