BeautifulSoup: 6k records - but stops after parsing 20 lines

user_525865 0
Viewed 229

For the goal to create a quick overview on a set of opportunities for free volunteering in Europe

It is aimed to get all the 6k target-pages: https://europa.eu/youth/volunteering/organisation/48592 see below - the images and the explanation and description of the aimed goals and the data which are wanted.

We fetch ..

https://europa.eu/youth/volunteering/organisation/50162
https://europa.eu/youth/volunteering/organisation/50163

and so forth and so forth

Since we have more than 6000 records - I admit that I get the results. But the script only gives back 20 records - i.e. 20 lines.

See my current approach: I run this mini-approach here:

import requests
from bs4 import BeautifulSoup
import re
import csv
from tqdm import tqdm

first = "https://europa.eu/youth/volunteering/organisations_en?page={}"
second = "https://europa.eu/youth/volunteering/organisation/{}_en"

def catch(url):
    with requests.Session() as req:
        pages = []
        print("Loading All IDS\n")
        for item in tqdm(range(0, 347)):
            r = req.get(url.format(item))
            soup = BeautifulSoup(r.content, 'html.parser')
            numbers = [item.get("href").split("/")[-1].split("_")[0] for item in soup.findAll(
                "a", href=re.compile("^/youth/volunteering/organisation/"), class_="btn btn-default")]
            pages.append(numbers)
        return numbers

def parse(url):
    links = catch(first)
    with requests.Session() as req:
        with open("Data.csv", 'w', newline="", encoding="UTF-8") as f:
            writer = csv.writer(f)
            writer.writerow(["Name", "Address", "Site", "Phone",
                             "Description", "Scope", "Rec", "Send", "PIC", "OID", "Topic"])
            print("\nParsing Now... \n")
            for link in tqdm(links):
                r = req.get(url.format(link))
                soup = BeautifulSoup(r.content, 'html.parser')
                task = soup.find("section", class_="col-sm-12").contents
                name = task[1].text
                add = task[3].find(
                    "i", class_="fa fa-location-arrow fa-lg").parent.text.strip()
                try:
                    site = task[3].find("a", class_="link-default").get("href")
                except:
                    site = "N/A"
                try:
                    phone = task[3].find(
                        "i", class_="fa fa-phone").next_element.strip()
                except:
                    phone = "N/A"
                desc = task[3].find(
                    "h3", class_="eyp-project-heading underline").find_next("p").text
                scope = task[3].findAll("span", class_="pull-right")[1].text
                rec = task[3].select("tbody td")[1].text
                send = task[3].select("tbody td")[-1].text
                pic = task[3].select(
                    "span.vertical-space")[0].text.split(" ")[1]
                oid = task[3].select(
                    "span.vertical-space")[-1].text.split(" ")[1]
                topic = [item.next_element.strip() for item in task[3].select(
                    "i.fa.fa-check.fa-lg")]
                writer.writerow([name, add, site, phone, desc,
                                 scope, rec, send, pic, oid, "".join(topic)])


parse(second)

But this stops after parsing 20 results

Note: I am wanting to return pages not numbers.

Since I want to iterate over pages not numbers; but anyway - if I change from return numbers to return pages - I get no better results.

Here I seem to have some mistakes: I guess that the catch function has a fancy mistake in it: here we are returning numbers, but I'm pretty sure that this is a mistake: we re intending to be returning pages, which means when we iterate over the results from catch(first) in the other function, we are not iterating over everything that is wanted. I guess that I need to include a fix: we need to return pages at the bottom of that function, instead of doing return numbers

That said: since I want to iterate over pages not numbers; but anyway - if I change from return numbers to return pages - I get no better results.

Any idea - how to get the parser to give out all the 6k results.

request beautifulsoup csv python
1 Answers

You can use this example to parse the the pages:

import pandas
import requests
import pandas as pd
from bs4 import BeautifulSoup


def safe_get(to_find, what_next, if_not_found="N/A"):
    if to_find:
        return what_next(to_find)
    return if_not_found


first_url = "https://europa.eu/youth/volunteering/organisations_en?page={}"

links = []
for page in range(0, 3):  # <--- increase number of pages here
    u = first_url.format(page)
    soup = BeautifulSoup(requests.get(u).content, "html.parser")

    for a in soup.select("h5 > a"):
        links.append("https://europa.eu" + a["href"])

data = []
for l in links:
    print(l)
    soup = BeautifulSoup(requests.get(l).content, "html.parser")

    name = safe_get(soup.select_one("h5"), lambda t: t.text)
    address = safe_get(
        soup.select_one(".fa-location-arrow"),
        lambda t: t.parent.get_text(strip=True),
    )
    link = safe_get(
        soup.select_one(".fa-external-link"), lambda t: t.find_next("a")["href"]
    )
    phone = safe_get(
        soup.select_one(".fa-phone"), lambda t: t.find_next(text=True).strip()
    )
    desc = safe_get(
        soup.select_one("h3 ~ p"),
        lambda t: t.get_text(strip=True, separator="\n"),
    )
    scope = safe_get(
        soup.select_one(".fa-asterisk"),
        lambda t: t.find_next("span").get_text(strip=True),
    )
    receiving = safe_get(
        soup.select_one('td:-soup-contains("Receiving") ~ td'),
        lambda t: t.get_text(strip=True),
    )
    sending = safe_get(
        soup.select_one('td:-soup-contains("Sending") ~ td'),
        lambda t: t.get_text(strip=True),
    )
    pic = safe_get(
        soup.find("span", text=lambda t: t and t.startswith("PIC")),
        lambda t: t.text.split()[-1],
    )
    oid = safe_get(
        soup.find("span", text=lambda t: t and t.startswith("OID")),
        lambda t: t.text.split()[-1],
    )
    topics = ", ".join(
        [t.find_next(text=True).strip() for t in soup.select("p > .fa-check")]
    )

    data.append(
        (
            name,
            address,
            link,
            phone,
            desc,
            scope,
            receiving,
            sending,
            pic,
            oid,
            topics,
        )
    )

df = pd.DataFrame(
    data,
    columns=[
        "Name",
        "Address",
        "Site",
        "Phone",
        "Description",
        "Scope",
        "Rec",
        "Send",
        "PIC",
        "OID",
        "Topic",
    ],
)
print(df)
df.to_csv("data.csv", index=False)

Creates data.csv (screenshot from Libre Office):

enter image description here

user_100359850
Related