How to match and remove wikipedia refences with python and re

user_15746556 0
Viewed 309
from bs4 import BeautifulSoup
import requests
import time
import keyboard
import re

def searchWiki():
    search = input("What do you want to search for? ").replace(" ", "_").replace("'", "%27")
    url = f"https://en.wikipedia.org/wiki/{search}"
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
    page = requests.get(url, headers=headers)
    soup = BeautifulSoup(page.content, "html.parser")
    title = soup.find("title").get_text()
    info = soup.find_all("p")
    print("Press enter to read the next paragraph")
    print(title)
    print(url)
    for p in info:
        print(p.text.strip())
        keyboard.wait('enter')



searchWiki()

For example, search for Tom Holland. It should come up with this:

Thomas Stanley Holland (born 1 June 1996)[1] is an English actor. A graduate of the BRIT School in London...

Want I want to do is remove the refence number and the brackets.

python-re python-requests wikipedia beautifulsoup python
2 Answers

You can do it using regular expressions.

For example with your p var:

import re

line = p.text.strip()
new_line = re.sub("\[[0-9]+\]", '', line)
print(new_line)
user_122386400

All of the footnotes are under the class reference, you can remove them using the decompose() method:

for tag in soup.find_all(class_="reference"):
    tag.decompose()

import requests
from bs4 import BeautifulSoup


def searchWiki():
    search = input("What do you want to search for? ").replace(" ", "_").replace("'", "%27")
    url = f"https://en.wikipedia.org/wiki/{search}"
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
    page = requests.get(url, headers=headers)
    soup = BeautifulSoup(page.content, "html.parser")
    # Remove all footnotes under the `reference` class
    for tag in soup.find_all(class_="reference"):
        tag.decompose()

    title = soup.find("title").get_text()
    info = soup.select("p")
    print("Press enter to read the next paragraph")
    print(title)
    print(url)
    for p in info:
        print(p.text.strip())


searchWiki()
user_123497340
Related