Select href link with python requests

user_14230862 0
Viewed 84

I was trying to use the request library in python to search here in stack overflow for questions in search bar, than take the first 3 links founded, get the content of the pages and sent to my notionAPI but I stuck on how did I take the html where the link is directing me in python

import requests
from bs4 import BeautifulSoup

#buscas online
def stackoverflow(question):
    questionAdjusted = question.replace(' ','+')
    Req = requests.get("https://pt.stackoverflow.com/search?q="+questionAdjusted)
    soup = BeautifulSoup(Req.text,"html.parser")
    questions = soup.select(".question-summary")

for que in questions:
    #print(que.select_one('.question-hyperlink').getText().replace('P: ',''))
    #print((que.select_one('.question-hyperlink').getText().replace('P: ', '').replace(' ','-').replace('--------','')))
    for link in soup.findAll('a', href=(que.select_one('.question-hyperlink').getText().replace('P: ', '').replace(' ','-').replace('--------',''))):
        print(link['href'])


stackoverflow('python database')

I just made this until now

python-requests python-3.x python
1 Answers

To get first 3 links + their description from the URL, you can use next example:

import requests
from bs4 import BeautifulSoup


def stackoverflow(question):
    url = "https://pt.stackoverflow.com/search"
    r = requests.get(url, params={"q": question})
    soup = BeautifulSoup(r.content, "html.parser")
    questions = soup.select(".question-hyperlink")

    for q in questions[:3]:  # <-- select ony first 3
        print(q.get_text(strip=True).replace("P: ", ""))
        print("https://pt.stackoverflow.com" + q["href"])
        print()


stackoverflow("python database")

Prints:

Select from e insert into em outro database com Python
https://pt.stackoverflow.com/questions/376648/select-from-e-insert-into-em-outro-database-com-python?r=SearchResults

Finalizando um projeto em python [duplicada]
https://pt.stackoverflow.com/questions/259591/finalizando-um-projeto-em-python?r=SearchResults

Erro de conexão com SQL Server 2012 com Python
https://pt.stackoverflow.com/questions/478779/erro-de-conex%c3%a3o-com-sql-server-2012-com-python?r=SearchResults
user_100359850
Related