Scraping aspx webpage with Python

user_5261759 0
Viewed 160

I am new to web scraping and trying to scape the following webpage: http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx

Here is what I have tried to access the contents from the webpage in Python. I am only trying to scrape the contents, for now, the parsing will come next.

import requests
from bs4 import BeautifulSoup

r_obj = requests.Session()

url = 'http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx'

r_soup = r_obj.get(url)

soup = BeautifulSoup(r_soup.content,'lxml')

hidden_inputs = soup.find_all("input",type="hidden")

data = {
        hidden_inputs[0]['name']:hidden_inputs[0]['value'],
        hidden_inputs[1]['name']:hidden_inputs[1]['value'],
        # hidden_inputs[2]['name']:hidden_inputs[2]['value'],
        # hidden_inputs[3]['name']:hidden_inputs[3]['value'],
        hidden_inputs[4]['name']:hidden_inputs[4]['value'],
        hidden_inputs[5]['name']:hidden_inputs[5]['value'],
        'allmarket$ctl00$ctl03$ctl00': 'Tomato, Potato, Onion',
        'allmarket$ctl00$ctl05$ctl00': 'Lahore',
        'allmarket$ctl00$ctl07$ctl00': '08/13/2021',
        'allmarket$ctl00$ctl03$ctl03$ctl01': 'on',
        'allmarket$ctl00$ctl03$ctl03$ctl02': 'on',
        'allmarket$ctl00$ctl03$ctl03$ctl03': 'on',
        'allmarket$ctl00$ctl05$ctl03$ctl01': 'on',
        'allmarket$ctl04':'', 
        'allmarket$ctl05':'', 
        'allmarket$ctl06': '0',
        'allmarket$ctl07': '0',
        'allmarket$ctl00$ctl00': 'View Report',
        }

url_needed = 'http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx'

final = r_obj.post(url_needed,verify=False,data=data)
soup1 = BeautifulSoup(final.content,"lxml")
detail_tab = soup1.find_all("table")

After the above code is run, the detail_tab does not contain the relevant table data but only the top headers. What am I missing?

python-requests web-scraping beautifulsoup asp.net python
1 Answers

To get data from the site, you can use next example (there are several <iframe> that needs to be parsed):

import bs4
import requests

url = "http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx"

with requests.session() as s:
    soup = BeautifulSoup(s.get(url).content, "html.parser")

    data = {}
    for inp in soup.select("input"):
        name = inp["name"]
        if name.startswith("all"):
            continue
        data[name] = inp["value"]

    data["allmarket$ctl00$ctl03$ctl00"] = "Tomato, Potato, Onion"
    data["allmarket$ctl00$ctl05$ctl00"] = "Lahore"
    data["allmarket$ctl00$ctl07$ctl00"] = "07/01/2021"
    data["allmarket$ctl00$ctl03$ctl03$ctl01"] = "on"
    data["allmarket$ctl00$ctl03$ctl03$ctl02"] = "on"
    data["allmarket$ctl00$ctl03$ctl03$ctl03"] = "on"
    data["allmarket$ctl00$ctl05$ctl03$ctl01"] = "on"
    data["allmarket$ctl01$ctl01$ctl02"] = "1"
    data["allmarket$ctl01$ctl05$ctl00"] = "Select a format"
    data["allmarket$ctl04"] = ""
    data["allmarket$ctl05"] = ""
    data["allmarket$ctl06"] = "1"
    data["allmarket$ctl07"] = "0"
    data["allmarket$ctl00$ctl00"] = "View Report"

    r = s.post(
        url,
        headers={
            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0",
            "X-MicrosoftAjax": "Delta=true",
        },
        data=data,
    )

    # 1. iframe

    soup = BeautifulSoup(r.content, "html.parser")
    soup = BeautifulSoup(
        s.get(
            "http://www.amis.pk"
            + soup.select_one("#ReportFrameallmarket")["src"]
        ).content,
        "html.parser",
    )

    # 2. iframe

    soup = BeautifulSoup(
        s.get("http://www.amis.pk" + soup.select_one("#report")["src"]).content,
        "html.parser",
    )

    for tr in soup.select("tr:not(:has(tr))"):
        tds = [td.text for td in tr.select("td")]
        print(*tds, sep=" ")

Prints:

Arrival Source & Target Detail in Quintal
    
ACrop Name Arrived From Province Name Arrived At Quantity
Onion Jacobabad Sindh Lahore 700
Onion Kahrorpacca Punjab Lahore 1500
Onion Swat KPK Lahore 600
Potato Abbottabad KPK Lahore 700
Potato Gilgit Gilgit Baldistan Lahore 400
Potato Lahore Punjab Lahore 1800
Potato Mansehra KPK Lahore 800
Tomato Peshawar KPK Lahore 2500
user_100359850
Related