Can't stratify output in a customized way

Viewed 66

I've created a script to parse few data points from an htmlfile link and write the same to a csv file according to this format.

I do locate the fields accordingly using the selectors I've already defined within the script, but I can't stratify the output in the right way so that I can write them later to a csv file.

location of data points:

Nature of association

`from 1st table`
Purpose
In cash (Previous balance)

`from 2nd table`
Donor Name
Address

`from 3rd table`
Country Name
Amount

Here is what I've tried (I suppose the htmlfile link works):

import requests
from bs4 import BeautifulSoup

file_link = 'https://filebin.redpill-linpro.com/zj2qqc27va5fatm0/index.html'

res = requests.get(file_link)
soup = BeautifulSoup(res.text,"lxml")
nature_of_asso = soup.select_one("td:contains('Nature of association') + td").get_text(strip=True)

for purpose_tr in soup.select("table:has(> tr > td:nth-of-type(1) + td:contains('Purpose')) tr")[3:]:
    try:
        purpose = purpose_tr.select_one('td:nth-of-type(2)').get_text(strip=True)
    except AttributeError: purpose = ""
    try:
        in_cash = purpose_tr.select_one('td:nth-of-type(3)').get_text(strip=True)
    except AttributeError: in_cash = ""
    print(purpose,in_cash)

for donor_tr in soup.select("table:has(> tr > td:nth-of-type(1) + td:contains('Donor Name')) tr")[2:]:
    try:
        donor_name = donor_tr.select_one('td:nth-of-type(2)').get_text(strip=True)
    except AttributeError: donor_name = ""
    try:
        address = donor_tr.select_one('td:nth-of-type(3)').get_text(strip=True)
    except AttributeError: address = ""
    print(donor_name,address)

for country_tr in soup.select("table:has(> tr > td:nth-of-type(1) + td:contains('Country Name')) tr")[1:]:
    try:
        country = country_tr.select_one('td:nth-of-type(2)').get_text(strip=True)
    except AttributeError: country = ""
    try:
        amount = country_tr.select_one('td:nth-of-type(3)').get_text(strip=True)
    except AttributeError: amount = ""
    print(country,amount)

How can I arrange output as per the image above in order to write the same to a csv file?

2 Answers

You could use pandas for the whole thing and clean the tables, then left join the main DataFrame, with most rows, on the others, using Sl.No.

import pandas as pd

tables = pd.read_html('https://filebin.redpill-linpro.com/zj2qqc27va5fatm0/index.html')
df = tables[4]
df = df.iloc[2:-1, :3]
df.columns = df.iloc[0, :]
df.drop(labels = 2, axis = 0, inplace = True)

df_donor = tables[8]
df_donor = df_donor.iloc[:-2, :]
df_donor.columns = df_donor.iloc[0, :]
df_donor = df_donor.iloc[2:, :3]

df_country = tables[10]
df_country = df_country.iloc[:-1, :]
df_country.columns = df_country.iloc[0, :]
df_country = df_country.iloc[1:, :]

df.rename(columns = {'Sl.No.':'Sl.No'}, inplace = True)
df = pd.merge(df, df_donor, on = df.columns[0], how = 'left')
df = pd.merge(df, df_country, on = df.columns[0], how = 'left')
df =  df.iloc[:, 1:]
df.insert(loc = 0, column= 'Nature of association', value = '')

df_association = tables[2]
association = df_association[df_association[0].str.contains('Nature of association')].iloc[:, 1].item()

df.iloc[0,0] = association
print(df)

If you want more surety over targeting the right tables, then bring in BeautifulSoup and :-soup-contains to target the right tables:

import pandas as pd
import requests
from bs4 import BeautifulSoup as bs

r = requests.get('https://filebin.redpill-linpro.com/zj2qqc27va5fatm0/index.html')
soup = bs(r.content, 'lxml')

df =  pd.read_html(str(soup.select_one('table:-soup-contains("Sl.No.")')))[0]
df_donor = pd.read_html(str(soup.select_one('table:-soup-contains("Donor Name")')))[0]
df_association = pd.read_html(str(soup.select_one('table:-soup-contains("Association details")')))[0]
df_country = pd.read_html(str(soup.select_one('table:-soup-contains("Country Name")')))[0]

df = df.iloc[2:-1, :3]
df.columns = df.iloc[0, :]
df.drop(labels = 2, axis = 0, inplace = True)

df_donor = df_donor.iloc[:-2, :]
df_donor.columns = df_donor.iloc[0, :]
df_donor = df_donor.iloc[2:, :3]

df_country = df_country.iloc[:-1, :]
df_country.columns = df_country.iloc[0, :]
df_country = df_country.iloc[1:, :]

df.rename(columns = {'Sl.No.':'Sl.No'}, inplace = True)
df = pd.merge(df, df_donor, on = df.columns[0], how = 'left')
df = pd.merge(df, df_country, on = df.columns[0], how = 'left')
df =  df.iloc[:, 1:]
df.insert(loc = 0, column= 'Nature of association', value = '')

association = df_association[df_association[0].str.contains('Nature of association')].iloc[:, 1].item()

df.iloc[0,0] = association
print(df)

You can then handle NaN as required, per column, and use pandas.DataFrame.to_csv method to write out to csv.


You could of course do most of this with BeautifulSoup alone, but you would need to retrieve the Sl.No so as to enable row matching for output when joining up results (given differing numbers of results for your current css selectors).


It is probably worth researching if dropping columns, rows is more/less efficient than subsetting.

@QHarr answer is already good, so go with it. I just show how you can slightly modify your script, "zip" the data together and write it to the CSV file:

import csv
import requests
from bs4 import BeautifulSoup
from itertools import zip_longest

#
# I'm using your script:
#

file_link = "https://filebin.redpill-linpro.com/zj2qqc27va5fatm0/index.html"

res = requests.get(file_link)
soup = BeautifulSoup(res.text, "lxml")
nature_of_asso = soup.select_one(
    "td:contains('Nature of association') + td"
).get_text(strip=True)

purpose_in_cash = []
for purpose_tr in soup.select(
    "table:has(> tr > td:nth-of-type(1) + td:contains('Purpose')) tr"
)[3:]:
    try:
        purpose = purpose_tr.select_one("td:nth-of-type(2)").get_text(
            strip=True
        )
    except AttributeError:
        purpose = ""
    try:
        in_cash = purpose_tr.select_one("td:nth-of-type(3)").get_text(
            strip=True
        )
    except AttributeError:
        in_cash = ""
    purpose_in_cash.append((purpose, in_cash))  # <--- Add the data into a list in form of tuples

donnor_address = []
for donor_tr in soup.select(
    "table:has(> tr > td:nth-of-type(1) + td:contains('Donor Name')) tr"
)[2:]:
    try:
        donor_name = donor_tr.select_one("td:nth-of-type(2)").get_text(
            strip=True
        )
    except AttributeError:
        donor_name = ""
    try:
        address = donor_tr.select_one("td:nth-of-type(3)").get_text(strip=True)
    except AttributeError:
        address = ""
    donnor_address.append((donor_name, address)) # <--- Add the data into a list in form of tuples

country_amount = []
for country_tr in soup.select(
    "table:has(> tr > td:nth-of-type(1) + td:contains('Country Name')) tr"
)[1:]:
    try:
        country = country_tr.select_one("td:nth-of-type(2)").get_text(
            strip=True
        )
    except AttributeError:
        country = ""
    try:
        amount = country_tr.select_one("td:nth-of-type(3)").get_text(strip=True)
    except AttributeError:
        amount = ""
    country_amount.append((country, amount)) # <--- Add the data into a list in form of tuples


# Zip it together using itertools.zip_longest

with open("data.csv", "w") as f_out:
    writer = csv.writer(f_out)

    writer.writerow(
        [
            "Nature of association",
            "Purpose",
            "In cash (Previous balance)",
            "Donor Name",
            "Address",
            "Country Name",
            "Amount",
        ]
    )

    for a, b, c, d in zip_longest(
        [nature_of_asso], purpose_in_cash, donnor_address, country_amount
    ):
        writer.writerow(
            [
                a if a else "",
                *(b if b else ("", "")),
                *(c if c else ("", "")),
                *(d if d else ("", "")),
            ]
        )

Saves data.csv (screenshot from LibreOffice):

enter image description here

Related