what is NavigableString in this error refers to, and why that happened?

user_15445436 0
Viewed 57
from bs4 import BeautifulSoup
import numpy as np
import requests 
from selenium import webdriver
from nltk.tokenize import sent_tokenize,word_tokenize

html = webdriver.Firefox(executable_path=r'D:\geckodriver.exe')
html.get("https://www.tsa.gov/coronavirus/passenger-throughput")

def TSA_travel_numbers(html):

    soup = BeautifulSoup(html,'lxml')
    for i,rows in enumerate(soup.find('div',class_='view-content'),1):
        # print(rows.content)
        for header in rows.find('tr'):
    

            number = rows.find_all('td',class_='views-field views-field field-2021-throughput views-align-center')

            print(number.text)


TSA_travel_numbers(html.page_source)

My error as follows :

Traceback (most recent call last):
  File "TSA_travel.py", line 23, in <module>
    TSA_travel_numbers(html.page_source)
  File "TSA_travel.py", line 15, in TSA_travel_numbers
    for header in rows.find('tr'):
TypeError: 'int' object is not iterable

What is happening here? I can't iter thru 'tr' tags, please help me to solve this problem. Sorry for your time and advance thanks!

web-scraping beautifulsoup python
1 Answers

As the error says, you can't iterate over an int, which is your rows.

Also, there's no need for a webdriver as data on the page is static.

Here's my take on it:

import requests
from bs4 import BeautifulSoup
from tabulate import tabulate


def get_page(url):
    return requests.get(url).text


def get_data(page):
    soup = BeautifulSoup(page, 'lxml')
    return [
        item.getText(strip=True) for item in soup.select(".views-align-center")
    ]


def build_table(table_rows):
    t = [table_rows[i:i + 4] for i in range(0, len(table_rows[1:]), 4)]
    h = t[0]
    return t[1:], h


if __name__ == '__main__':
    source = "https://www.tsa.gov/coronavirus/passenger-throughput"
    table, header = build_table(get_data(get_page(source)))
    print(tabulate(table, headers=header, tablefmt="pretty"))

Output:

+------------+--------------------------+--------------------------+--------------------------+
|    Date    | 2021 Traveler Throughput | 2020 Traveler Throughput | 2019 Traveler Throughput |
+------------+--------------------------+--------------------------+--------------------------+
|  5/9/2021  |        1,707,805         |         200,815          |        2,419,114         |
|  5/8/2021  |        1,429,657         |         169,580          |        1,985,942         |
|  5/7/2021  |        1,703,267         |         215,444          |        2,602,631         |
|  5/6/2021  |        1,644,050         |         190,863          |        2,555,342         |
|  5/5/2021  |        1,268,938         |         140,409          |        2,270,662         |
|  5/4/2021  |        1,134,103         |         130,601          |        2,106,597         |
|  5/3/2021  |        1,463,672         |         163,692          |        2,470,969         |
|  5/2/2021  |        1,626,962         |         170,254          |        2,512,598         |
|  5/1/2021  |        1,335,535         |         134,261          |        1,968,278         |
| 4/30/2021  |        1,558,553         |         171,563          |        2,546,029         |
| 4/29/2021  |        1,526,681         |         154,695          |        2,499,461         |
| 4/28/2021  |        1,184,326         |         119,629          |        2,256,442         |
| 4/27/2021  |        1,077,199         |         110,913          |        2,102,068         |
| 4/26/2021  |        1,369,410         |         119,854          |        2,412,770         |
| 4/25/2021  |        1,571,220         |         128,875          |        2,506,809         |
| 4/24/2021  |        1,259,724         |         114,459          |        1,990,464         |
| 4/23/2021  |        1,521,393         |         123,464          |        2,521,897         |
| 4/22/2021  |        1,509,649         |         111,627          |        2,526,961         |
| 4/21/2021  |        1,164,099         |          98,968          |        2,254,209         |
| 4/20/2021  |        1,082,443         |          92,859          |        2,227,475         |
| 4/19/2021  |        1,412,500         |          99,344          |        2,594,171         |
| 4/18/2021  |        1,572,383         |         105,382          |        2,356,802         |
| 4/17/2021  |        1,277,815         |          97,236          |        1,988,205         |
| 4/16/2021  |        1,468,218         |         106,385          |        2,457,133         |
| 4/15/2021  |        1,491,435         |          95,085          |        2,616,158         |
| 4/14/2021  |        1,152,703         |          90,784          |        2,317,381         |
| 4/13/2021  |        1,085,034         |          87,534          |        2,208,688         |
| 4/12/2021  |        1,468,972         |         102,184          |        2,484,580         |
| 4/11/2021  |        1,561,495         |          90,510          |        2,446,801         |

and so on ...

Or, an ever shorter approach, just use pandas:

import pandas as pd
import requests
from tabulate import tabulate

if __name__ == '__main__':
    source = "https://www.tsa.gov/coronavirus/passenger-throughput"
    df = pd.read_html(requests.get(source).text, flavor="bs4")[0]
    print(tabulate(df.head(10), tablefmt="pretty", showindex=False))

Output:

+-----------+-----------+--------+---------+
| 5/9/2021  | 1707805.0 | 200815 | 2419114 |
| 5/8/2021  | 1429657.0 | 169580 | 1985942 |
| 5/7/2021  | 1703267.0 | 215444 | 2602631 |
| 5/6/2021  | 1644050.0 | 190863 | 2555342 |
| 5/5/2021  | 1268938.0 | 140409 | 2270662 |
| 5/4/2021  | 1134103.0 | 130601 | 2106597 |
| 5/3/2021  | 1463672.0 | 163692 | 2470969 |
| 5/2/2021  | 1626962.0 | 170254 | 2512598 |
| 5/1/2021  | 1335535.0 | 134261 | 1968278 |
| 4/30/2021 | 1558553.0 | 171563 | 2546029 |
+-----------+-----------+--------+---------+
user_61067910
Related