I'm hoping for help in debugging this script. When I am simply printing out the iterative loop I don't have any issues. How would I now write that content to a csv file under the proper field names? Script is below
'''
from bs4 import BeautifulSoup
import csv, requests, re
# The following code is used to set the pythonssl default cipher list - needed if the server has an outdated openssl
try:
requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST += 'HIGH:!DH:!aNULL'
except AttributeError:
# no pyopenssl support used / needed / available
pass
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL'
# Set HTML headers to emulate Firefox browser to avoid bot security
# Potentially unused, but good practice to enable proper script function
html_headers = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET',
'Access-Control-Allow-Headers': 'Content-Type',
'Access-Control-Max-Age': '3600',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'
}
# Download BMO branch entries
base_url = "https://www.yellowpages.ca/search/si/{}/Bmo+Banque+De+Montr%C3%A9al/Canada" # URL structure of entries
first_page_request = requests.get(base_url.format("1"), headers=html_headers) # Format method used to insert page number into greater URL
first_page_content = BeautifulSoup(first_page_request.content, "html.parser") # Grabs all page content using Beautiful Soup module and standard Python parser
# Get total HTML page count for BMO entries
page_count_class = first_page_content.find_all(class_="pageCount")[0]
total_pages = re.search("([0-9]+)</span>", str(page_count_class)).group(1) # Extract the last page number from the total page count at the bottom of the page
# Initial CSV handling
f = open("bmo_data.csv", "w", newline="") # Chose w mode to write to a file
field_names = ["name", "street_address", "city", "province", "postal_code"]
writer = csv.DictWriter(f, fieldnames=field_names) # Setting the writer and field names
writer.writeheader()
# Running iterative loop through each page of the yellowpages listings for BMO
for page in range(1, int(total_pages) + 1):
page_request = requests.get(base_url.format(page), headers=html_headers)
page_html_content = BeautifulSoup(page_request.content, "html.parser")
entry_names = page_html_content.find_all(class_="jsListingName") # Get store name
entry_addresses = page_html_content.find_all("span", itemprop="streetAddress") # Get street address HTML object
entry_cities = page_html_content.find_all("span", itemprop="addressLocality") # Get city HTML object
entry_provinces = page_html_content.find_all("span", itemprop="addressRegion") # Get province HTML object
entry_postal = page_html_content.find_all("span", itemprop="postalCode") # Get postal code HTML object
for name, address, city, province, postalCode in zip(entry_names, entry_addresses, entry_cities, entry_provinces, entry_postal):
writer.writerow(page, name.get_text(), address.get_text(), city.get_text(), province.get_text(), postalCode.get_text())
f.close()
'''