I'm trying to pratice some data scraping and am trying to scrape financials from yahoo finance. When I do the data scraping it seems that I do not get all the breakdowns and values from the P&L statement (url: https://finance.yahoo.com/quote/AAPL/financials?p=AAPL).
It seems that it is only the top categories that comes out and the subcategories are left out. For instance the Total Revenue is scraped but the subcategory "Operating Revenue" is left out (Hence: There can be multiple subcategories but in this case there is only one).
How can I scrape the subcateogires as well?
My code is as follows:
import requests
from bs4 import BeautifulSoup
import pandas as pd
stock_abb = ['AAPL']
df = pd.DataFrame()
for s in stock_abb:
url = 'https://finance.yahoo.com/quote/' + s + '/financials?p=' + s
header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36'}
r = requests.get(url, headers={"User-Agent": "Requests"}).text
soup = BeautifulSoup(r, 'html.parser')
#Scrape first part
financials_number_1 = soup.find_all('div', {'class': 'Ta(c) Py(6px) Bxz(bb) BdB Bdc($seperatorColor) Miw(120px) Miw(100px)--pnclg D(tbc)'})
financials_title = soup.find_all('div', {'class': 'D(ib) Va(m) Ell Mt(-3px) W(215px)--mv2 W(200px) undefined'})
financial_period_2 = soup.find_all('div', {'class': 'Ta(c) Py(6px) Bxz(bb) BdB Bdc($seperatorColor) Miw(120px) Miw(100px)--pnclg D(ib) Fw(b)'})
df_first_part = pd.DataFrame ({'Company':[],'Period':[], 'Type':[],'Value':[]})
t = 0
i = 0
for t in range(0,len(financials_number_1)):
try:
f1 = financials_number_1[t].find_all('span')[0].get_text()
if (t % 2):
Type = financials_title[i].find_all('span')[0].get_text()
Year = financial_period_2[1].find_all('span')[0].get_text()
i = i + 1
else:
Type = financials_title[i].find_all('span')[0].get_text()
Year = financial_period_2[0].find_all('span')[0].get_text()
df2 = pd.DataFrame ({'Company':[s],'Period':[Year], 'Type':[Type],'Value':[f1]})
data = [df_first_part, df2]
df_first_part = pd.concat(data)
except:
if (t % 2):
i = i + 1
#Scrape second part
financials_number_2 = soup.find_all('div', {'class': 'Ta(c) Py(6px) Bxz(bb) BdB Bdc($seperatorColor) Miw(120px) Miw(100px)--pnclg Bgc($lv1BgColor) fi-row:h_Bgc($hoverBgColor) D(tbc)'})
financials_title = soup.find_all('div', {'class': 'D(ib) Va(m) Ell Mt(-3px) W(215px)--mv2 W(200px) undefined'})
financial_period_1 = soup.find_all('div', {'class': 'Ta(c) Py(6px) Bxz(bb) BdB Bdc($seperatorColor) Miw(120px) Miw(100px)--pnclg D(ib) Fw(b) Tt(u) Bgc($lv1BgColor)'})
financial_period_3 = soup.find_all('div', {'class': 'Ta(c) Py(6px) Bxz(bb) BdB Bdc($seperatorColor) Miw(120px) Miw(100px)--pnclg D(ib) Fw(b) Bgc($lv1BgColor)'})
df_second_part = pd.DataFrame ({'Period':[], 'Type':[],'Value':[]})
t = 0
i = 0
u = 0
for t in range(0,len(financials_number_2)):
try:
f2 = financials_number_2[t].find_all('span')[0].get_text()
if t == 0 or (t % 3 == 0):
Year = financial_period_1[0].find_all('span')[0].get_text()
else:
if u == 0:
Year = financial_period_3[0].find_all('span')[0].get_text()
u = u + 1
else:
Year = financial_period_3[1].find_all('span')[0].get_text()
u = 0
if t == 0:
Type = financials_title[0].find_all('span')[0].get_text()
elif (t % 3 == 0):
i = i + 1
Type = financials_title[i].find_all('span')[0].get_text()
else:
Type = financials_title[i].find_all('span')[0].get_text()
df2 = pd.DataFrame ({'Company':[s], 'Period':[Year], 'Type':[Type],'Value':[f2]})
data = [df_second_part, df2]
df_second_part = pd.concat(data)
except:
if (t % 3 != 0):
if u == 0:
u = u + 1
if (t % 3 == 0):
i = i + 1
else: u = 0
data = [df,df_second_part,df_first_part]
df = pd.concat(data)
For the revenue the is seperated in two. One for the top categories and one for the subcategories: enter image description here
With in the subcategory div I want to scrape this value: enter image description here
Hence: In this case it is the same value but for other stocks there are multiple categories/values.