I am trying to scrape data from the urls in the url list and then write it down into a csv. It works fine if I just scrape the first two links:
"21Shares Bitcoin ETP": 'https://21shares.com/product/abtc',
"21Shares Ethereum ETP":'https://21shares.com/product/aeth/'
However when I add all the other links I get the Error below:
ValueError: Length of values (16) does not match length of index (17)
I can't figure out where the problem lies and how I can solve it. What needs to be changed in order for my code to be able to scrape through all links and then store the data in a csv?
Here is my code:
import pandas as pd
import requests
from bs4 import BeautifulSoup
from datetime import date
class sharesWebscrapping:
today = date.today()
dict_words = {'<span class="value">': "",
'<span class="value value-up">': "",
'</span>': "",
',': "",
'$': "",
'<div class="value"><div>': "",
'<div>': "",
'</div>': "",
'<div class="value"><!-- -->': "",
'<span class="value value-down">':""
}
index_name = ["AUM",
"Securities Outstanding",
"NAV per Unit",
"Daily Change",
"VALOR",
"Fee",
"SEDOL",
"ISIN",
"Reuters",
"Issuer",
"WKN",
"iNAV",
"Bloomberg",
"Custody",
"Market Maker",
"Authorized Participants",
"Collateral Agent"
]
url = {"21Shares Bitcoin ETP": 'https://21shares.com/product/abtc',
"21Shares Ethereum ETP":'https://21shares.com/product/aeth/',
"21Shares Bitcoin Cash ETP": 'https://21shares.com/product/abch',
"21Shares Cardano ETP": 'https://21shares.com/product/aada',
"21Shares Aave ETP": 'https://21shares.com/product/aave',
"21Shares Bitcoin Suisse Index ETP": 'https://21shares.com/product/abba',
"21Shares Bitcoin Cash ETP": 'https://21shares.com/product/abch',
"21Shares Binance BNB ETP": 'https://21shares.com/product/abnb',
"21Shares Polkadot ETP": 'https://21shares.com/product/adot',
"21Shares FTX Token ETP": 'https://21shares.com/product/aftt',
"21Shares Algorand ETP": 'https://21shares.com/product/algo',
"21Shares Crypto Mid-Cap Index ETP": 'https://21shares.com/product/alts',
"21Shares Solana ETP": 'https://21shares.com/product/asol',
"21Shares Cosmos ETP": 'https://21shares.com/product/atom',
"21Shares Uniswap ETP": 'https://21shares.com/product/auni',
"21Shares Avalanche ETP": 'https://21shares.com/product/avax',
"21Shares Stellar ETP": 'https://21shares.com/product/axlm',
"21Shares Ripple XRP ETP": 'https://21shares.com/product/axrp',
"21Shares Tezos ETP": 'https://21shares.com/product/axtz',
"21Shares Crypto Basket Index ETP": 'https://21shares.com/product/hodl',
"21Shares Crypto Basket Equal Weight ETP": 'https://21shares.com/product/hodlv',
"21Shares Crypto Basket 10 ETP": 'https://21shares.com/product/hodlx',
"21Shares Bitwise Select 10 ETP": 'https://21shares.com/product/keys',
"21Shares Chainlink ETP": 'https://21shares.com/product/link',
"21Shares Terra ETP": 'https://21shares.com/product/luna',
"21Shares Decentraland ETP": 'https://21shares.com/product/mana',
"21Shares Sygnum Platform Winners Index ETP": 'https://21shares.com/product/moon',
"21Shares Polygon ETP": 'https://21shares.com/product/poly',
"21Shares Short Bitcoin ETP": 'https://21shares.com/product/sbtc'
}
data = pd.DataFrame(index=index_name, columns=url)
for key in url:
print("Downloading " + key + "...")
response = requests.get(url[key])
soup = BeautifulSoup(response.text, 'html.parser')
list_value = list(soup.findAll(class_="value"))
new_list_value = []
for i in list_value:
new_list_value.append(str(i))
data[key] = new_list_value
data.replace(dict_words, inplace=True, regex=True)
data.to_csv(r"/Users/a/documents" + str(today) + ".csv")