Error while grabbing the table data from a website

Question

I am trying to grab some stock related data from the web for my project.I encountered couple of problems.
Problem 1:
I tried to grab the table from this site http://sharesansar.com/c/today-share-price.html
It worked but the columns aren't grabbed in order.For eg: Column 'Company Name' has values of 'Open price'. How can I solve this?
Problem 2:
I also tried to grab a company specific data from http://merolagani.com/CompanyDetail.aspx?symbol=ADBL under 'Price History' tab.
This time I got an error while grabbing the table data.The error I got was:

self.data[key].append(cols[index].get_text())

IndexError: list index out of range

The code is as shown below:

import logging
import requests
from bs4 import BeautifulSoup
import pandas


module_logger = logging.getLogger('mainApp.dataGrabber')


class DataGrabberTable:
    ''' Grabs the table data from a certain url. '''

    def __init__(self, url, csvfilename, columnName=[], tableclass=None):
        module_logger.info("Inside 'DataGrabberTable' constructor.")
        self.pgurl = url
        self.tableclass = tableclass
        self.csvfile = csvfilename
        self.columnName = columnName

        self.tableattrs = {'class':tableclass} #to be passed in find()

        module_logger.info("Done.")


    def run(self):
        '''Call this to run the datagrabber. Returns 1 if error occurs.'''

        module_logger.info("Inside 'DataGrabberTable.run()'.")

        try:
            self.rawpgdata = (requests.get(self.pgurl, timeout=5)).text
        except Exception as e:
            module_logger.warning('Error occured: {0}'.format(e))
            return 1

        #module_logger.info('Headers from the server:
 {0}'.format(self.rawpgdata.headers))

        soup = BeautifulSoup(self.rawpgdata, 'lxml')

        module_logger.info('Connected and parsed the data.')

        table = soup.find('table',attrs = self.tableattrs)
        rows = table.find_all('tr')[1:]

        #initializing a dict in a format below
        # data = {'col1' : [...], 'col2' : [...], }
        #col1 and col2 are from columnName list
        self.data = {}
        self.data = dict(zip(self.columnName, [list() for i in range(len(self.columnName))]))

        module_logger.info('Inside for loop.')
        for row in rows:
            cols = row.find_all('td')
            index = 0
            for key in self.data:
                if index > len(cols): break
                self.data[key].append(cols[index].get_text())
                index += 1
        module_logger.info('Completed the for loop.')

        self.dataframe = pandas.DataFrame(self.data)    #make pandas dataframe

        module_logger.info('writing to file {0}'.format(self.csvfile))
        self.dataframe.to_csv(self.csvfile)
        module_logger.info('written to file {0}'.format(self.csvfile))

        module_logger.info("Done.")
        return 0

    def getData(self):
        """"Returns 'data' dictionary."""
        return self.data




    # Usage example

    def main():
        url = "http://sharesansar.com/c/today-share-price.html"
        classname = "table"
        fname = "data/sharesansardata.csv"
        cols = [str(i) for i in range(18)] #make a list of columns

        '''cols = [
          'S.No', 'Company Name', 'Symbol', 'Open price', 'Max price', 
         'Min price','Closing price', 'Volume', 'Previous closing', 
         'Turnover','Difference',
         'Diff percent', 'Range', 'Range percent', '90 days', '180 days',
         '360 days', '52 weeks high', '52 weeks low']'''

        d = DataGrabberTable(url, fname, cols, classname)
        if d.run() is 1:
            print('Data grabbing failed!')
        else:
            print('Data grabbing done.') 


    if __name__ == '__main__':
        main()

Few suggestions would help.Thank you!

Error while grabbing the table data from a website

1 Answers