Why my asyncio func is returning a list of lists (null)?

Viewed 71

have a list of urls that i want to scrape:

urls = [https://www.netflix.com/ar/title/81488310.
       https://www.netflix.com/ar/title/81486358
       https://www.netflix.com/ar/title/81486558
       https://www.netflix.com/ar/title/81488216
       https://www.netflix.com/ar/title/81488090
       https://www.netflix.com/ar/title/81489616
       https://www.netflix.com/ar/title/81487371
       https://www.netflix.com/ar/title/81487432
       https://www.netflix.com/ar/title/81485995
       https://www.netflix.com/ar/title/81489610
       https://www.netflix.com/ar/title/81488496]
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
asyncio.run(self.main(urls)) # here i call my main function

the following methods to __scrape__all the urls are:

def parse(self, html):
    ''' Normal method to parse response.text into soup '''

    soup = BeautifulSoup(html,'html.parser')
    status = soup.find('div', class_='error-page not-found') # contents that are not avialable (404)
    if not status:
        scripts = soup.find_all('script')
        return scripts
    else:
        return None

async def get_response(self, session, url):
    ''' Asynchronous requests with ClientSession'''

    try:
        async with session.get(url=url) as response:
            print(f'Successfully got url {url}.')
            return await response.text()
    except Exception as e:
        print(f"Unable to get url {url} due to {e.__class__}.")

async def response_soup(self, session, url):
    ''' Calls methods to get response and parse into soup
    with a separated thread '''

    html = await self.get_response(session, url)
    loop = asyncio.get_event_loop()
    soup = await loop.run_in_executor(None, self.parse, html) # run parse(html) in a separate thread
    return soup

async def main(self, urls):
    ''' Main where we setup semaphore and run the threads 
    for all urls with ClientSession '''

    async with asyncio.BoundedSemaphore(3), aiohttp.ClientSession() as session:
        results = await asyncio.gather(*[self.response_soup(session, url) for url in urls])
    #final = [result for result in results if result != None]
    print(results)
    print("Finalized all. Return is a list of len {} outputs.".format(len(results)))

but my output is list of null lists like this!

[
[][][][][][][][][][][][]][][][][]
]

console output (there are like 17000 urls in the real script)

what am i doing wrong?

0 Answers
Related