python coroutine asyncio/ await / aiohttp

user_16443622 0
Viewed 147

new in asyncio world.

going straight to the point...

I want to do/make a request(aiohttp) to a site. if the wait for an answer pass than N seconds I want to stop the process of waiting. Do the process again by setting a limit of attempts if needed.

async def search_skiping_add(name_search):
    start_time = time()
    async with aiohttp.ClientSession() as session:
        url = f'https://somesitehere.com'
        r = await session.get(url)

    
    final_time = time()
    result_time =round(final_time-start_time)
    print(result_time)

Maybe, I know, have some way to do it synchronously, but it's an excuse to start using asyncio somehow too.

aiohttp async-await python-asyncio
1 Answers

This should give you an idea of how to use async with aiohttp:

from aiohttp import ClientSession
from asyncio import gather, create_task, sleep, run
from traceback import format_exc

def returnPartionedList(inputlist: list, x: int = 100) -> list: # returns inputlist split into x parts, default is 100
    return([inputlist[i:i + x] for i in range(0, len(inputlist), x)])

# You could change validate to an integer and thereby increasing the retry count as needed.
async def GetRessource(url: str, session: ClientSession, validate: bool = False) -> dict:
    try:
        async with session.get(url) as response:
            if response.status == 200:
                r: dict = await response.json() # Set equal to .text() to get results as a string
                return(r)
            else:
                r: str = await response.text()
                if not validate:
                    await sleep(3) # Sleep for x amount of seconds before retry
                    return(await GetRessource(url, session, True))
                print(f"Error, got response code: {response.status} message: {r}")
    except Exception:
        print(f"General Exception:\n{format_exc()}")
    return({})

async def GetUrls(urls: list) -> list:
    resultsList: list = []
    UrlPartitions: list = returnPartionedList(urls, 20) # Rate limit to 20 requests per loop
    async with ClientSession(timeout=15) as session: # Timeout is equal to the time to wait in seconds before terminating, default is 300 seconds or 5 minutes.
        for partition in UrlPartitions:
            partitionTasks: list = [create_task(GetRessource(url, session)) for url in partition]
            resultsList.append(await gather(*partitionTasks, return_exceptions=False))
    return(resultsList) # Or you can do more processing here before returning

async def main():
    urls: list = ["...", "...", "..."] # list of urls to get from
    results: list = await GetUrls(urls)
    print(results)

if __name__ == "__main__":
    run(main())
user_113559260
Related