asyncio + aiohttp, 2 different codes, but the execution speed varies at times, although they are described the same way

Viewed 12

There are 2 asynchronous codes, one runs very fast, the second is quite slow, with the same number of requests, what could be the problem and how can this problem be solved? the intervals for which the code is executed are marked in the code.

this is a slow option

async def request(url, params=None):
    async with aiohttp.ClientSession(headers=headers) as session:
        async with session.get(url, params=params) as response:
            return await response.json()


async def get_entity(session, url, params=None):
    async with session.get(url, params=params) as response:
        return await response.json()


async def parse_all():
    start = time.time()
    result = (
        await request(url + 'api/v1/companies/', params={"limit": 50})
    )['_embedded']['companies']
    task_list = []
    print(time.time() - start)  # 0.4720320701599121
    async with aiohttp.ClientSession(headers=headers) as session:
        for company in result:
            for page_count in range(1, len(company['_embedded']['leads']) // 250 + 2):
                task_list.append(
                    get_entity(
                        session,
                        url + 'api/v1/leads/',
                        {'page': page_count, 'company': company['id']}
                    )
                )
            for page_count in range(1, len(company['_embedded']['contacts']) // 250 + 2):
                task_list.append(
                    get_entity(
                        session,
                        url + 'api/v1/contacts/',
                        {'page': page_count, 'company': company['id']}
                    )
                )
        print(time.time() - start)  # 0.4190201759338379
        return await asyncio.gather(*task_list)


if __name__ == '__main__':
    start = time.time()
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    loop.run_until_complete(parse_all())
    print(time.time() - start)  # 15.692176580429077

But actually fast. What's the difference?

async def request(session, url):
    async with session.get(url) as response:
        return await response.json()


async def parse():
    start = time.time()
    task = []
    async with aiohttp.ClientSession() as session:
        for i in range(1, 101):
            task.append(
                request(
                    session, f'https://jsonplaceholder.typicode.com/posts/{i}')
            )
        print(time.time() - start)
        return await asyncio.gather(*task)


if __name__ == '__main__':
    start = time.time()
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    loop.run_until_complete(parse())
    print(time.time() - start) # 0.44432830810546875
0 Answers
Related