There are 2 asynchronous codes, one runs very fast, the second is quite slow, with the same number of requests, what could be the problem and how can this problem be solved? the intervals for which the code is executed are marked in the code.
this is a slow option
async def request(url, params=None):
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url, params=params) as response:
return await response.json()
async def get_entity(session, url, params=None):
async with session.get(url, params=params) as response:
return await response.json()
async def parse_all():
start = time.time()
result = (
await request(url + 'api/v1/companies/', params={"limit": 50})
)['_embedded']['companies']
task_list = []
print(time.time() - start) # 0.4720320701599121
async with aiohttp.ClientSession(headers=headers) as session:
for company in result:
for page_count in range(1, len(company['_embedded']['leads']) // 250 + 2):
task_list.append(
get_entity(
session,
url + 'api/v1/leads/',
{'page': page_count, 'company': company['id']}
)
)
for page_count in range(1, len(company['_embedded']['contacts']) // 250 + 2):
task_list.append(
get_entity(
session,
url + 'api/v1/contacts/',
{'page': page_count, 'company': company['id']}
)
)
print(time.time() - start) # 0.4190201759338379
return await asyncio.gather(*task_list)
if __name__ == '__main__':
start = time.time()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(parse_all())
print(time.time() - start) # 15.692176580429077
But actually fast. What's the difference?
async def request(session, url):
async with session.get(url) as response:
return await response.json()
async def parse():
start = time.time()
task = []
async with aiohttp.ClientSession() as session:
for i in range(1, 101):
task.append(
request(
session, f'https://jsonplaceholder.typicode.com/posts/{i}')
)
print(time.time() - start)
return await asyncio.gather(*task)
if __name__ == '__main__':
start = time.time()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(parse())
print(time.time() - start) # 0.44432830810546875