I have a python script which makes http requests asynchronously, provided a lists of urls, and here is the example code.
#!/usr/local/bin/python3
import httpx
import asyncio
async def httpx_get(client, url):
resp1 = await client.get(url, timeout=5, allow_redirects=True)
# ... other computation ...
resp2 = await client.get(url, timeout=5, allow_redirects=True) # what happens during this second await?
async def fetch_pages(urls):
async with httpx.AsyncClient(verify=False) as client:
# create task for all URLs
coros = []
for url in urls:
coros.append(asyncio.create_task(httpx_get(client, url)))
# await all tasks
for coro in asyncio.as_completed(coros):
resp = await coro
def main():
urls = [ "https://google.com", "https://yahoo.com", "https://youtube.com" ]
asyncio.run(fetch_pages(urls))
if __name__ == "__main__":
main()
I realized there is a lack of understanding on my part with when asyncio would handle the second await, resp2 = await client.get(.., in my httpx_get function.
Since for coro in asyncio.as_completed(coros): resp = await coro only resolves the first await, resp1 = await client.get, for each coroutine, when does the second await resp2 = await client.get get resolved?
is the second await/request performed and waited for all at once? or does something else happen? Any clarification would be appreciated, thanks!