I'm new to the asyncio module. Until recently I used to use requests for the task I'm about to write about.
I'm trying to scale a script I work with which works well with up to 120 calls on Requests. However, with request being single threaded, it would take forever to get 1000 API calls (which is what I'm trying to achieve). This is where I found asyncio which makes asynchonous requests.
This is a script I put together for asyncio with the help of this article.
url = 'https://api.url.com/search?api_key=api_key&api_params=multiple_params'
queries = ['online slots', 'metaverse', 'sports betting', 'basketball odds', 'soccer odds', 'online poker', 'best casinos in germany', 'barbecue grills', 'outdoor pizza ovens']
results = []
def get_tasks(session):
tasks=[]
for q in queries:
tasks.append(asyncio.create_task(session.get(url.format(q), ssl=False)))
return tasks
timeout = ClientTimeout(total=500)
async def get_queries():
async with aiohttp.ClientSession(timeout=timeout) as session:
tasks = get_tasks(session)
responses = await asyncio.gather(*tasks)
for response in responses:
results.append(await response.json())
asyncio.run(get_queries())
It seems to work fine in most instances. But, it seems to timeout on many occasions. When I'm using the German queries and when it is making more than 500 API calls.
Below is what I keep getting back. As you can see in the script I've added a client timeout.
---------------------------------------------------------------------------
TimeoutError Traceback (most recent call last)
<ipython-input-4-8c48df090394> in <module>
33 results.append(await response.json())
34
---> 35 asyncio.run(get_queries())
/opt/anaconda3/lib/python3.8/site-packages/nest_asyncio.py in run(future, debug)
30 loop = asyncio.get_event_loop()
31 loop.set_debug(debug)
---> 32 return loop.run_until_complete(future)
33
34 if sys.version_info >= (3, 6, 0):
/opt/anaconda3/lib/python3.8/site-packages/nest_asyncio.py in run_until_complete(self, future)
68 raise RuntimeError(
69 'Event loop stopped before Future completed.')
---> 70 return f.result()
71
72 def _run_once(self):
/opt/anaconda3/lib/python3.8/asyncio/futures.py in result(self)
176 self.__log_traceback = False
177 if self._exception is not None:
--> 178 raise self._exception
179 return self._result
180
/opt/anaconda3/lib/python3.8/asyncio/tasks.py in __step(***failed resolving arguments***)
278 # We use the `send` method directly, because coroutines
279 # don't have `__iter__` and `__next__` methods.
--> 280 result = coro.send(None)
281 else:
282 result = coro.throw(exc)
<ipython-input-4-8c48df090394> in get_queries()
29 async with aiohttp.ClientSession(timeout=timeout) as session:
30 tasks = get_tasks(session)
---> 31 responses = await asyncio.gather(*tasks)
32 for response in responses:
33 results.append(await response.json())
/opt/anaconda3/lib/python3.8/asyncio/tasks.py in __wakeup(self, future)
347 def __wakeup(self, future):
348 try:
--> 349 future.result()
350 except BaseException as exc:
351 # This may also be a cancellation.
/opt/anaconda3/lib/python3.8/asyncio/tasks.py in __step(***failed resolving arguments***)
280 result = coro.send(None)
281 else:
--> 282 result = coro.throw(exc)
283 except StopIteration as exc:
284 if self._must_cancel:
/opt/anaconda3/lib/python3.8/site-packages/aiohttp/client.py in throw(self, arg)
1123
1124 def throw(self, arg: BaseException) -> None: # type: ignore[arg-type,override]
-> 1125 self._coro.throw(arg)
1126
1127 def close(self) -> None:
/opt/anaconda3/lib/python3.8/site-packages/aiohttp/client.py in _request(self, method, str_or_url, params, data, json, cookies, headers, skip_auto_headers, auth, allow_redirects, max_redirects, compress, chunked, expect100, raise_for_status, read_until_eof, proxy, proxy_auth, timeout, verify_ssl, fingerprint, ssl_context, ssl, proxy_headers, trace_request_ctx, read_bufsize)
557 resp = await req.send(conn)
558 try:
--> 559 await resp.start(conn)
560 except BaseException:
561 resp.close()
/opt/anaconda3/lib/python3.8/site-packages/aiohttp/client_reqrep.py in start(self, connection)
911 if self._continue is not None:
912 set_result(self._continue, True)
--> 913 self._continue = None
914
915 # payload eof handler
/opt/anaconda3/lib/python3.8/site-packages/aiohttp/helpers.py in __exit__(self, exc_type, exc_val, exc_tb)
719
720 if exc_type is asyncio.CancelledError and self._cancelled:
--> 721 raise asyncio.TimeoutError from None
722 return None
723
TimeoutError:
Can anyone help me figure what I'm doing wrong? And how to avoid timeouts for large amounts of API calls on asyncio?
Much appreciated!