Python asyncio / aiohttp API 429 response error

Viewed 17

I'm trying to write an asynchronous API request, I found a suitable example, but in response I get an error: "429 Too Many Requests (Rate Limit Exceeded)" Am I wrong somewhere or is this an API limitation? How to make it work?

import asyncio
from datetime import datetime


async def fetch(session, task):  # fetching urls and mark result of execution
    async with session.post(f'https://api.urlname.com/users/{task["url"][1]}',  # user_id
                            headers={'Authorization': f"Bearer {task['url'][0]}"}) as response:  # token
        if response.status != 200:
            task['result'] = response.status
            task['status'] = response.json()
        elif response.status != 429:
            task['result'] = response.status
            task['status'] = response.json()
        await response.text()
        print(f"{str(datetime.now())}: Got result of {task['url'][1]}")  # logging
        task['result'] = response.status
        task['status'] = 'done'


async def fetch_all(session, urls, persecond):
    url_tasks = [{'url': i, 'result': None, 'status': 'new'} for i in urls]
    n = 0  # counter
    while True:
        running_tasks = len([i for i in url_tasks if i['status'] in ['fetch']])
        is_tasks_to_wait = len([i for i in url_tasks if i['status'] != 'done'])
        if n < len(url_tasks) and running_tasks < persecond:
            url_tasks[n]['status'] = 'fetch'
            asyncio.create_task(fetch(session, url_tasks[n]))
            n += 1
            print(f'Schedule tasks {n}. '
                  f'Running {running_tasks} '
                  f'Remain {is_tasks_to_wait}')
        if running_tasks >= persecond:
            print('Throttling')
            await asyncio.sleep(1)
        if is_tasks_to_wait != 0:
            await asyncio.sleep(0.1)  # wait all tasks done
        else:
            break
    return url_tasks


async def main():
    async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
        res = await fetch_all(session, list_of_tokens_n_id, 3)
        print(res)


if __name__ == '__main__':
    list_of_tokens_n_id = [['token0', '0000'], ['token1', '0001'], ['token2', '0002'], ['token3', '0003']]
    asyncio.run(main())```
0 Answers
Related