How to nest asyncio tasks in groups and sub groups?

Viewed 271

I want to create multiple nested asyncio task groups. I don't want to use trio nursery also subtasks may contain async generators. However, I am encountering some problems:

  • Exiting with keyboard interrupt causes could not close task as event loop in closed
  • Sometimes tasks get cancelled and program gets stuck but this rarely happens
  • I want to make sure they everything is properly close as I am working with files and sockets

How can I fix those problems?

import asyncio
from asyncio.tasks import Task
from contextlib import AsyncExitStack
from os import error


async def cancel_tasks(tasks: set):
    task: Task
    for task in tasks:
        await task.cancel()


async def demo_producer(queue: asyncio.Queue):
    while True:
        await queue.put()


async def demo_consumer(queue: asyncio.Queue):
    while True:
        await queue.get()


async def demo_task():
    while True:
        await asyncio.sleep(2)  # Do Something


async def demo_task_grp(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
    async with AsyncExitStack() as stack:
        tasks = set()
        stack.push_async_callback(cancel_tasks, tasks)

        task = asyncio.create_task(demo_producer(qns_queue))
        tasks.add(task)
        task = asyncio.create_task(demo_consumer(ans_queue))
        tasks.add(task)
        task = asyncio.create_task(demo_task())
        tasks.add(task)

        asyncio.gather(*tasks)


async def question_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
    while True:
        try:
            await demo_task_grp(qns_queue, ans_queue)
        except error:
            print(error)
        finally:
            await asyncio.sleep(2)  # retry


async def answer_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
    while True:
        try:
            await demo_task_grp(qns_queue, ans_queue)
        except error:
            print(error)
        finally:
            await asyncio.sleep(2)  # retry


async def main():
    # await mqtt_setup()
    # asyncio.ensure_future(mqtt_setup())
    async with AsyncExitStack() as stack:
        question_queue = asyncio.Queue()
        answer_queue = asyncio.Queue()

        tasks = set()
        stack.push_async_callback(cancel_tasks, tasks)

        task = asyncio.create_task(question_func(question_queue, answer_queue))
        tasks.add(task)
        task = asyncio.create_task(answer_func(question_queue, answer_queue))
        tasks.add(task)

        asyncio.gather(*tasks)

asyncio.run(main())
0 Answers
Related