(Flask) Celery Beat won't execute a task... however Celery Worker works

Viewed 9

I am running a FastAPI backend and have the worker on its own Flask server.

Unfortunately, I can't seem to get Celery Beat to execute a task. The worker is configured correctly. Both the worker and beat have running PID's per supervisord. The worker works fine but I think I may have the periodic scheduler misconfigured? Not sure.

Execution context is in a docker container via compose.

All help appreciated, thank you! Below are relevant code excerpts and directory paths.

server/worker/supervisord.conf

[supervisord]
nodaemon=true


[program:celeryworker]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
command=celery -A worker.scheduler.schedule.celery_app worker -l info


[program:celerybeat]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
command=celery -A worker.scheduler.schedule.celery_app beat -l info

server/worker/scheduler/init.py

try:
    import os
    from flask import Flask
    from celery import Celery
    from celery.schedules import crontab
    from datetime import timedelta
except Exception as e:
    print("Error : {} ".format(e))


def make_celery(app):
    celery = Celery(app.import_name, backend=app.config['CELERY_BACKEND'],
                    broker=app.config['CELERY_BROKER_URL'])
    celery.conf.update(app.config)
    TaskBase = celery.Task

    class ContextTask(TaskBase):
        abstract = True

        def __call__(self, *args, **kwargs):
            with app.app_context():
                return TaskBase.__call__(self, *args, **kwargs)
    celery.Task = ContextTask

    return celery


app = Flask(__name__)
app.config['CELERY_BACKEND'] = os.getenv(
    "CELERY_RESULT_BACKEND", "redis://redis:6000/0")
app.config['CELERY_BROKER_URL'] = os.getenv(
    "CELERY_BROKER_URL", "amqp://guest:guest@rabbitmq:5672//")
app.config['task_track_started'] = True
app.config['timezone'] = 'UTC'


app.config['beat_schedule'] = {
    'test_beat': {
        'task': 'test_beat',
        'schedule': crontab(minute='1')
    },
}


celery_app = make_celery(app)

server/worker/scheduler/schedule.py

from celery.utils.log import get_task_logger
from datetime import date, timedelta
from fastapi import Depends
from sqlalchemy.orm import Session
from worker.scheduler import celery_app
from yahoo_fin import options, stock_info
import json
import numpy as np
import pandas as pd
import redis

from db.database import get_db
from db.models import Company

logger = get_task_logger(__name__)
r = redis.Redis(db=0, host="redis", port=6000)


@celery_app.on_after_finalize.connect
def setup_periodic_tasks(sender, **kwargs):
    print('SETUP PERIODIC TASKS', flush=True)
    print(sender, flush=True)
    print(**kwargs, flush=True)
    sender.add_periodic_task(5.0, test_beat.s('HELLO BEAT'))

@celery_app.task
def test_beat(arg):
    print(arg, flush=True)

##########
# WORKER #
##########

...
0 Answers
Related