I am running a FastAPI backend and have the worker on its own Flask server.
Unfortunately, I can't seem to get Celery Beat to execute a task. The worker is configured correctly. Both the worker and beat have running PID's per supervisord. The worker works fine but I think I may have the periodic scheduler misconfigured? Not sure.
Execution context is in a docker container via compose.
All help appreciated, thank you! Below are relevant code excerpts and directory paths.
server/worker/supervisord.conf
[supervisord]
nodaemon=true
[program:celeryworker]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
command=celery -A worker.scheduler.schedule.celery_app worker -l info
[program:celerybeat]
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
command=celery -A worker.scheduler.schedule.celery_app beat -l info
server/worker/scheduler/init.py
try:
import os
from flask import Flask
from celery import Celery
from celery.schedules import crontab
from datetime import timedelta
except Exception as e:
print("Error : {} ".format(e))
def make_celery(app):
celery = Celery(app.import_name, backend=app.config['CELERY_BACKEND'],
broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
app = Flask(__name__)
app.config['CELERY_BACKEND'] = os.getenv(
"CELERY_RESULT_BACKEND", "redis://redis:6000/0")
app.config['CELERY_BROKER_URL'] = os.getenv(
"CELERY_BROKER_URL", "amqp://guest:guest@rabbitmq:5672//")
app.config['task_track_started'] = True
app.config['timezone'] = 'UTC'
app.config['beat_schedule'] = {
'test_beat': {
'task': 'test_beat',
'schedule': crontab(minute='1')
},
}
celery_app = make_celery(app)
server/worker/scheduler/schedule.py
from celery.utils.log import get_task_logger
from datetime import date, timedelta
from fastapi import Depends
from sqlalchemy.orm import Session
from worker.scheduler import celery_app
from yahoo_fin import options, stock_info
import json
import numpy as np
import pandas as pd
import redis
from db.database import get_db
from db.models import Company
logger = get_task_logger(__name__)
r = redis.Redis(db=0, host="redis", port=6000)
@celery_app.on_after_finalize.connect
def setup_periodic_tasks(sender, **kwargs):
print('SETUP PERIODIC TASKS', flush=True)
print(sender, flush=True)
print(**kwargs, flush=True)
sender.add_periodic_task(5.0, test_beat.s('HELLO BEAT'))
@celery_app.task
def test_beat(arg):
print(arg, flush=True)
##########
# WORKER #
##########
...