I am running the setup that follows(just showing the beat setup for simplicity) for daemonizing my celery and beat workers on Elastic beanstalk. I am able daemonize the processes successfully however too many processes are being spawned.
Current Output
root 20409 0.7 9.1 473560 92452 ? S 02:59 0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root 20412 0.6 7.8 388152 79228 ? S 02:59 0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO
root 20509 0.0 7.1 388748 72412 ? S 02:59 0:00 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root 20585 0.6 7.7 387624 78340 ? S 03:00 0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO
root 20679 1.1 9.1 473560 92584 ? S 03:01 0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root 20685 0.0 7.1 388768 72460 ? S 03:01 0:00 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
Desired output as achieved by after the environment deploys running kill -9 $(pgrep celery)
root 20794 20.6 7.7 387624 78276 ? S 03:03 0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO
root 20797 24.3 9.1 473560 92564 ? S 03:03 0:01 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
root 20806 0.0 7.1 388656 72272 ? S 03:03 0:00 /opt/python/run/venv/bin/python3.6 /opt/python/run/venv/bin/celery -A djangoApp worker --loglevel=INFO
celeryBeat.sh
#!/usr/bin/env bash
/opt/python/run/venv/bin/celery -A djangoApp beat --loglevel=INFO
supervisor.conf
[unix_http_server]
file=/opt/python/run/supervisor.sock ; (the path to the socket file)
;chmod=0700 ; socket file mode (default 0700)
;chown=nobody:nogroup ; socket file uid:gid owner
[supervisord]
logfile=/opt/python/log/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=10MB ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10 ; (num of main logfile rotation backups;default 10)
loglevel=info ; (log level;default info; others: debug,warn,trace)
pidfile=/opt/python/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
minfds=1024 ; (min. avail startup file descriptors;default 1024)
minprocs=200 ; (min. avail process descriptors;default 200)
directory=/opt/python/current/app ; (default is not to cd during start)
;nocleanup=true ; (don't clean up tempfiles at start;default false)
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///opt/python/run/supervisor.sock
[program:httpd]
command=/opt/python/bin/httpdlaunch
numprocs=1
directory=/opt/python/current/app
autostart=true
autorestart=unexpected
startsecs=1 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
killasgroup=false ; SIGKILL the UNIX process group (def false)
redirect_stderr=false
[include]
files: celery.conf
celery.conf
[program:beat:]
; Set full path to celery program if using virtualenv
command=sh /opt/python/etc/celeryBeat.sh
directory=/opt/python/current/app
; user=nobody
numprocs=1
stdout_logfile=/var/log/celery-beat.log
stderr_logfile=/var/log/celery-beat.log
autostart=true
autorestart=true
startsecs=60
; Need to wait for currently executing tasks to finish at shutdown.
; Increase this if you have very long running tasks.
stopwaitsecs = 60
; When resorting to send SIGKILL to the program to terminate it
; send SIGKILL to its whole process group instead,
; taking care of its children as well.
killasgroup=true
; if rabbitmq is supervised, set its priority higher
; so it starts first
priority=998
container commands
container_commands:
01_celery_tasks:
command: "cat .ebextensions/files/celery_configuration.txt > /opt/elasticbeanstalk/hooks/appdeploy/post/run_supervised_celeryd.sh && chmod 744 /opt/elasticbeanstalk/hooks/appdeploy/post/run_supervised_celeryd.sh"
leader_only: true
02_celery_tasks_run:
command: "cat .ebextensions/files/beat_configuration.txt > /opt/python/etc/beat.sh && chmod 744 /opt/python/etc/celeryBeat.sh"
leader_only: true
03_celery_tasks_run:
command: "/opt/elasticbeanstalk/hooks/appdeploy/post/run_supervised_celeryd.sh"
leader_only: true