Is it possible to have different log formats for celery tasks and Django?
What I want is two different log formats. In celery logs i want additional task_id and processName parameters.
Below is my Django logger settings,
class LoggerSettingsMixin(object):
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'default': {
'format': '[%(asctime)s] [rid:%(request_id)s] [crid:%(client_request_id)s] [aid:%(actor_id)s] [%(levelname)s] [%(lineno)d:%(name)s] - %(message)s'
},
'celery': {
'format': '[%(asctime)s] [%(processName)s] [%(levelname)s] [%(lineno)d:%(name)s] - %(message)s'
},
},
'filters': {
'request_id': {
'()': 'request_id.logging.RequestIdFilter'
},
'client_request_id': {
'()': 'apps.commons.logging_filters.ClientRequestIdFilter'
},
'actor_id': {
'()': 'apps.commons.logging_filters.ActorFilter'
},
'task_id': {
'()': 'apps.commons.logging_filters.CeleryTaskIdFilter'
}
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'filters': ['request_id', 'actor_id', 'client_request_id', 'task_id'],
'formatter': 'default',
},
'celery': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'celery',
}
},
'loggers': {
'statsd': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': False,
},
'celery': {
'handlers': ['celery'],
'level': 'INFO',
'propagate': False,
},
'celery.redirected': {
'handlers': ['celery'],
'level': 'INFO',
'propagate': False,
},
'': {
'handlers': ['console'],
'level': 'INFO',
'propagate': True,
}
}
}
Below is my signals, where i am trying to override the celery root logger ,
@after_setup_logger.connnect
def setup_logger(**options):
logger = get_logger('')
logger.handlers = []
logger.addFilter(ActorFilter())
logger.addFilter(ClientRequestIdFilter())
logger.addFilter(CeleryTaskIdFilter())
logger.addFilter(RequestIdFilter())
formatter = logging.Formatter('[%(asctime)s] [%(processName)s][rid:%(request_id)s [crid:%(client_request_id)s] [aid:%(actor_id)s] [taskid:%(task_id)s] [%(levelname)s] [%(lineno)d:%(name)s] - %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
Also i have tried with setting up a logger for celery.redirected but its still not working.
in settings, CELERYD_HIJACK_ROOT_LOGGER = False
On trying to execute task it gives the log error,
[2017-07-26 21:09:17,377] [MainProcess] [INFO] [402:celery.worker.job] - Task apps.apis.reports.tasks.update_pipeline_reports[a6698044-a209-4fdf-8ab6-e6a1affb5efd] succeeded in 0.158420855005s: None
[2017-07-26 21:09:19,927] [PoolWorker-4] [WARNING] [282:celery.redirected] - Traceback (most recent call last):
[2017-07-26 21:09:19,927] [PoolWorker-4] [WARNING] [282:celery.redirected] - File "/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 861, in emit
[2017-07-26 21:09:19,927] [PoolWorker-4] [WARNING] [282:celery.redirected] - msg = self.format(record)
[2017-07-26 21:09:19,928] [PoolWorker-4] [WARNING] [282:celery.redirected] - File "/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 734, in format
[2017-07-26 21:09:19,928] [PoolWorker-4] [WARNING] [282:celery.redirected] - return fmt.format(record)
[2017-07-26 21:09:19,928] [PoolWorker-4] [WARNING] [282:celery.redirected] - File "/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 469, in format
[2017-07-26 21:09:19,929] [PoolWorker-4] [WARNING] [282:celery.redirected] - s = self._fmt % record.__dict__
[2017-07-26 21:09:19,929] [PoolWorker-4] [WARNING] [282:celery.redirected] - KeyError: 'request_id'
[2017-07-26 21:09:19,929] [PoolWorker-4] [WARNING] [282:celery.redirected] - Logged from file tasks.py, line 32
Also when i am not trying to override the root logger in after_setup_logger signal , it picks up the default Django format and prints logs without error.