How to make Prefect.io format file handler so that log entries will be just like stdout

Viewed 754

I am trying to have a Prefect.io log file, created by file handler, to look exactly like the log the is printed out by the Prefect.io logger, but I get just the message.

instead of

[2021-06-11 12:46:02-0700] INFO - prefect.FlowRunner | Beginning Flow run for 'test'
[2021-06-11 12:46:02-0700] INFO - prefect.TaskRunner | Task 'pull_project': Starting task run...
[2021-06-11 12:46:02-0700] INFO - prefect.pull_project | Pull latest test version from git.
[2021-06-11 12:46:02-0700] WARNING - prefect.pull_project | A warning message....
[2021-06-11 12:46:02-0700] INFO - prefect.TaskRunner | Task 'pull_project': Finished task run for task with final state: 'Success'
[2021-06-11 12:46:02-0700] INFO - prefect.FlowRunner | Flow run SUCCESS: all reference tasks succeeded

In the log file I see

Beginning Flow run for 'test'
Task 'pull_project': Starting task run...
Pull latest test version from git.
A warning message....
Task 'pull_project': Finished task run for task with final state: 'Success'
Flow run SUCCESS: all reference tasks succeeded

Code:

from datetime import timedelta, datetime
from prefect import task, Flow, Parameter
import prefect
import logging
from prefect.utilities.logging import get_logger
from pathlib import Path
import os


SRC_PATH = os.path.dirname(__file__)
LOG_PATH = os.path.join(Path(SRC_PATH).parents[0], 'logs')


class MyFileLogger(logging.FileHandler):

    def __init__(self, filename, mode='a', encoding=None, delay=False):
        filename = os.path.join(LOG_PATH, filename)
        super(MyFileLogger, self).__init__(filename, mode, encoding, delay)
        # logging.Formatter = "[%(asctime)s] %(levelname)s - %(name)s | %(message)s"


@task(max_retries=3, retry_delay=timedelta(seconds=1), name='pull_project', log_stdout=True)
def pull_project(project_name):
    logger = prefect.context.get("logger")
    logger.info(f"Pull latest {project_name} version from git.")
    logger.warning("A warning message....")
    return {'project_name': project_name}


def main():
    with Flow("test") as flow:
        project_info = pull_project(project_name='test')


    file_logger = get_logger()
    file_logger.addHandler(MyFileLogger('auto_ai.log'))

    flow.run()



if __name__ == "__main__":
    main()

Any suggestions on how to make the log file mirror the stdout ?

1 Answers

Change MyFileLogger to

class MyFileLogger(logging.FileHandler):

    def __init__(self, filename, mode='a', encoding=None, delay=False):
        filename = os.path.join(LOG_PATH, filename)
        super(MyFileLogger, self).__init__(filename, mode, encoding, delay)
        self.setFormatter(logging.Formatter("[%(asctime)s] %(levelname)s - %(name)s | %(message)s"))

Solved the issue

Related