Unable to create logfile minute wise in python while using multiprocessing

The problem is that the logger uses a threading.RLock to provide thread safety between multiple threads logging to the same file but cannot really handle multiple processes logging to the same file.

Possibly the best solution is to create a managed logging object that "lives" in a separate address space owned by a subclass of multiprocessing.BaseManager, which starts a thread that waits for method requests to this logging object. What is actually being passed to your processes is a proxy object that makes the equivalent of a remote method call to the aforementioned managed logger, which then executes these calls single-threaded.

I have modified the code so that it is more likely to fail if there is a problem and also more likely to terminate.

import multiprocessing
from multiprocessing.managers import BaseManager
import logging.handlers


def init_log():
    log_file_name = "test.log"
    logging_level = logging.DEBUG
    formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
    handler = logging.handlers.TimedRotatingFileHandler(log_file_name, when='M', interval=1)
    handler.suffix = "%Y-%m-%d"
    handler.setFormatter(formatter)
    logger = logging.getLogger(log_file_name)
    logger.addHandler(handler)
    logger.setLevel(logging_level)
    return logger


class LoggerManager(BaseManager):
    pass


class MyLogger:
    def __init__(self):
        self.logger = init_log()

    def info(self, *args, **kwargs):
        self.logger.info(*args, **kwargs)

    # define other methods such as degug if required:



def main_1(logger):
    print("main 1")
    for _ in range(20):
        logger.info("A Sample Log Statement main_1")

def main_2(logger):
    print("main 2")
    for _ in range(20):
        logger.info("A Sample Log Statement main_2")


def isReady():
    LoggerManager.register('Logger', MyLogger)
    with LoggerManager() as manager:
        my_logger = manager.Logger()
        for _ in range(5):
            p1 = multiprocessing.Process(target=main_1, args=(my_logger,))
            p2 = multiprocessing.Process(target=main_2, args=(my_logger,))
            p1.start()
            p2.start()
            p1.join()
            p2.join()

if __name__ == "__main__":
    isReady()

An alterative is to solve the problem by subclassing logging.handlers.TimedRotatingFileHandler and instantiating it with a multiprocessing.RLock instance. I then override the base class's createLock method by initializing the lock attribute with the multiprocessing.RLock instance. Needless to say, all processes must be using the same multiprocessing.RLock instance.

import multiprocessing
import logging.handlers


class MyHandler(logging.handlers.TimedRotatingFileHandler):
    def __init__(self, lock, *args, **kwargs):
        self._mp_lock = lock # for multiprocessing "thread" safety
        super().__init__(*args, **kwargs)

    def createLock(self):
        self.lock = self._mp_lock


def init_log(lock):
    global logger

    log_file_name = "test.log"
    logging_level = logging.DEBUG
    formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
    handler = MyHandler(lock, log_file_name, when='M', interval=1)
    handler.suffix = "%Y-%m-%d"
    handler.setFormatter(formatter)
    logger = logging.getLogger(log_file_name)
    logger.addHandler(handler)
    logger.setLevel(logging_level)

def main_1(lock):
    init_log(lock)
    print("main 1")
    for _ in range(20):
        logger.info("A Sample Log Statement main_1")

def main_2(lock):
    init_log(lock)
    print("main 2")
    for _ in range(20):
        logger.info("A Sample Log Statement main_2")


def isReady():
    lock = multiprocessing.RLock()
    for _ in range(5):
        p1 = multiprocessing.Process(target=main_1, args=(lock,))
        p2 = multiprocessing.Process(target=main_2, args=(lock,))
        p1.start()
        p2.start()
        p1.join()
        p2.join()

if __name__ == "__main__":
    isReady()