Rotating file handler for JSON logs in Python

Python does not care about the log file name.

You can use the rotating handler which you used for .log file for .json file also.

See sample example below

# logging_example.py

import logging
import logging.handlers
import os
import time

logfile = os.path.join("/tmp", "demo_logging.json")

logger = logging.getLogger(__name__)

fh = logging.handlers.RotatingFileHandler(logfile, mode='a', maxBytes=1000, backupCount=5)  # noqa:E501

fh.setLevel(logging.DEBUG)

formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")

fh.setFormatter(formatter)

logger.addHandler(fh)
logger.setLevel(logging.DEBUG)

while 1:
    time.sleep(1)
    logger.info("Long string to increase the file size")

You can also look at logrotate if you are working in Unix environment. It is a great and simple tool with good documentation to do just exactly what you need.

1. You can implement structured logging with RotatingFileHandler

import json
import logging
import logging.handlers
from datetime import datetime

class StructuredMessage:
    def __init__(self, message, /, **kwargs):
        self.message = message
        self.kwargs = kwargs

    def __str__(self):
        return '%s >>> %s' % (self.message, json.dumps(self.kwargs))

_ = StructuredMessage   # optional, to improve readability

log_json = {}
log_json["Data"] = {}

log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"

log_json['Created'] = datetime.utcnow().isoformat()

LOG_FILENAME = 'logging_rotatingfile_example.out'

# Set up a specific logger with our desired output level
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

# Add the log message handler to the logger
handler = logging.handlers.RotatingFileHandler(
              LOG_FILENAME, maxBytes=20, backupCount=5)
bf = logging.Formatter('%(message)s')
handler.setFormatter(bf)

logger.addHandler(handler)
logger.info(_('INFO', **log_json))

Note: check here for more info about structured-logging-python

2. you can use also use json-logging-python with RotatingFileHandler

import logging
import json
import traceback
from datetime import datetime
import copy
import json_logging
import sys

json_logging.ENABLE_JSON_LOGGING = True


def extra(**kw):
    '''Add the required nested props layer'''
    return {'extra': {'props': kw}}


class CustomJSONLog(logging.Formatter):
    """
    Customized logger
    """

    def get_exc_fields(self, record):
        if record.exc_info:
            exc_info = self.format_exception(record.exc_info)
        else:
            exc_info = record.exc_text
        return {'python.exc_info': exc_info}

    @classmethod
    def format_exception(cls, exc_info):
        return ''.join(traceback.format_exception(*exc_info)) if exc_info else ''

    def format(self, record):
        json_log_object = {"@timestamp": datetime.utcnow().isoformat(),
                           "level": record.levelname,
                           "message": record.getMessage(),
                           "caller": record.filename + '::' + record.funcName
                           }
        json_log_object['data'] = {
            "python.logger_name": record.name,
            "python.module": record.module,
            "python.funcName": record.funcName,
            "python.filename": record.filename,
            "python.lineno": record.lineno,
            "python.thread": record.threadName,
            "python.pid": record.process
        }
        if hasattr(record, 'props'):
            json_log_object['data'].update(record.props)

        if record.exc_info or record.exc_text:
            json_log_object['data'].update(self.get_exc_fields(record))

        return json.dumps(json_log_object)


json_logging.init_non_web(custom_formatter=CustomJSONLog, enable_json=True)

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
LOG_FILENAME = 'logging_rotating_json_example.out'

handler = logging.handlers.RotatingFileHandler(
              LOG_FILENAME, maxBytes=20, backupCount=5)

logger.addHandler(handler)

log_json = {}
log_json["Data"] = {}

log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"

logger.info('Starting')
logger.debug('Working', extra={"props":log_json})

Note: check here for more info about json-logging-python

You can try this just before you write / append to the file. This should check to see if the file has reached the max lines size, then it will remove one line of code from the beginning of the file before you append as usual to the end of the file.

filename = 'file.txt'
maxLines = 100

count = len(open(filename).readlines())

if(count > maxLines) {
  with open(filename, 'r') as fin:
    data = fin.read().splitlines(True)
  with open(filename, 'w') as fout:
    fout.writelines(data[1:])
}