Trying to parse a log file and create a CVS report with tables in python. The code returns empty tables for some reason

user_15126492 0
Viewed 976

The assignment:
Imagine your company uses a server that runs a service called ticky, an internal ticketing system. The service logs events to syslog, both when it runs successfully and when it encounters errors. The service's developers need your help getting some information from those logs so that they can better understand how their software is used and how to improve it. So, for this lab, you'll write some automation scripts that will process the system log and generate reports based on the information extracted from the log files.

What you'll do Use regex to parse a log file Append and modify values in a dictionary Write to a file in CSV format Move files to the appropriate directory for use with the CSV->HTML converter

Also, here is example from the log file:

Jan 31 16:35:46 ubuntu.local ticky: ERROR Timeout while retrieving information (oren)
Jan 31 16:53:54 ubuntu.local ticky: INFO Commented on ticket [#3813] (mcintosh)
Jan 31 16:54:18 ubuntu.local ticky: ERROR Connection to DB failed (bpacheco)
Jan 31 17:15:47 ubuntu.local ticky: ERROR The ticket was modified while updating (mcintosh)
Jan 31 17:29:11 ubuntu.local ticky: ERROR Connection to DB failed (oren)
Jan 31 17:51:52 ubuntu.local ticky: INFO Closed ticket [#8604] (mcintosh)

Here is my code, which returns empty tables:

 #!/usr/bin/env python3
    
import sys
import re
import csv
import operator

errors = {} #number of diff error messages
per_user = {}

errors_report = 'error_report.csv'
per_user_report = 'per_user_report.csv'

logfile='syslog.log'

error_pattern = r'\w* [\d]* [\d?:]*[\w?' '|.]*: [ERROR]*: (?P<message>[\w+' ']*)\[\#\d+\] \((?P<username>\w+)\)'
info_pattern = r'\w* [\d]* [\d?:]*[\w?' '|.]*: [INFO]*: [\w+' ']*\[\#\d+\] \((?P<username>\w+)\)'

with open(logfile, 'r') as file:
  for line in file.readlines():
    if re.search(error_pattern, line):
      error_result = re.search(error_pattern, line)
      errors.setdefault(error_result.group('message'),0)
      errors[error_result.group('message')]+=1
      per_user.setdefault(error_result.group('username'),[0,0])[1]+=1
    if re.search(info_pattern, line):
      info = re.search(info_pattern, line)
      per_user.setdefault(info.group('username'),[0,0])[0]+=1

error_sorted=sorted(errors.items(), key=operator.itemgetter(1), reverse = True)
per_user_sorted=sorted(per_user.items())

with open('error_report.csv', 'w') as error_report:
  writer = csv.writer(error_report)
  writer.writerow(['Error', 'Count'])
  writer.writerows(error_sorted)

with open('per_user_report.csv', 'w') as user_report:
  writer = csv.writer(user_report)
  writer.writerow(['Username', 'INFO', 'ERROR'])
  for item in per_user_sorted:
      onerow = [item[0],item[1][0],item[1][1]]
      writer.writerow(onerow)
logfile csv report python-3.x python
3 Answers
#!/usr/bin/env python3

import sys
import re
import csv
import operator

errors = {}  # number of diff error messages
per_user = {}

errors_report = 'error_message.csv'
per_user_report = 'user_statistics.csv'

logfile = 'syslog.log'


pattern = r'(?P<messageType>INFO|ERROR):?\s*(?P<message>.*?)\((?P<username>\w+)\)$'

with open(logfile, 'r') as file:
    for line in file.readlines():
        regex_result = re.search(pattern, line)
        if regex_result:
            message_type = regex_result.group('messageType')
            message = regex_result.group('message')
            username = regex_result.group('username')
            if message_type == 'ERROR':
                errors.setdefault(message, 0)
                errors[message] += 1
                per_user.setdefault(username, [0, 0])[1] += 1
            else:
                per_user.setdefault(username, [0, 0])[0] += 1

error_sorted = sorted(errors.items(), key=operator.itemgetter(1), reverse=True)
per_user_sorted = sorted(per_user.items())

with open(errors_report, 'w', newline='') as error_report:
    writer = csv.writer(error_report)
    writer.writerow(['Error', 'Count'])
    writer.writerows(error_sorted)

with open(per_user_report, 'w', newline='') as user_report:
    writer = csv.writer(user_report)
    writer.writerow(['Username', 'INFO', 'ERROR'])
    for item in per_user_sorted:
        onerow = [item[0], item[1][0], item[1][1]]
        writer.writerow(onerow)
user_151264920

If you are simply looking for usernames (according to errors or info), then you can simply look for the group that have parentheses and see if error or info in that line, something like this:

#!/usr/bin/env python3
import sys
import os
import re
import csv
import operator
per_user_errors = {}
per_error = {}
with open("syslog.log") as f:
    for log in f:
      pattern=r" \((\w+)\)$"
      result = re.search(pattern,log)
      user=result[1]
      if "ERROR" in log:
          if user not in per_user_errors:
              per_user_errors[user]={"ERRORS":0,"INFO":0}
          per_user_errors[user]["ERRORS"]+=1
          error_pattern=r"[EROR]{5} .* \("
          res=re.search(error_pattern,log)
          error=res.group()[6:-2]
          if error not in per_error:
              per_error[error] = 0
          per_error[error] += 1
      elif "INFO" in log:
          if user not in per_user_errors:
              per_user_errors[user]={"ERRORS":0,"INFO":0}
          per_user_errors[user]["INFO"]+=1
          
a=dict(sorted(per_user_errors.items(), key=lambda item:item[0]))          
b=dict(sorted(per_error.items(), key=lambda item:item[1], reverse=True))          

with open("per_user_report.csv","w") as file:
    please = csv.writer(file)
    please.writerow(['Username', 'INFO', 'ERROR'])
    for i in a:
        please.writerow([i, a[i]["ERRORS"], a[i]["INFO"]])

with open("per_error_report.csv","w") as csvf:
    please = csv.writer(csvf)
    please.writerow(["Error_message", "Count"])
    for m in b:
        please.writerow([m, b[m]])
user_151208200

would you please share some more details on what you want to achieve, It seems like your regex is not picking anything from the syslog file. What are you looking for exactly on those errors?

user_151208200
Related