Coverage for curator/logtools.py: 85%
68 statements
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-16 15:27 -0600
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-16 15:27 -0600
1"""Logging tools"""
2import sys
3import json
4import logging
5import time
6from curator.exceptions import LoggingException
8def de_dot(dot_string, msg):
9 """Turn message and dotted string into a nested dictionary"""
10 arr = dot_string.split('.')
11 arr.append(msg)
12 retval = None
13 for idx in range(len(arr), 1, -1):
14 if not retval:
15 try:
16 retval = {arr[idx-2]: arr[idx-1]}
17 except Exception as err:
18 raise LoggingException(err)
19 else:
20 try:
21 new_d = {arr[idx-2]: retval}
22 retval = new_d
23 except Exception as err:
24 raise LoggingException(err)
25 return retval
27def deepmerge(source, destination):
28 """Merge deeply nested dictionary structures"""
29 for key, value in source.items():
30 if isinstance(value, dict):
31 node = destination.setdefault(key, {})
32 deepmerge(value, node)
33 else:
34 destination[key] = value
35 return destination
36class LogstashFormatter(logging.Formatter):
37 """Logstash formatting (JSON)"""
38 # The LogRecord attributes we want to carry over to the Logstash message,
39 # mapped to the corresponding output key.
40 WANTED_ATTRS = {
41 'levelname': 'loglevel',
42 'funcName': 'function',
43 'lineno': 'linenum',
44 'message': 'message',
45 'name': 'name'
46 }
48 def format(self, record):
49 self.converter = time.gmtime
50 timestamp = '%s.%03dZ' % (
51 self.formatTime(record, datefmt='%Y-%m-%dT%H:%M:%S'), record.msecs)
52 result = {'@timestamp': timestamp}
53 available = record.__dict__
54 # This is cleverness because 'message' is NOT a member key of ``record.__dict__``
55 # the ``getMessage()`` method is effectively ``msg % args`` (actual keys)
56 # By manually adding 'message' to ``available``, it simplifies the code
57 available['message'] = record.getMessage()
58 for attribute in set(self.WANTED_ATTRS).intersection(available):
59 result = deepmerge(
60 de_dot(self.WANTED_ATTRS[attribute], getattr(record, attribute)), result
61 )
62 # The following is mostly for the ecs format. You can't have 2x 'message' keys in
63 # WANTED_ATTRS, so we set the value to 'log.original' in ecs, and this code block
64 # guarantees it still appears as 'message' too.
65 if 'message' not in result.items():
66 result['message'] = available['message']
67 return json.dumps(result, sort_keys=True)
69class ECSFormatter(LogstashFormatter):
70 """Elastic Common Schema formatting (ECS)"""
71 # Overload LogstashFormatter attribute
72 WANTED_ATTRS = {
73 'levelname': 'log.level',
74 'funcName': 'log.origin.function',
75 'lineno': 'log.origin.file.line',
76 'message': 'log.original',
77 'name': 'log.logger'
78 }
80class Whitelist(logging.Filter):
81 """How to whitelist logs"""
82 def __init__(self, *whitelist):
83 self.whitelist = [logging.Filter(name) for name in whitelist]
85 def filter(self, record):
86 return any(f.filter(record) for f in self.whitelist)
88class Blacklist(Whitelist):
89 """Blacklist monkey-patch of Whitelist"""
90 def filter(self, record):
91 return not Whitelist.filter(self, record)
93class LogInfo(object):
94 """Logging Class"""
95 def __init__(self, cfg):
96 cfg['loglevel'] = 'INFO' if not 'loglevel' in cfg else cfg['loglevel']
97 cfg['logfile'] = None if not 'logfile' in cfg else cfg['logfile']
98 cfg['logformat'] = 'default' if not 'logformat' in cfg else cfg['logformat']
99 self.numeric_log_level = getattr(logging, cfg['loglevel'].upper(), None)
100 self.format_string = '%(asctime)s %(levelname)-9s %(message)s'
101 if not isinstance(self.numeric_log_level, int):
102 raise ValueError('Invalid log level: {0}'.format(cfg['loglevel']))
104 self.handler = logging.StreamHandler(
105 open(cfg['logfile'], 'a') if cfg['logfile'] else sys.stdout
106 )
108 if self.numeric_log_level == 10: # DEBUG
109 self.format_string = (
110 '%(asctime)s %(levelname)-9s %(name)22s '
111 '%(funcName)22s:%(lineno)-4d %(message)s'
112 )
114 if cfg['logformat'] == 'json' or cfg['logformat'] == 'logstash':
115 self.handler.setFormatter(LogstashFormatter())
116 elif cfg['logformat'] == 'ecs':
117 self.handler.setFormatter(ECSFormatter())
118 else:
119 self.handler.setFormatter(logging.Formatter(self.format_string))