Coverage for curator/logtools.py: 85%

68 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-16 15:27 -0600

1"""Logging tools""" 

2import sys 

3import json 

4import logging 

5import time 

6from curator.exceptions import LoggingException 

7 

8def de_dot(dot_string, msg): 

9 """Turn message and dotted string into a nested dictionary""" 

10 arr = dot_string.split('.') 

11 arr.append(msg) 

12 retval = None 

13 for idx in range(len(arr), 1, -1): 

14 if not retval: 

15 try: 

16 retval = {arr[idx-2]: arr[idx-1]} 

17 except Exception as err: 

18 raise LoggingException(err) 

19 else: 

20 try: 

21 new_d = {arr[idx-2]: retval} 

22 retval = new_d 

23 except Exception as err: 

24 raise LoggingException(err) 

25 return retval 

26 

27def deepmerge(source, destination): 

28 """Merge deeply nested dictionary structures""" 

29 for key, value in source.items(): 

30 if isinstance(value, dict): 

31 node = destination.setdefault(key, {}) 

32 deepmerge(value, node) 

33 else: 

34 destination[key] = value 

35 return destination 

36class LogstashFormatter(logging.Formatter): 

37 """Logstash formatting (JSON)""" 

38 # The LogRecord attributes we want to carry over to the Logstash message, 

39 # mapped to the corresponding output key. 

40 WANTED_ATTRS = { 

41 'levelname': 'loglevel', 

42 'funcName': 'function', 

43 'lineno': 'linenum', 

44 'message': 'message', 

45 'name': 'name' 

46 } 

47 

48 def format(self, record): 

49 self.converter = time.gmtime 

50 timestamp = '%s.%03dZ' % ( 

51 self.formatTime(record, datefmt='%Y-%m-%dT%H:%M:%S'), record.msecs) 

52 result = {'@timestamp': timestamp} 

53 available = record.__dict__ 

54 # This is cleverness because 'message' is NOT a member key of ``record.__dict__`` 

55 # the ``getMessage()`` method is effectively ``msg % args`` (actual keys) 

56 # By manually adding 'message' to ``available``, it simplifies the code 

57 available['message'] = record.getMessage() 

58 for attribute in set(self.WANTED_ATTRS).intersection(available): 

59 result = deepmerge( 

60 de_dot(self.WANTED_ATTRS[attribute], getattr(record, attribute)), result 

61 ) 

62 # The following is mostly for the ecs format. You can't have 2x 'message' keys in 

63 # WANTED_ATTRS, so we set the value to 'log.original' in ecs, and this code block 

64 # guarantees it still appears as 'message' too. 

65 if 'message' not in result.items(): 

66 result['message'] = available['message'] 

67 return json.dumps(result, sort_keys=True) 

68 

69class ECSFormatter(LogstashFormatter): 

70 """Elastic Common Schema formatting (ECS)""" 

71 # Overload LogstashFormatter attribute 

72 WANTED_ATTRS = { 

73 'levelname': 'log.level', 

74 'funcName': 'log.origin.function', 

75 'lineno': 'log.origin.file.line', 

76 'message': 'log.original', 

77 'name': 'log.logger' 

78 } 

79 

80class Whitelist(logging.Filter): 

81 """How to whitelist logs""" 

82 def __init__(self, *whitelist): 

83 self.whitelist = [logging.Filter(name) for name in whitelist] 

84 

85 def filter(self, record): 

86 return any(f.filter(record) for f in self.whitelist) 

87 

88class Blacklist(Whitelist): 

89 """Blacklist monkey-patch of Whitelist""" 

90 def filter(self, record): 

91 return not Whitelist.filter(self, record) 

92 

93class LogInfo(object): 

94 """Logging Class""" 

95 def __init__(self, cfg): 

96 cfg['loglevel'] = 'INFO' if not 'loglevel' in cfg else cfg['loglevel'] 

97 cfg['logfile'] = None if not 'logfile' in cfg else cfg['logfile'] 

98 cfg['logformat'] = 'default' if not 'logformat' in cfg else cfg['logformat'] 

99 self.numeric_log_level = getattr(logging, cfg['loglevel'].upper(), None) 

100 self.format_string = '%(asctime)s %(levelname)-9s %(message)s' 

101 if not isinstance(self.numeric_log_level, int): 

102 raise ValueError('Invalid log level: {0}'.format(cfg['loglevel'])) 

103 

104 self.handler = logging.StreamHandler( 

105 open(cfg['logfile'], 'a') if cfg['logfile'] else sys.stdout 

106 ) 

107 

108 if self.numeric_log_level == 10: # DEBUG 

109 self.format_string = ( 

110 '%(asctime)s %(levelname)-9s %(name)22s ' 

111 '%(funcName)22s:%(lineno)-4d %(message)s' 

112 ) 

113 

114 if cfg['logformat'] == 'json' or cfg['logformat'] == 'logstash': 

115 self.handler.setFormatter(LogstashFormatter()) 

116 elif cfg['logformat'] == 'ecs': 

117 self.handler.setFormatter(ECSFormatter()) 

118 else: 

119 self.handler.setFormatter(logging.Formatter(self.format_string))