Coverage for fluent/handler.py: 100%

123 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-02-29 15:22 +0900

1import json 

2import logging 

3import socket 

4 

5from fluent import sender 

6 

7 

8class FluentRecordFormatter(logging.Formatter): 

9 """A structured formatter for Fluent. 

10 

11 Best used with server storing data in an ElasticSearch cluster for example. 

12 

13 :param fmt: a dict or a callable with format string as values to map to provided keys. 

14 If callable, should accept a single argument `LogRecord` and return a dict, 

15 and have a field `usesTime` that is callable and return a bool as would 

16 `FluentRecordFormatter.usesTime` 

17 :param datefmt: strftime()-compatible date/time format string. 

18 :param style: '%', '{' or '$' (used only with Python 3.2 or above) 

19 :param fill_missing_fmt_key: if True, do not raise a KeyError if the format 

20 key is not found. Put None if not found. 

21 :param format_json: if True, will attempt to parse message as json. If not, 

22 will use message as-is. Defaults to True 

23 :param exclude_attrs: switches this formatter into a mode where all attributes 

24 except the ones specified by `exclude_attrs` are logged with the record as is. 

25 If `None`, operates as before, otherwise `fmt` is ignored. 

26 Can be an iterable. 

27 """ 

28 

29 def __init__( 

30 self, 

31 fmt=None, 

32 datefmt=None, 

33 style="%", 

34 fill_missing_fmt_key=False, 

35 format_json=True, 

36 exclude_attrs=None, 

37 ): 

38 super().__init__(None, datefmt) 

39 

40 if style != "%": 

41 self.__style, basic_fmt_dict = { 

42 "{": ( 

43 logging.StrFormatStyle, 

44 { 

45 "sys_host": "{hostname}", 

46 "sys_name": "{name}", 

47 "sys_module": "{module}", 

48 }, 

49 ), 

50 "$": ( 

51 logging.StringTemplateStyle, 

52 { 

53 "sys_host": "${hostname}", 

54 "sys_name": "${name}", 

55 "sys_module": "${module}", 

56 }, 

57 ), 

58 }[style] 

59 else: 

60 self.__style = None 

61 basic_fmt_dict = { 

62 "sys_host": "%(hostname)s", 

63 "sys_name": "%(name)s", 

64 "sys_module": "%(module)s", 

65 } 

66 

67 if exclude_attrs is not None: 

68 self._exc_attrs = set(exclude_attrs) 

69 self._fmt_dict = None 

70 self._formatter = self._format_by_exclusion 

71 self.usesTime = super().usesTime 

72 else: 

73 self._exc_attrs = None 

74 if not fmt: 

75 self._fmt_dict = basic_fmt_dict 

76 self._formatter = self._format_by_dict 

77 self.usesTime = self._format_by_dict_uses_time 

78 else: 

79 if hasattr(fmt, "__call__"): 

80 self._formatter = fmt 

81 self.usesTime = fmt.usesTime 

82 else: 

83 self._fmt_dict = fmt 

84 self._formatter = self._format_by_dict 

85 self.usesTime = self._format_by_dict_uses_time 

86 

87 if format_json: 

88 self._format_msg = self._format_msg_json 

89 else: 

90 self._format_msg = self._format_msg_default 

91 

92 self.hostname = socket.gethostname() 

93 

94 self.fill_missing_fmt_key = fill_missing_fmt_key 

95 

96 def format(self, record): 

97 # Compute attributes handled by parent class. 

98 super().format(record) 

99 # Add ours 

100 record.hostname = self.hostname 

101 

102 # Apply format 

103 data = self._formatter(record) 

104 

105 self._structuring(data, record) 

106 return data 

107 

108 def usesTime(self): 

109 """This method is substituted on construction based on settings for performance reasons""" 

110 

111 def _structuring(self, data, record): 

112 """Melds `msg` into `data`. 

113 

114 :param data: dictionary to be sent to fluent server 

115 :param msg: :class:`LogRecord`'s message to add to `data`. 

116 `msg` can be a simple string for backward compatibility with 

117 :mod:`logging` framework, a JSON encoded string or a dictionary 

118 that will be merged into dictionary generated in :meth:`format. 

119 """ 

120 msg = record.msg 

121 

122 if isinstance(msg, dict): 

123 self._add_dic(data, msg) 

124 elif isinstance(msg, str): 

125 self._add_dic(data, self._format_msg(record, msg)) 

126 else: 

127 self._add_dic(data, {"message": msg}) 

128 

129 def _format_msg_json(self, record, msg): 

130 try: 

131 json_msg = json.loads(str(msg)) 

132 if isinstance(json_msg, dict): 

133 return json_msg 

134 else: 

135 return self._format_msg_default(record, msg) 

136 except ValueError: 

137 return self._format_msg_default(record, msg) 

138 

139 def _format_msg_default(self, record, msg): 

140 return {"message": super().format(record)} 

141 

142 def _format_by_exclusion(self, record): 

143 data = {} 

144 for key, value in record.__dict__.items(): 

145 if key not in self._exc_attrs: 

146 data[key] = value 

147 return data 

148 

149 def _format_by_dict(self, record): 

150 data = {} 

151 for key, value in self._fmt_dict.items(): 

152 try: 

153 if self.__style: 

154 value = self.__style(value).format(record) 

155 else: 

156 value = value % record.__dict__ 

157 except KeyError as exc: 

158 value = None 

159 if not self.fill_missing_fmt_key: 

160 raise exc 

161 

162 data[key] = value 

163 return data 

164 

165 def _format_by_dict_uses_time(self): 

166 if self.__style: 

167 search = self.__style.asctime_search 

168 else: 

169 search = "%(asctime)" 

170 return any([value.find(search) >= 0 for value in self._fmt_dict.values()]) 

171 

172 @staticmethod 

173 def _add_dic(data, dic): 

174 for key, value in dic.items(): 

175 if isinstance(key, str): 

176 data[key] = value 

177 

178 

179class FluentHandler(logging.Handler): 

180 """ 

181 Logging Handler for fluent. 

182 """ 

183 

184 def __init__( 

185 self, 

186 tag, 

187 host="localhost", 

188 port=24224, 

189 timeout=3.0, 

190 verbose=False, 

191 buffer_overflow_handler=None, 

192 msgpack_kwargs=None, 

193 nanosecond_precision=False, 

194 **kwargs, 

195 ): 

196 self.tag = tag 

197 self._host = host 

198 self._port = port 

199 self._timeout = timeout 

200 self._verbose = verbose 

201 self._buffer_overflow_handler = buffer_overflow_handler 

202 self._msgpack_kwargs = msgpack_kwargs 

203 self._nanosecond_precision = nanosecond_precision 

204 self._kwargs = kwargs 

205 self._sender = None 

206 logging.Handler.__init__(self) 

207 

208 def getSenderClass(self): 

209 return sender.FluentSender 

210 

211 @property 

212 def sender(self): 

213 if self._sender is None: 

214 self._sender = self.getSenderInstance( 

215 tag=self.tag, 

216 host=self._host, 

217 port=self._port, 

218 timeout=self._timeout, 

219 verbose=self._verbose, 

220 buffer_overflow_handler=self._buffer_overflow_handler, 

221 msgpack_kwargs=self._msgpack_kwargs, 

222 nanosecond_precision=self._nanosecond_precision, 

223 **self._kwargs, 

224 ) 

225 return self._sender 

226 

227 def getSenderInstance( 

228 self, 

229 tag, 

230 host, 

231 port, 

232 timeout, 

233 verbose, 

234 buffer_overflow_handler, 

235 msgpack_kwargs, 

236 nanosecond_precision, 

237 **kwargs, 

238 ): 

239 sender_class = self.getSenderClass() 

240 return sender_class( 

241 tag, 

242 host=host, 

243 port=port, 

244 timeout=timeout, 

245 verbose=verbose, 

246 buffer_overflow_handler=buffer_overflow_handler, 

247 msgpack_kwargs=msgpack_kwargs, 

248 nanosecond_precision=nanosecond_precision, 

249 **kwargs, 

250 ) 

251 

252 def emit(self, record): 

253 data = self.format(record) 

254 _sender = self.sender 

255 return _sender.emit_with_time( 

256 None, 

257 sender.EventTime(record.created) 

258 if _sender.nanosecond_precision 

259 else int(record.created), 

260 data, 

261 ) 

262 

263 def close(self): 

264 self.acquire() 

265 try: 

266 try: 

267 self.sender.close() 

268 finally: 

269 super().close() 

270 finally: 

271 self.release() 

272 

273 def __enter__(self): 

274 return self 

275 

276 def __exit__(self, exc_type, exc_val, exc_tb): 

277 self.close()