Coverage for fluent/handler.py: 100%
123 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-02-29 15:22 +0900
« prev ^ index » next coverage.py v7.4.3, created at 2024-02-29 15:22 +0900
1import json
2import logging
3import socket
5from fluent import sender
8class FluentRecordFormatter(logging.Formatter):
9 """A structured formatter for Fluent.
11 Best used with server storing data in an ElasticSearch cluster for example.
13 :param fmt: a dict or a callable with format string as values to map to provided keys.
14 If callable, should accept a single argument `LogRecord` and return a dict,
15 and have a field `usesTime` that is callable and return a bool as would
16 `FluentRecordFormatter.usesTime`
17 :param datefmt: strftime()-compatible date/time format string.
18 :param style: '%', '{' or '$' (used only with Python 3.2 or above)
19 :param fill_missing_fmt_key: if True, do not raise a KeyError if the format
20 key is not found. Put None if not found.
21 :param format_json: if True, will attempt to parse message as json. If not,
22 will use message as-is. Defaults to True
23 :param exclude_attrs: switches this formatter into a mode where all attributes
24 except the ones specified by `exclude_attrs` are logged with the record as is.
25 If `None`, operates as before, otherwise `fmt` is ignored.
26 Can be an iterable.
27 """
29 def __init__(
30 self,
31 fmt=None,
32 datefmt=None,
33 style="%",
34 fill_missing_fmt_key=False,
35 format_json=True,
36 exclude_attrs=None,
37 ):
38 super().__init__(None, datefmt)
40 if style != "%":
41 self.__style, basic_fmt_dict = {
42 "{": (
43 logging.StrFormatStyle,
44 {
45 "sys_host": "{hostname}",
46 "sys_name": "{name}",
47 "sys_module": "{module}",
48 },
49 ),
50 "$": (
51 logging.StringTemplateStyle,
52 {
53 "sys_host": "${hostname}",
54 "sys_name": "${name}",
55 "sys_module": "${module}",
56 },
57 ),
58 }[style]
59 else:
60 self.__style = None
61 basic_fmt_dict = {
62 "sys_host": "%(hostname)s",
63 "sys_name": "%(name)s",
64 "sys_module": "%(module)s",
65 }
67 if exclude_attrs is not None:
68 self._exc_attrs = set(exclude_attrs)
69 self._fmt_dict = None
70 self._formatter = self._format_by_exclusion
71 self.usesTime = super().usesTime
72 else:
73 self._exc_attrs = None
74 if not fmt:
75 self._fmt_dict = basic_fmt_dict
76 self._formatter = self._format_by_dict
77 self.usesTime = self._format_by_dict_uses_time
78 else:
79 if hasattr(fmt, "__call__"):
80 self._formatter = fmt
81 self.usesTime = fmt.usesTime
82 else:
83 self._fmt_dict = fmt
84 self._formatter = self._format_by_dict
85 self.usesTime = self._format_by_dict_uses_time
87 if format_json:
88 self._format_msg = self._format_msg_json
89 else:
90 self._format_msg = self._format_msg_default
92 self.hostname = socket.gethostname()
94 self.fill_missing_fmt_key = fill_missing_fmt_key
96 def format(self, record):
97 # Compute attributes handled by parent class.
98 super().format(record)
99 # Add ours
100 record.hostname = self.hostname
102 # Apply format
103 data = self._formatter(record)
105 self._structuring(data, record)
106 return data
108 def usesTime(self):
109 """This method is substituted on construction based on settings for performance reasons"""
111 def _structuring(self, data, record):
112 """Melds `msg` into `data`.
114 :param data: dictionary to be sent to fluent server
115 :param msg: :class:`LogRecord`'s message to add to `data`.
116 `msg` can be a simple string for backward compatibility with
117 :mod:`logging` framework, a JSON encoded string or a dictionary
118 that will be merged into dictionary generated in :meth:`format.
119 """
120 msg = record.msg
122 if isinstance(msg, dict):
123 self._add_dic(data, msg)
124 elif isinstance(msg, str):
125 self._add_dic(data, self._format_msg(record, msg))
126 else:
127 self._add_dic(data, {"message": msg})
129 def _format_msg_json(self, record, msg):
130 try:
131 json_msg = json.loads(str(msg))
132 if isinstance(json_msg, dict):
133 return json_msg
134 else:
135 return self._format_msg_default(record, msg)
136 except ValueError:
137 return self._format_msg_default(record, msg)
139 def _format_msg_default(self, record, msg):
140 return {"message": super().format(record)}
142 def _format_by_exclusion(self, record):
143 data = {}
144 for key, value in record.__dict__.items():
145 if key not in self._exc_attrs:
146 data[key] = value
147 return data
149 def _format_by_dict(self, record):
150 data = {}
151 for key, value in self._fmt_dict.items():
152 try:
153 if self.__style:
154 value = self.__style(value).format(record)
155 else:
156 value = value % record.__dict__
157 except KeyError as exc:
158 value = None
159 if not self.fill_missing_fmt_key:
160 raise exc
162 data[key] = value
163 return data
165 def _format_by_dict_uses_time(self):
166 if self.__style:
167 search = self.__style.asctime_search
168 else:
169 search = "%(asctime)"
170 return any([value.find(search) >= 0 for value in self._fmt_dict.values()])
172 @staticmethod
173 def _add_dic(data, dic):
174 for key, value in dic.items():
175 if isinstance(key, str):
176 data[key] = value
179class FluentHandler(logging.Handler):
180 """
181 Logging Handler for fluent.
182 """
184 def __init__(
185 self,
186 tag,
187 host="localhost",
188 port=24224,
189 timeout=3.0,
190 verbose=False,
191 buffer_overflow_handler=None,
192 msgpack_kwargs=None,
193 nanosecond_precision=False,
194 **kwargs,
195 ):
196 self.tag = tag
197 self._host = host
198 self._port = port
199 self._timeout = timeout
200 self._verbose = verbose
201 self._buffer_overflow_handler = buffer_overflow_handler
202 self._msgpack_kwargs = msgpack_kwargs
203 self._nanosecond_precision = nanosecond_precision
204 self._kwargs = kwargs
205 self._sender = None
206 logging.Handler.__init__(self)
208 def getSenderClass(self):
209 return sender.FluentSender
211 @property
212 def sender(self):
213 if self._sender is None:
214 self._sender = self.getSenderInstance(
215 tag=self.tag,
216 host=self._host,
217 port=self._port,
218 timeout=self._timeout,
219 verbose=self._verbose,
220 buffer_overflow_handler=self._buffer_overflow_handler,
221 msgpack_kwargs=self._msgpack_kwargs,
222 nanosecond_precision=self._nanosecond_precision,
223 **self._kwargs,
224 )
225 return self._sender
227 def getSenderInstance(
228 self,
229 tag,
230 host,
231 port,
232 timeout,
233 verbose,
234 buffer_overflow_handler,
235 msgpack_kwargs,
236 nanosecond_precision,
237 **kwargs,
238 ):
239 sender_class = self.getSenderClass()
240 return sender_class(
241 tag,
242 host=host,
243 port=port,
244 timeout=timeout,
245 verbose=verbose,
246 buffer_overflow_handler=buffer_overflow_handler,
247 msgpack_kwargs=msgpack_kwargs,
248 nanosecond_precision=nanosecond_precision,
249 **kwargs,
250 )
252 def emit(self, record):
253 data = self.format(record)
254 _sender = self.sender
255 return _sender.emit_with_time(
256 None,
257 sender.EventTime(record.created)
258 if _sender.nanosecond_precision
259 else int(record.created),
260 data,
261 )
263 def close(self):
264 self.acquire()
265 try:
266 try:
267 self.sender.close()
268 finally:
269 super().close()
270 finally:
271 self.release()
273 def __enter__(self):
274 return self
276 def __exit__(self, exc_type, exc_val, exc_tb):
277 self.close()