codekingpro/portable-devtools
114k
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4"""
5File: jsonformatter.py
6Author: Me
7Email: yourname@email.com
8Github: https://github.com/yourname
9Description: jsonformatter.py
10"""
11import inspect
12import json
13import logging
14import sys
15import warnings
16from functools import wraps, partial
17from string import Template
18
19# From python3.7, dict is in ordered,so do json package's load(s)/dump(s).
20# https://docs.python.org/3.7/library/stdtypes.html#dict
21# Changed in version 3.7: Dictionary order is guaranteed to be insertion order. This behavior was an implementation detail of CPython from 3.6.
22if sys.version_info >= (3, 7):
23 dictionary = dict
24else:
25 from collections import OrderedDict
26 dictionary = OrderedDict
27
28# compatible python2, python 3 no long type, start
29if sys.version_info >= (3, 0):
30 long = int
31# compatible python2, python 3 no long type, end
32
33
34class PercentStyle(object):
35
36 asctime_format = '%(asctime)s'
37 asctime_search = '%(asctime)'
38
39 def __init__(self, fmt):
40 self._fmt = ''
41
42 def usesTime(self):
43 return self._fmt.find(self.asctime_search) >= 0
44
45 def format(self, record):
46 return self._fmt % record.__dict__
47
48
49class StrFormatStyle(PercentStyle):
50
51 asctime_format = '{asctime}'
52 asctime_search = '{asctime'
53
54 def format(self, record):
55 return self._fmt.format(**record.__dict__)
56
57
58class StringTemplateStyle(PercentStyle):
59
60 asctime_format = '${asctime}'
61 asctime_search = '${asctime}'
62
63 def __init__(self, fmt):
64 PercentStyle.__init__(self, fmt)
65 self._tpl = {}
66 for _, v in fmt.items():
67 self._tpl[v] = Template(v)
68
69 def usesTime(self):
70 fmt = self._fmt
71 return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0
72
73 def format(self, record):
74 return self._tpl[self._fmt].substitute(**record.__dict__)
75
76
77BASIC_FORMAT = dictionary([
78 ('levelname', 'levelname'),
79 ('name', 'name'),
80 ('message', 'message')
81])
82
83_STYLES = {
84 '%': PercentStyle,
85 '{': StrFormatStyle,
86 '$': StringTemplateStyle,
87}
88
89_LogRecordDefaultAttributes = {
90 'name',
91 'msg',
92 'args',
93 'levelname',
94 'levelno',
95 'pathname',
96 'filename',
97 'module',
98 'exc_info',
99 'exc_text',
100 'stack_info',
101 'lineno',
102 'funcName',
103 'created',
104 'msecs',
105 'relativeCreated',
106 'thread',
107 'threadName',
108 'processName',
109 'process',
110 'message',
111 'asctime'
112}
113
114_MIX_EXTRA_ORDER = {
115 'head',
116 'tail',
117 'mix'
118}
119
120
121class JsonFormatter(logging.Formatter):
122 """
123 Formatter instances are used to convert a LogRecord to text.
124
125 Formatters need to know how a LogRecord is constructed. They are
126 responsible for converting a LogRecord to (usually) a string which can
127 be interpreted by either a human or an external system. The base Formatter
128 allows a formatting string to be specified. If none is supplied, the
129 default value of "%s(message)" is used.
130
131 The Formatter can be initialized with a format string which makes use of
132 knowledge of the LogRecord attributes - e.g. the default value mentioned
133 above makes use of the fact that the user's message and arguments are pre-
134 formatted into a LogRecord's message attribute. Currently, the useful
135 attributes in a LogRecord are described by:
136
137 %(name)s Name of the logger (logging channel)
138 %(levelno)s Numeric logging level for the message (DEBUG, INFO,
139 WARNING, ERROR, CRITICAL)
140 %(levelname)s Text logging level for the message ("DEBUG", "INFO",
141 "WARNING", "ERROR", "CRITICAL")
142 %(pathname)s Full pathname of the source file where the logging
143 call was issued (if available)
144 %(filename)s Filename portion of pathname
145 %(module)s Module (name portion of filename)
146 %(lineno)d Source line number where the logging call was issued
147 (if available)
148 %(funcName)s Function name
149 %(created)f Time when the LogRecord was created (time.time()
150 return value)
151 %(asctime)s Textual time when the LogRecord was created
152 %(msecs)d Millisecond portion of the creation time
153 %(relativeCreated)d Time in milliseconds when the LogRecord was created,
154 relative to the time the logging module was loaded
155 (typically at application startup time)
156 %(thread)d Thread ID (if available)
157 %(threadName)s Thread name (if available)
158 %(process)d Process ID (if available)
159 %(message)s The result of record.getMessage(), computed just as
160 the record is emitted
161 """
162
163 def parseFmt(self, fmt):
164 if isinstance(fmt, str):
165 return json.loads(fmt, object_pairs_hook=dictionary)
166 elif isinstance(fmt, dictionary):
167 return fmt
168 elif isinstance(fmt, dict):
169 warnings.warn(
170 "Current python version is lower than 3.7.0, the key's order of dict may be different with definition, please use `OrderedDict` replace.", UserWarning)
171 return dictionary((k, fmt[k]) for k in sorted(fmt.keys()))
172 else:
173 raise TypeError(
174 '`%s` type is not supported, `fmt` must be `json`, `OrderedDcit` or `dict` type. ' % type(fmt))
175
176 def checkRecordCustomAttrs(self, record_custom_attrs):
177 def _patch_no_params_func_accept_kwargs(func):
178 @wraps(func)
179 def wrapper(*args, **kwargs):
180 return func(*args)
181 return wrapper
182
183 if record_custom_attrs:
184 if isinstance(record_custom_attrs, dict):
185 for attr, func in record_custom_attrs.items():
186 if not callable(func):
187 raise TypeError('`%s` is not callable.' % func)
188
189 if inspect.isfunction(func):
190 argspec = (getattr(inspect, 'getfullargspec', None) or inspect.getargspec)(func)
191 if argspec.args:
192 raise TypeError(
193 "`%s` must no Positional Parameters." % func.__name__
194 )
195 else:
196 if not (
197 getattr(argspec, 'keywords', False) or
198 getattr(argspec, 'varkw', False)
199 ):
200 record_custom_attrs[attr] = _patch_no_params_func_accept_kwargs(
201 func
202 )
203 else:
204 if isinstance(func, partial):
205 warnings.warn(
206 "`%s` is a partial function, please make sure no positional parameters in function signature." % (func), UserWarning)
207 elif hasattr(func, '__call__'):
208 warnings.warn(
209 "`%s` is a callable instance, please make sure no positional parameters in method signature." % (func), UserWarning)
210 else:
211 warnings.warn(
212 "`%s` is a unknown callable type, please make sure no positional parameters in function/method signature." % (func), UserWarning)
213 else:
214 raise TypeError('`record_custom_attrs` must be `dict` type.')
215 else:
216 return
217
218 def __init__(self, fmt=BASIC_FORMAT, datefmt=None, style='%', record_custom_attrs=None, mix_extra=False, mix_extra_position='tail', skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, encoding='utf-8', default=None, sort_keys=False, **kw):
219 """
220 If ``style`` not in ``['%', '{', '$']``, a ``ValueError`` will be raised.
221
222 If ``record_custom_attrs`` is not ``None``, it must be a ``dict``
223 type, the key of dict will be setted as ``LogRecord``'s attribute, the
224 value of key must be a callable object and no positional
225 parameters(e.g. ``lambda: None``, ``lambda **log_reocrd_attrs: None``), it
226 returned will be setted as attribute's value of ``LogRecord``.
227
228 If ``mix_extra`` is ``True``, the keys of ``extra`` different with ``fmt``
229 will be added to log, the keys' value of ``extra`` same with ``fmt``
230 will overwrite ``fmt``.
231
232 If ``mix_extra_position`` not in ``['head', 'tail' or 'mix']``, a
233 ``ValueError`` will be raised.
234
235 If ``skipkeys`` is true then ``dict`` keys that are not basic types
236 (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped
237 instead of raising a ``TypeError``.
238
239 If ``ensure_ascii`` is false, then the return value can contain non-ASCII
240 characters if they appear in strings contained in ``obj``. Otherwise, all
241 such characters are escaped in JSON strings.
242
243 If ``check_circular`` is false, then the circular reference check
244 for container types will be skipped and a circular reference will
245 result in an ``OverflowError`` (or worse).
246
247 If ``allow_nan`` is false, then it will be a ``ValueError`` to
248 serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
249 strict compliance of the JSON specification, instead of using the
250 JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
251
252 If ``indent`` is a non-negative integer, then JSON array elements and
253 object members will be pretty-printed with that indent level. An indent
254 level of 0 will only insert newlines. ``None`` is the most compact
255 representation.
256
257 If specified, ``separators`` should be an ``(item_separator, key_separator)``
258 tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
259 ``(',', ': ')`` otherwise. To get the most compact JSON representation,
260 you should specify ``(',', ':')`` to eliminate whitespace.
261
262 ``encoding`` is the character encoding for str instances, only
263 supported by python 2.7, default is UTF-8.
264
265 ``default(obj)`` is a function that should return a serializable version
266 of obj or raise TypeError. The default simply raises TypeError.
267
268 If *sort_keys* is true (default: ``False``), then the output of
269 dictionaries will be sorted by key.
270
271 To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
272 ``.default()`` method to serialize additional types), specify it with
273 the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
274
275 """
276 if style not in _STYLES:
277 raise ValueError('`style` must be one of: %s' % ','.join(
278 _STYLES.keys()))
279 if mix_extra_position not in _MIX_EXTRA_ORDER:
280 raise ValueError('`mix_extra_position` must be one of: %s' % ','.join(
281 _MIX_EXTRA_ORDER))
282 # compatible python2 start
283 if sys.version_info < (3, 0):
284 kw.update(encoding=encoding)
285 logging.Formatter.__init__(
286 self, fmt='', datefmt=datefmt)
287 else:
288 logging.Formatter.__init__(
289 self, fmt='', datefmt=datefmt, style=style)
290 # compatible python2 end
291
292 self.json_fmt = self.parseFmt(fmt)
293 self.record_custom_attrs = record_custom_attrs
294 self._style = _STYLES[style](self.json_fmt)
295 self._style._fmt = ''
296 self.mix_extra = mix_extra
297 self.mix_extra_position = mix_extra_position
298
299 self.checkRecordCustomAttrs(self.record_custom_attrs)
300
301 # support `json.dumps` parameters start
302 self.skipkeys = skipkeys
303 self.ensure_ascii = ensure_ascii
304 self.check_circular = check_circular
305 self.allow_nan = allow_nan
306 self.cls = cls
307 self.indent = indent
308 self.separators = separators
309 self.encoding = encoding
310 self.default = default
311 self.sort_keys = sort_keys
312 self.kw = kw
313 # support `json.dumps` parameters end
314
315 def setRecordMessage(self, record):
316 if isinstance(record.msg, (int, long, float, bool, type(None))):
317 # keep these types without quote when output
318 record.message = record.msg
319 else:
320 record.message = str(record.msg)
321
322 if record.args:
323 record.message = record.getMessage()
324
325 if record.exc_info:
326 # Cache the traceback text to avoid converting it multiple times
327 # (it's constant anyway)
328 if not record.exc_text:
329 record.exc_text = self.formatException(record.exc_info)
330 if record.exc_text:
331 record.message = str(record.message)
332 if record.message[-1:] != "\n":
333 record.message = record.message + "\n"
334 record.message = record.message + record.exc_text
335 # compatible python2, record no stack_info attribute in python2, start
336 if getattr(record, 'stack_info', None):
337 record.message = str(record.message)
338 if record.message[-1:] != "\n":
339 record.message = record.message + "\n"
340 record.message = record.message + \
341 self.formatStack(record.stack_info)
342 # compatible python2, record no stack_info attribute in python2, end
343
344 def getRecordExtraAttrs(self, record):
345 extras = {
346 k: record.__dict__[k]
347 for k in record.__dict__
348 if k not in _LogRecordDefaultAttributes
349 }
350 if sys.version_info >= (3, 7):
351 return extras
352 else:
353 return dictionary((k, extras[k]) for k in sorted(extras.keys()))
354
355 def setRecordCustomAttrs(self, record):
356 for k, v in self.record_custom_attrs.items():
357 setattr(record, k, v(**record.__dict__))
358
359 def formatMessage(self, record):
360 return self._style.format(record)
361
362 def format(self, record):
363 def _set_extra_to_result():
364 for k, v in extra.items():
365 if k not in self.json_fmt:
366 result[k] = v
367
368 def _set_fmt_to_result():
369 # this is for keeping `record` attribute `type`
370 if v in record.__dict__:
371 result[k] = getattr(record, v, None)
372 # this is for convert to string
373 else:
374 self._style._fmt = v
375 result[k] = self.formatMessage(record)
376
377 result = dictionary()
378
379 self.setRecordMessage(record)
380
381 record.asctime = self.formatTime(record, self.datefmt)
382
383 # pop stored __extra start
384 extra = record.__dict__.pop('__extra', None) or record.__dict__.pop('_JsonFormatter__extra', None)
385 if extra is None:
386 # extra is dictionary
387 extra = self.getRecordExtraAttrs(record)
388 # pop stored __extra end
389
390 if self.record_custom_attrs:
391 self.setRecordCustomAttrs(record)
392
393 # compatible python2 start
394 if sys.version_info < (3, 0):
395 for k, v in record.__dict__.items():
396 if isinstance(v, str):
397 record.__dict__.update({k: v.decode(self.encoding)})
398 # compatible python2 end
399
400 if not self.mix_extra:
401 for k, v in self.json_fmt.items():
402 _set_fmt_to_result()
403 else:
404 if self.mix_extra_position == 'head':
405 _set_extra_to_result()
406 for k, v in self.json_fmt.items():
407 if k in extra:
408 result[k] = extra[k]
409 else:
410 _set_fmt_to_result()
411 if self.mix_extra_position == 'tail':
412 _set_extra_to_result()
413 if self.mix_extra_position == 'mix':
414 _set_extra_to_result()
415 result = dictionary(
416 (k, result[k])
417 for k in sorted(result.keys())
418 )
419
420 self._style._fmt = ''
421
422 # store __extra start
423 record.__extra = extra
424 # store __extra end
425
426 return json.dumps(
427 result,
428 skipkeys=self.skipkeys,
429 ensure_ascii=self.ensure_ascii,
430 check_circular=self.check_circular,
431 allow_nan=self.allow_nan,
432 cls=self.cls,
433 indent=self.indent,
434 separators=self.separators,
435 default=self.default,
436 sort_keys=self.sort_keys,
437 **self.kw
438 )
439
440
441def basicConfig(**kwargs):
442 """
443 Do basic configuration for the logging system.
444
445 This function does nothing if the root logger already has handlers
446 configured. It is a convenience method intended for use by simple scripts
447 to do one-shot configuration of the logging package.
448
449 The default behaviour is to create a StreamHandler which writes to
450 sys.stderr, set a formatter using the BASIC_FORMAT format string, and
451 add the handler to the root logger.
452
453 A number of optional keyword arguments may be specified, which can alter
454 the default behaviour.
455
456 filename Specifies that a FileHandler be created, using the specified
457 filename, rather than a StreamHandler.
458 filemode Specifies the mode to open the file, if filename is specified
459 (if filemode is unspecified, it defaults to 'a').
460 format Use the specified format string for the handler.
461 datefmt Use the specified date/time format.
462 style If a format string is specified, use this to specify the
463 type of format string (possible values '%', '{', '$', for
464 %-formatting, :meth:`str.format` and :class:`string.Template`
465 - defaults to '%').
466 level Set the root logger level to the specified level.
467 stream Use the specified stream to initialize the StreamHandler. Note
468 that this argument is incompatible with 'filename' - if both
469 are present, 'stream' is ignored.
470 handlers If specified, this should be an iterable of already created
471 handlers, which will be added to the root handler. Any handler
472 in the list which does not have a formatter assigned will be
473 assigned the formatter created in this function.
474
475 Note that you could specify a stream created using open(filename, mode)
476 rather than passing the filename and mode in. However, it should be
477 remembered that StreamHandler does not close its stream (since it may be
478 using sys.stdout or sys.stderr), whereas FileHandler closes its stream
479 when the handler is closed.
480
481 .. versionchanged:: 3.2
482 Added the ``style`` parameter.
483
484 .. versionchanged:: 3.3
485 Added the ``handlers`` parameter. A ``ValueError`` is now thrown for
486 incompatible arguments (e.g. ``handlers`` specified together with
487 ``filename``/``filemode``, or ``filename``/``filemode`` specified
488 together with ``stream``, or ``handlers`` specified together with
489 ``stream``.
490 """
491 # Add thread safety in case someone mistakenly calls
492 # basicConfig() from multiple threads
493 logging._acquireLock()
494 try:
495 if len(logging.root.handlers) == 0:
496 handlers = kwargs.pop("handlers", None)
497 if handlers is None:
498 if "stream" in kwargs and "filename" in kwargs:
499 raise ValueError("'stream' and 'filename' should not be "
500 "specified together")
501 else:
502 if "stream" in kwargs or "filename" in kwargs:
503 raise ValueError("'stream' or 'filename' should not be "
504 "specified together with 'handlers'")
505 if handlers is None:
506 filename = kwargs.pop("filename", None)
507 mode = kwargs.pop("filemode", 'a')
508 if filename:
509 h = logging.FileHandler(filename, mode)
510 else:
511 stream = kwargs.pop("stream", None)
512 h = logging.StreamHandler(stream)
513 handlers = [h]
514 dfs = kwargs.pop("datefmt", None)
515 style = kwargs.pop("style", '%')
516 if style not in _STYLES:
517 raise ValueError('Style must be one of: %s' % ','.join(
518 _STYLES.keys()))
519 fs = kwargs.pop("format", BASIC_FORMAT)
520 fmt = JsonFormatter(fs, dfs, style)
521 for h in handlers:
522 if h.formatter is None:
523 h.setFormatter(fmt)
524 logging.root.addHandler(h)
525 level = kwargs.pop("level", None)
526 if level is not None:
527 logging.root.setLevel(level)
528 if kwargs:
529 keys = ', '.join(kwargs.keys())
530 raise ValueError('Unrecognised argument(s): %s' % keys)
531 finally:
532 logging._releaseLock()
533