codekingpro/portable-devtools
114k
1#
2# Copyright 2009 Facebook
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""A command line parsing module that lets modules define their own options.
17
18This module is inspired by Google's `gflags
19<https://github.com/google/python-gflags>`_. The primary difference
20with libraries such as `argparse` is that a global registry is used so
21that options may be defined in any module (it also enables
22`tornado.log` by default). The rest of Tornado does not depend on this
23module, so feel free to use `argparse` or other configuration
24libraries if you prefer them.
25
26Options must be defined with `tornado.options.define` before use,
27generally at the top level of a module. The options are then
28accessible as attributes of `tornado.options.options`::
29
30 # myapp/db.py
31 from tornado.options import define, options
32
33 define("mysql_host", default="127.0.0.1:3306", help="Main user DB")
34 define("memcache_hosts", default="127.0.0.1:11011", multiple=True,
35 help="Main user memcache servers")
36
37 def connect():
38 db = database.Connection(options.mysql_host)
39 ...
40
41 # myapp/server.py
42 from tornado.options import define, options
43
44 define("port", default=8080, help="port to listen on")
45
46 def start_server():
47 app = make_app()
48 app.listen(options.port)
49
50The ``main()`` method of your application does not need to be aware of all of
51the options used throughout your program; they are all automatically loaded
52when the modules are loaded. However, all modules that define options
53must have been imported before the command line is parsed.
54
55Your ``main()`` method can parse the command line or parse a config file with
56either `parse_command_line` or `parse_config_file`::
57
58 import myapp.db, myapp.server
59 import tornado
60
61 if __name__ == '__main__':
62 tornado.options.parse_command_line()
63 # or
64 tornado.options.parse_config_file("/etc/server.conf")
65
66.. note::
67
68 When using multiple ``parse_*`` functions, pass ``final=False`` to all
69 but the last one, or side effects may occur twice (in particular,
70 this can result in log messages being doubled).
71
72`tornado.options.options` is a singleton instance of `OptionParser`, and
73the top-level functions in this module (`define`, `parse_command_line`, etc)
74simply call methods on it. You may create additional `OptionParser`
75instances to define isolated sets of options, such as for subcommands.
76
77.. note::
78
79 By default, several options are defined that will configure the
80 standard `logging` module when `parse_command_line` or `parse_config_file`
81 are called. If you want Tornado to leave the logging configuration
82 alone so you can manage it yourself, either pass ``--logging=none``
83 on the command line or do the following to disable it in code::
84
85 from tornado.options import options, parse_command_line
86 options.logging = None
87 parse_command_line()
88
89.. note::
90
91 `parse_command_line` or `parse_config_file` function should called after
92 logging configuration and user-defined command line flags using the
93 ``callback`` option definition, or these configurations will not take effect.
94
95.. versionchanged:: 4.3
96 Dashes and underscores are fully interchangeable in option names;
97 options can be defined, set, and read with any mix of the two.
98 Dashes are typical for command-line usage while config files require
99 underscores.
100"""
101
102import datetime
103import numbers
104import re
105import sys
106import os
107import textwrap
108
109from tornado.escape import _unicode, native_str
110from tornado.log import define_logging_options
111from tornado.util import basestring_type, exec_in
112
113from typing import (
114 Any,
115 Iterator,
116 Iterable,
117 Tuple,
118 Set,
119 Dict,
120 Callable,
121 List,
122 TextIO,
123 Optional,
124)
125
126
127class Error(Exception):
128 """Exception raised by errors in the options module."""
129
130 pass
131
132
133class OptionParser:
134 """A collection of options, a dictionary with object-like access.
135
136 Normally accessed via static functions in the `tornado.options` module,
137 which reference a global instance.
138 """
139
140 def __init__(self) -> None:
141 # we have to use self.__dict__ because we override setattr.
142 self.__dict__["_options"] = {}
143 self.__dict__["_parse_callbacks"] = []
144 self.define(
145 "help",
146 type=bool,
147 help="show this help information",
148 callback=self._help_callback,
149 )
150
151 def _normalize_name(self, name: str) -> str:
152 return name.replace("_", "-")
153
154 def __getattr__(self, name: str) -> Any:
155 name = self._normalize_name(name)
156 if isinstance(self._options.get(name), _Option):
157 return self._options[name].value()
158 raise AttributeError("Unrecognized option %r" % name)
159
160 def __setattr__(self, name: str, value: Any) -> None:
161 name = self._normalize_name(name)
162 if isinstance(self._options.get(name), _Option):
163 return self._options[name].set(value)
164 raise AttributeError("Unrecognized option %r" % name)
165
166 def __iter__(self) -> Iterator:
167 return (opt.name for opt in self._options.values())
168
169 def __contains__(self, name: str) -> bool:
170 name = self._normalize_name(name)
171 return name in self._options
172
173 def __getitem__(self, name: str) -> Any:
174 return self.__getattr__(name)
175
176 def __setitem__(self, name: str, value: Any) -> None:
177 return self.__setattr__(name, value)
178
179 def items(self) -> Iterable[Tuple[str, Any]]:
180 """An iterable of (name, value) pairs.
181
182 .. versionadded:: 3.1
183 """
184 return [(opt.name, opt.value()) for name, opt in self._options.items()]
185
186 def groups(self) -> Set[str]:
187 """The set of option-groups created by ``define``.
188
189 .. versionadded:: 3.1
190 """
191 return {opt.group_name for opt in self._options.values()}
192
193 def group_dict(self, group: str) -> Dict[str, Any]:
194 """The names and values of options in a group.
195
196 Useful for copying options into Application settings::
197
198 from tornado.options import define, parse_command_line, options
199
200 define('template_path', group='application')
201 define('static_path', group='application')
202
203 parse_command_line()
204
205 application = Application(
206 handlers, **options.group_dict('application'))
207
208 .. versionadded:: 3.1
209 """
210 return {
211 opt.name: opt.value()
212 for name, opt in self._options.items()
213 if not group or group == opt.group_name
214 }
215
216 def as_dict(self) -> Dict[str, Any]:
217 """The names and values of all options.
218
219 .. versionadded:: 3.1
220 """
221 return {opt.name: opt.value() for name, opt in self._options.items()}
222
223 def define(
224 self,
225 name: str,
226 default: Any = None,
227 type: Optional[type] = None,
228 help: Optional[str] = None,
229 metavar: Optional[str] = None,
230 multiple: bool = False,
231 group: Optional[str] = None,
232 callback: Optional[Callable[[Any], None]] = None,
233 ) -> None:
234 """Defines a new command line option.
235
236 ``type`` can be any of `str`, `int`, `float`, `bool`,
237 `~datetime.datetime`, or `~datetime.timedelta`. If no ``type``
238 is given but a ``default`` is, ``type`` is the type of
239 ``default``. Otherwise, ``type`` defaults to `str`.
240
241 If ``multiple`` is True, the option value is a list of ``type``
242 instead of an instance of ``type``.
243
244 ``help`` and ``metavar`` are used to construct the
245 automatically generated command line help string. The help
246 message is formatted like::
247
248 --name=METAVAR help string
249
250 ``group`` is used to group the defined options in logical
251 groups. By default, command line options are grouped by the
252 file in which they are defined.
253
254 Command line option names must be unique globally.
255
256 If a ``callback`` is given, it will be run with the new value whenever
257 the option is changed. This can be used to combine command-line
258 and file-based options::
259
260 define("config", type=str, help="path to config file",
261 callback=lambda path: parse_config_file(path, final=False))
262
263 With this definition, options in the file specified by ``--config`` will
264 override options set earlier on the command line, but can be overridden
265 by later flags.
266
267 """
268 normalized = self._normalize_name(name)
269 if normalized in self._options:
270 raise Error(
271 "Option %r already defined in %s"
272 % (normalized, self._options[normalized].file_name)
273 )
274 frame = sys._getframe(0)
275 if frame is not None:
276 options_file = frame.f_code.co_filename
277
278 # Can be called directly, or through top level define() fn, in which
279 # case, step up above that frame to look for real caller.
280 if (
281 frame.f_back is not None
282 and frame.f_back.f_code.co_filename == options_file
283 and frame.f_back.f_code.co_name == "define"
284 ):
285 frame = frame.f_back
286
287 assert frame.f_back is not None
288 file_name = frame.f_back.f_code.co_filename
289 else:
290 file_name = "<unknown>"
291 if file_name == options_file:
292 file_name = ""
293 if type is None:
294 if not multiple and default is not None:
295 type = default.__class__
296 else:
297 type = str
298 if group:
299 group_name = group # type: Optional[str]
300 else:
301 group_name = file_name
302 option = _Option(
303 name,
304 file_name=file_name,
305 default=default,
306 type=type,
307 help=help,
308 metavar=metavar,
309 multiple=multiple,
310 group_name=group_name,
311 callback=callback,
312 )
313 self._options[normalized] = option
314
315 def parse_command_line(
316 self, args: Optional[List[str]] = None, final: bool = True
317 ) -> List[str]:
318 """Parses all options given on the command line (defaults to
319 `sys.argv`).
320
321 Options look like ``--option=value`` and are parsed according
322 to their ``type``. For boolean options, ``--option`` is
323 equivalent to ``--option=true``
324
325 If the option has ``multiple=True``, comma-separated values
326 are accepted. For multi-value integer options, the syntax
327 ``x:y`` is also accepted and equivalent to ``range(x, y)``.
328
329 Note that ``args[0]`` is ignored since it is the program name
330 in `sys.argv`.
331
332 We return a list of all arguments that are not parsed as options.
333
334 If ``final`` is ``False``, parse callbacks will not be run.
335 This is useful for applications that wish to combine configurations
336 from multiple sources.
337
338 """
339 if args is None:
340 args = sys.argv
341 remaining = [] # type: List[str]
342 for i in range(1, len(args)):
343 # All things after the last option are command line arguments
344 if not args[i].startswith("-"):
345 remaining = args[i:]
346 break
347 if args[i] == "--":
348 remaining = args[i + 1 :]
349 break
350 arg = args[i].lstrip("-")
351 name, equals, value = arg.partition("=")
352 name = self._normalize_name(name)
353 if name not in self._options:
354 self.print_help()
355 raise Error("Unrecognized command line option: %r" % name)
356 option = self._options[name]
357 if not equals:
358 if option.type == bool:
359 value = "true"
360 else:
361 raise Error("Option %r requires a value" % name)
362 option.parse(value)
363
364 if final:
365 self.run_parse_callbacks()
366
367 return remaining
368
369 def parse_config_file(self, path: str, final: bool = True) -> None:
370 """Parses and loads the config file at the given path.
371
372 The config file contains Python code that will be executed (so
373 it is **not safe** to use untrusted config files). Anything in
374 the global namespace that matches a defined option will be
375 used to set that option's value.
376
377 Options may either be the specified type for the option or
378 strings (in which case they will be parsed the same way as in
379 `.parse_command_line`)
380
381 Example (using the options defined in the top-level docs of
382 this module)::
383
384 port = 80
385 mysql_host = 'mydb.example.com:3306'
386 # Both lists and comma-separated strings are allowed for
387 # multiple=True.
388 memcache_hosts = ['cache1.example.com:11011',
389 'cache2.example.com:11011']
390 memcache_hosts = 'cache1.example.com:11011,cache2.example.com:11011'
391
392 If ``final`` is ``False``, parse callbacks will not be run.
393 This is useful for applications that wish to combine configurations
394 from multiple sources.
395
396 .. note::
397
398 `tornado.options` is primarily a command-line library.
399 Config file support is provided for applications that wish
400 to use it, but applications that prefer config files may
401 wish to look at other libraries instead.
402
403 .. versionchanged:: 4.1
404 Config files are now always interpreted as utf-8 instead of
405 the system default encoding.
406
407 .. versionchanged:: 4.4
408 The special variable ``__file__`` is available inside config
409 files, specifying the absolute path to the config file itself.
410
411 .. versionchanged:: 5.1
412 Added the ability to set options via strings in config files.
413
414 """
415 config = {"__file__": os.path.abspath(path)}
416 with open(path, "rb") as f:
417 exec_in(native_str(f.read()), config, config)
418 for name in config:
419 normalized = self._normalize_name(name)
420 if normalized in self._options:
421 option = self._options[normalized]
422 if option.multiple:
423 if not isinstance(config[name], (list, str)):
424 raise Error(
425 "Option %r is required to be a list of %s "
426 "or a comma-separated string"
427 % (option.name, option.type.__name__)
428 )
429
430 if type(config[name]) is str and (
431 option.type is not str or option.multiple
432 ):
433 option.parse(config[name])
434 else:
435 option.set(config[name])
436
437 if final:
438 self.run_parse_callbacks()
439
440 def print_help(self, file: Optional[TextIO] = None) -> None:
441 """Prints all the command line options to stderr (or another file)."""
442 if file is None:
443 file = sys.stderr
444 print("Usage: %s [OPTIONS]" % sys.argv[0], file=file)
445 print("\nOptions:\n", file=file)
446 by_group = {} # type: Dict[str, List[_Option]]
447 for option in self._options.values():
448 by_group.setdefault(option.group_name, []).append(option)
449
450 for filename, o in sorted(by_group.items()):
451 if filename:
452 print("\n%s options:\n" % os.path.normpath(filename), file=file)
453 o.sort(key=lambda option: option.name)
454 for option in o:
455 # Always print names with dashes in a CLI context.
456 prefix = self._normalize_name(option.name)
457 if option.metavar:
458 prefix += "=" + option.metavar
459 description = option.help or ""
460 if option.default is not None and option.default != "":
461 description += " (default %s)" % option.default
462 lines = textwrap.wrap(description, 79 - 35)
463 if len(prefix) > 30 or len(lines) == 0:
464 lines.insert(0, "")
465 print(" --%-30s %s" % (prefix, lines[0]), file=file)
466 for line in lines[1:]:
467 print("%-34s %s" % (" ", line), file=file)
468 print(file=file)
469
470 def _help_callback(self, value: bool) -> None:
471 if value:
472 self.print_help()
473 sys.exit(0)
474
475 def add_parse_callback(self, callback: Callable[[], None]) -> None:
476 """Adds a parse callback, to be invoked when option parsing is done."""
477 self._parse_callbacks.append(callback)
478
479 def run_parse_callbacks(self) -> None:
480 for callback in self._parse_callbacks:
481 callback()
482
483 def mockable(self) -> "_Mockable":
484 """Returns a wrapper around self that is compatible with
485 `unittest.mock.patch`.
486
487 The `unittest.mock.patch` function is incompatible with objects like ``options`` that
488 override ``__getattr__`` and ``__setattr__``. This function returns an object that can be
489 used with `mock.patch.object <unittest.mock.patch.object>` to modify option values::
490
491 with mock.patch.object(options.mockable(), 'name', value):
492 assert options.name == value
493 """
494 return _Mockable(self)
495
496
497class _Mockable:
498 """`mock.patch` compatible wrapper for `OptionParser`.
499
500 As of ``mock`` version 1.0.1, when an object uses ``__getattr__``
501 hooks instead of ``__dict__``, ``patch.__exit__`` tries to delete
502 the attribute it set instead of setting a new one (assuming that
503 the object does not capture ``__setattr__``, so the patch
504 created a new attribute in ``__dict__``).
505
506 _Mockable's getattr and setattr pass through to the underlying
507 OptionParser, and delattr undoes the effect of a previous setattr.
508 """
509
510 def __init__(self, options: OptionParser) -> None:
511 # Modify __dict__ directly to bypass __setattr__
512 self.__dict__["_options"] = options
513 self.__dict__["_originals"] = {}
514
515 def __getattr__(self, name: str) -> Any:
516 return getattr(self._options, name)
517
518 def __setattr__(self, name: str, value: Any) -> None:
519 assert name not in self._originals, "don't reuse mockable objects"
520 self._originals[name] = getattr(self._options, name)
521 setattr(self._options, name, value)
522
523 def __delattr__(self, name: str) -> None:
524 setattr(self._options, name, self._originals.pop(name))
525
526
527class _Option:
528 # This class could almost be made generic, but the way the types
529 # interact with the multiple argument makes this tricky. (default
530 # and the callback use List[T], but type is still Type[T]).
531 UNSET = object()
532
533 def __init__(
534 self,
535 name: str,
536 default: Any = None,
537 type: Optional[type] = None,
538 help: Optional[str] = None,
539 metavar: Optional[str] = None,
540 multiple: bool = False,
541 file_name: Optional[str] = None,
542 group_name: Optional[str] = None,
543 callback: Optional[Callable[[Any], None]] = None,
544 ) -> None:
545 if default is None and multiple:
546 default = []
547 self.name = name
548 if type is None:
549 raise ValueError("type must not be None")
550 self.type = type
551 self.help = help
552 self.metavar = metavar
553 self.multiple = multiple
554 self.file_name = file_name
555 self.group_name = group_name
556 self.callback = callback
557 self.default = default
558 self._value = _Option.UNSET # type: Any
559
560 def value(self) -> Any:
561 return self.default if self._value is _Option.UNSET else self._value
562
563 def parse(self, value: str) -> Any:
564 _parse = {
565 datetime.datetime: self._parse_datetime,
566 datetime.timedelta: self._parse_timedelta,
567 bool: self._parse_bool,
568 basestring_type: self._parse_string,
569 }.get(
570 self.type, self.type
571 ) # type: Callable[[str], Any]
572 if self.multiple:
573 self._value = []
574 for part in value.split(","):
575 if issubclass(self.type, numbers.Integral):
576 # allow ranges of the form X:Y (inclusive at both ends)
577 lo_str, _, hi_str = part.partition(":")
578 lo = _parse(lo_str)
579 hi = _parse(hi_str) if hi_str else lo
580 self._value.extend(range(lo, hi + 1))
581 else:
582 self._value.append(_parse(part))
583 else:
584 self._value = _parse(value)
585 if self.callback is not None:
586 self.callback(self._value)
587 return self.value()
588
589 def set(self, value: Any) -> None:
590 if self.multiple:
591 if not isinstance(value, list):
592 raise Error(
593 "Option %r is required to be a list of %s"
594 % (self.name, self.type.__name__)
595 )
596 for item in value:
597 if item is not None and not isinstance(item, self.type):
598 raise Error(
599 "Option %r is required to be a list of %s"
600 % (self.name, self.type.__name__)
601 )
602 else:
603 if value is not None and not isinstance(value, self.type):
604 raise Error(
605 "Option %r is required to be a %s (%s given)"
606 % (self.name, self.type.__name__, type(value))
607 )
608 self._value = value
609 if self.callback is not None:
610 self.callback(self._value)
611
612 # Supported date/time formats in our options
613 _DATETIME_FORMATS = [
614 "%a %b %d %H:%M:%S %Y",
615 "%Y-%m-%d %H:%M:%S",
616 "%Y-%m-%d %H:%M",
617 "%Y-%m-%dT%H:%M",
618 "%Y%m%d %H:%M:%S",
619 "%Y%m%d %H:%M",
620 "%Y-%m-%d",
621 "%Y%m%d",
622 "%H:%M:%S",
623 "%H:%M",
624 ]
625
626 def _parse_datetime(self, value: str) -> datetime.datetime:
627 for format in self._DATETIME_FORMATS:
628 try:
629 return datetime.datetime.strptime(value, format)
630 except ValueError:
631 pass
632 raise Error("Unrecognized date/time format: %r" % value)
633
634 _TIMEDELTA_ABBREV_DICT = {
635 "h": "hours",
636 "m": "minutes",
637 "min": "minutes",
638 "s": "seconds",
639 "sec": "seconds",
640 "ms": "milliseconds",
641 "us": "microseconds",
642 "d": "days",
643 "w": "weeks",
644 }
645
646 _FLOAT_PATTERN = r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?"
647
648 _TIMEDELTA_PATTERN = re.compile(
649 r"\s*(%s)\s*(\w*)\s*" % _FLOAT_PATTERN, re.IGNORECASE
650 )
651
652 def _parse_timedelta(self, value: str) -> datetime.timedelta:
653 try:
654 sum = datetime.timedelta()
655 start = 0
656 while start < len(value):
657 m = self._TIMEDELTA_PATTERN.match(value, start)
658 if not m:
659 raise Exception()
660 num = float(m.group(1))
661 units = m.group(2) or "seconds"
662 units = self._TIMEDELTA_ABBREV_DICT.get(units, units)
663
664 sum += datetime.timedelta(**{units: num})
665 start = m.end()
666 return sum
667 except Exception:
668 raise
669
670 def _parse_bool(self, value: str) -> bool:
671 return value.lower() not in ("false", "0", "f")
672
673 def _parse_string(self, value: str) -> str:
674 return _unicode(value)
675
676
677options = OptionParser()
678"""Global options object.
679
680All defined options are available as attributes on this object.
681"""
682
683
684def define(
685 name: str,
686 default: Any = None,
687 type: Optional[type] = None,
688 help: Optional[str] = None,
689 metavar: Optional[str] = None,
690 multiple: bool = False,
691 group: Optional[str] = None,
692 callback: Optional[Callable[[Any], None]] = None,
693) -> None:
694 """Defines an option in the global namespace.
695
696 See `OptionParser.define`.
697 """
698 return options.define(
699 name,
700 default=default,
701 type=type,
702 help=help,
703 metavar=metavar,
704 multiple=multiple,
705 group=group,
706 callback=callback,
707 )
708
709
710def parse_command_line(
711 args: Optional[List[str]] = None, final: bool = True
712) -> List[str]:
713 """Parses global options from the command line.
714
715 See `OptionParser.parse_command_line`.
716 """
717 return options.parse_command_line(args, final=final)
718
719
720def parse_config_file(path: str, final: bool = True) -> None:
721 """Parses global options from a config file.
722
723 See `OptionParser.parse_config_file`.
724 """
725 return options.parse_config_file(path, final=final)
726
727
728def print_help(file: Optional[TextIO] = None) -> None:
729 """Prints all the command line options to stderr (or another file).
730
731 See `OptionParser.print_help`.
732 """
733 return options.print_help(file)
734
735
736def add_parse_callback(callback: Callable[[], None]) -> None:
737 """Adds a parse callback, to be invoked when option parsing is done.
738
739 See `OptionParser.add_parse_callback`
740 """
741 options.add_parse_callback(callback)
742
743
744# Default options
745define_logging_options(options)
746