codekingpro/portable-devtools
115k
1"""Blocking and non-blocking HTTP client interfaces.
2
3This module defines a common interface shared by two implementations,
4``simple_httpclient`` and ``curl_httpclient``. Applications may either
5instantiate their chosen implementation class directly or use the
6`AsyncHTTPClient` class from this module, which selects an implementation
7that can be overridden with the `AsyncHTTPClient.configure` method.
8
9The default implementation is ``simple_httpclient``, and this is expected
10to be suitable for most users' needs. However, some applications may wish
11to switch to ``curl_httpclient`` for reasons such as the following:
12
13* ``curl_httpclient`` has some features not found in ``simple_httpclient``,
14 including support for HTTP proxies and the ability to use a specified
15 network interface.
16
17* ``curl_httpclient`` is more likely to be compatible with sites that are
18 not-quite-compliant with the HTTP spec, or sites that use little-exercised
19 features of HTTP.
20
21* ``curl_httpclient`` is faster.
22
23Note that if you are using ``curl_httpclient``, it is highly
24recommended that you use a recent version of ``libcurl`` and
25``pycurl``. Currently the minimum supported version of libcurl is
267.22.0, and the minimum version of pycurl is 7.18.2. It is highly
27recommended that your ``libcurl`` installation is built with
28asynchronous DNS resolver (threaded or c-ares), otherwise you may
29encounter various problems with request timeouts (for more
30information, see
31http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS
32and comments in curl_httpclient.py).
33
34To select ``curl_httpclient``, call `AsyncHTTPClient.configure` at startup::
35
36 AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
37"""
38
39import datetime
40import functools
41from io import BytesIO
42import ssl
43import time
44import weakref
45
46from tornado.concurrent import (
47 Future,
48 future_set_result_unless_cancelled,
49 future_set_exception_unless_cancelled,
50)
51from tornado.escape import utf8, native_str
52from tornado import gen, httputil
53from tornado.ioloop import IOLoop
54from tornado.util import Configurable
55
56from typing import Type, Any, Union, Dict, Callable, Optional, cast
57
58
59class HTTPClient:
60 """A blocking HTTP client.
61
62 This interface is provided to make it easier to share code between
63 synchronous and asynchronous applications. Applications that are
64 running an `.IOLoop` must use `AsyncHTTPClient` instead.
65
66 Typical usage looks like this::
67
68 http_client = httpclient.HTTPClient()
69 try:
70 response = http_client.fetch("http://www.google.com/")
71 print(response.body)
72 except httpclient.HTTPError as e:
73 # HTTPError is raised for non-200 responses; the response
74 # can be found in e.response.
75 print("Error: " + str(e))
76 except Exception as e:
77 # Other errors are possible, such as IOError.
78 print("Error: " + str(e))
79 http_client.close()
80
81 .. versionchanged:: 5.0
82
83 Due to limitations in `asyncio`, it is no longer possible to
84 use the synchronous ``HTTPClient`` while an `.IOLoop` is running.
85 Use `AsyncHTTPClient` instead.
86
87 """
88
89 def __init__(
90 self,
91 async_client_class: "Optional[Type[AsyncHTTPClient]]" = None,
92 **kwargs: Any,
93 ) -> None:
94 # Initialize self._closed at the beginning of the constructor
95 # so that an exception raised here doesn't lead to confusing
96 # failures in __del__.
97 self._closed = True
98 self._io_loop = IOLoop(make_current=False)
99 if async_client_class is None:
100 async_client_class = AsyncHTTPClient
101
102 # Create the client while our IOLoop is "current", without
103 # clobbering the thread's real current IOLoop (if any).
104 async def make_client() -> "AsyncHTTPClient":
105 await gen.sleep(0)
106 assert async_client_class is not None
107 return async_client_class(**kwargs)
108
109 self._async_client = self._io_loop.run_sync(make_client)
110 self._closed = False
111
112 def __del__(self) -> None:
113 self.close()
114
115 def close(self) -> None:
116 """Closes the HTTPClient, freeing any resources used."""
117 if not self._closed:
118 self._async_client.close()
119 self._io_loop.close()
120 self._closed = True
121
122 def fetch(
123 self, request: Union["HTTPRequest", str], **kwargs: Any
124 ) -> "HTTPResponse":
125 """Executes a request, returning an `HTTPResponse`.
126
127 The request may be either a string URL or an `HTTPRequest` object.
128 If it is a string, we construct an `HTTPRequest` using any additional
129 kwargs: ``HTTPRequest(request, **kwargs)``
130
131 If an error occurs during the fetch, we raise an `HTTPError` unless
132 the ``raise_error`` keyword argument is set to False.
133 """
134 response = self._io_loop.run_sync(
135 functools.partial(self._async_client.fetch, request, **kwargs)
136 )
137 return response
138
139
140class AsyncHTTPClient(Configurable):
141 """An non-blocking HTTP client.
142
143 Example usage::
144
145 async def f():
146 http_client = AsyncHTTPClient()
147 try:
148 response = await http_client.fetch("http://www.google.com")
149 except Exception as e:
150 print("Error: %s" % e)
151 else:
152 print(response.body)
153
154 The constructor for this class is magic in several respects: It
155 actually creates an instance of an implementation-specific
156 subclass, and instances are reused as a kind of pseudo-singleton
157 (one per `.IOLoop`). The keyword argument ``force_instance=True``
158 can be used to suppress this singleton behavior. Unless
159 ``force_instance=True`` is used, no arguments should be passed to
160 the `AsyncHTTPClient` constructor. The implementation subclass as
161 well as arguments to its constructor can be set with the static
162 method `configure()`
163
164 All `AsyncHTTPClient` implementations support a ``defaults``
165 keyword argument, which can be used to set default values for
166 `HTTPRequest` attributes. For example::
167
168 AsyncHTTPClient.configure(
169 None, defaults=dict(user_agent="MyUserAgent"))
170 # or with force_instance:
171 client = AsyncHTTPClient(force_instance=True,
172 defaults=dict(user_agent="MyUserAgent"))
173
174 .. versionchanged:: 5.0
175 The ``io_loop`` argument (deprecated since version 4.1) has been removed.
176
177 """
178
179 _instance_cache = None # type: Dict[IOLoop, AsyncHTTPClient]
180
181 @classmethod
182 def configurable_base(cls) -> Type[Configurable]:
183 return AsyncHTTPClient
184
185 @classmethod
186 def configurable_default(cls) -> Type[Configurable]:
187 from tornado.simple_httpclient import SimpleAsyncHTTPClient
188
189 return SimpleAsyncHTTPClient
190
191 @classmethod
192 def _async_clients(cls) -> Dict[IOLoop, "AsyncHTTPClient"]:
193 attr_name = "_async_client_dict_" + cls.__name__
194 if not hasattr(cls, attr_name):
195 setattr(cls, attr_name, weakref.WeakKeyDictionary())
196 return getattr(cls, attr_name)
197
198 def __new__(cls, force_instance: bool = False, **kwargs: Any) -> "AsyncHTTPClient":
199 io_loop = IOLoop.current()
200 if force_instance:
201 instance_cache = None
202 else:
203 instance_cache = cls._async_clients()
204 if instance_cache is not None and io_loop in instance_cache:
205 return instance_cache[io_loop]
206 instance = super().__new__(cls, **kwargs) # type: ignore
207 # Make sure the instance knows which cache to remove itself from.
208 # It can't simply call _async_clients() because we may be in
209 # __new__(AsyncHTTPClient) but instance.__class__ may be
210 # SimpleAsyncHTTPClient.
211 instance._instance_cache = instance_cache
212 if instance_cache is not None:
213 instance_cache[instance.io_loop] = instance
214 return instance
215
216 def initialize(self, defaults: Optional[Dict[str, Any]] = None) -> None:
217 self.io_loop = IOLoop.current()
218 self.defaults = dict(HTTPRequest._DEFAULTS)
219 if defaults is not None:
220 self.defaults.update(defaults)
221 self._closed = False
222
223 def close(self) -> None:
224 """Destroys this HTTP client, freeing any file descriptors used.
225
226 This method is **not needed in normal use** due to the way
227 that `AsyncHTTPClient` objects are transparently reused.
228 ``close()`` is generally only necessary when either the
229 `.IOLoop` is also being closed, or the ``force_instance=True``
230 argument was used when creating the `AsyncHTTPClient`.
231
232 No other methods may be called on the `AsyncHTTPClient` after
233 ``close()``.
234
235 """
236 if self._closed:
237 return
238 self._closed = True
239 if self._instance_cache is not None:
240 cached_val = self._instance_cache.pop(self.io_loop, None)
241 # If there's an object other than self in the instance
242 # cache for our IOLoop, something has gotten mixed up. A
243 # value of None appears to be possible when this is called
244 # from a destructor (HTTPClient.__del__) as the weakref
245 # gets cleared before the destructor runs.
246 if cached_val is not None and cached_val is not self:
247 raise RuntimeError("inconsistent AsyncHTTPClient cache")
248
249 def fetch(
250 self,
251 request: Union[str, "HTTPRequest"],
252 raise_error: bool = True,
253 **kwargs: Any,
254 ) -> "Future[HTTPResponse]":
255 """Executes a request, asynchronously returning an `HTTPResponse`.
256
257 The request may be either a string URL or an `HTTPRequest` object.
258 If it is a string, we construct an `HTTPRequest` using any additional
259 kwargs: ``HTTPRequest(request, **kwargs)``
260
261 This method returns a `.Future` whose result is an
262 `HTTPResponse`. By default, the ``Future`` will raise an
263 `HTTPError` if the request returned a non-200 response code
264 (other errors may also be raised if the server could not be
265 contacted). Instead, if ``raise_error`` is set to False, the
266 response will always be returned regardless of the response
267 code.
268
269 If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
270 In the callback interface, `HTTPError` is not automatically raised.
271 Instead, you must check the response's ``error`` attribute or
272 call its `~HTTPResponse.rethrow` method.
273
274 .. versionchanged:: 6.0
275
276 The ``callback`` argument was removed. Use the returned
277 `.Future` instead.
278
279 The ``raise_error=False`` argument only affects the
280 `HTTPError` raised when a non-200 response code is used,
281 instead of suppressing all errors.
282 """
283 if self._closed:
284 raise RuntimeError("fetch() called on closed AsyncHTTPClient")
285 if not isinstance(request, HTTPRequest):
286 request = HTTPRequest(url=request, **kwargs)
287 else:
288 if kwargs:
289 raise ValueError(
290 "kwargs can't be used if request is an HTTPRequest object"
291 )
292 # We may modify this (to add Host, Accept-Encoding, etc),
293 # so make sure we don't modify the caller's object. This is also
294 # where normal dicts get converted to HTTPHeaders objects.
295 request.headers = httputil.HTTPHeaders(request.headers)
296 request_proxy = _RequestProxy(request, self.defaults)
297 future = Future() # type: Future[HTTPResponse]
298
299 def handle_response(response: "HTTPResponse") -> None:
300 if response.error:
301 if raise_error or not response._error_is_response_code:
302 future_set_exception_unless_cancelled(future, response.error)
303 return
304 future_set_result_unless_cancelled(future, response)
305
306 self.fetch_impl(cast(HTTPRequest, request_proxy), handle_response)
307 return future
308
309 def fetch_impl(
310 self, request: "HTTPRequest", callback: Callable[["HTTPResponse"], None]
311 ) -> None:
312 raise NotImplementedError()
313
314 @classmethod
315 def configure(
316 cls, impl: "Union[None, str, Type[Configurable]]", **kwargs: Any
317 ) -> None:
318 """Configures the `AsyncHTTPClient` subclass to use.
319
320 ``AsyncHTTPClient()`` actually creates an instance of a subclass.
321 This method may be called with either a class object or the
322 fully-qualified name of such a class (or ``None`` to use the default,
323 ``SimpleAsyncHTTPClient``)
324
325 If additional keyword arguments are given, they will be passed
326 to the constructor of each subclass instance created. The
327 keyword argument ``max_clients`` determines the maximum number
328 of simultaneous `~AsyncHTTPClient.fetch()` operations that can
329 execute in parallel on each `.IOLoop`. Additional arguments
330 may be supported depending on the implementation class in use.
331
332 Example::
333
334 AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
335 """
336 super().configure(impl, **kwargs)
337
338
339class HTTPRequest:
340 """HTTP client request object."""
341
342 _headers = None # type: Union[Dict[str, str], httputil.HTTPHeaders]
343
344 # Default values for HTTPRequest parameters.
345 # Merged with the values on the request object by AsyncHTTPClient
346 # implementations.
347 _DEFAULTS = dict(
348 connect_timeout=20.0,
349 request_timeout=20.0,
350 follow_redirects=True,
351 max_redirects=5,
352 decompress_response=True,
353 proxy_password="",
354 allow_nonstandard_methods=False,
355 validate_cert=True,
356 )
357
358 def __init__(
359 self,
360 url: str,
361 method: str = "GET",
362 headers: Optional[Union[Dict[str, str], httputil.HTTPHeaders]] = None,
363 body: Optional[Union[bytes, str]] = None,
364 auth_username: Optional[str] = None,
365 auth_password: Optional[str] = None,
366 auth_mode: Optional[str] = None,
367 connect_timeout: Optional[float] = None,
368 request_timeout: Optional[float] = None,
369 if_modified_since: Optional[Union[float, datetime.datetime]] = None,
370 follow_redirects: Optional[bool] = None,
371 max_redirects: Optional[int] = None,
372 user_agent: Optional[str] = None,
373 use_gzip: Optional[bool] = None,
374 network_interface: Optional[str] = None,
375 streaming_callback: Optional[Callable[[bytes], None]] = None,
376 header_callback: Optional[Callable[[str], None]] = None,
377 prepare_curl_callback: Optional[Callable[[Any], None]] = None,
378 proxy_host: Optional[str] = None,
379 proxy_port: Optional[int] = None,
380 proxy_username: Optional[str] = None,
381 proxy_password: Optional[str] = None,
382 proxy_auth_mode: Optional[str] = None,
383 allow_nonstandard_methods: Optional[bool] = None,
384 validate_cert: Optional[bool] = None,
385 ca_certs: Optional[str] = None,
386 allow_ipv6: Optional[bool] = None,
387 client_key: Optional[str] = None,
388 client_cert: Optional[str] = None,
389 body_producer: Optional[
390 Callable[[Callable[[bytes], None]], "Future[None]"]
391 ] = None,
392 expect_100_continue: bool = False,
393 decompress_response: Optional[bool] = None,
394 ssl_options: Optional[Union[Dict[str, Any], ssl.SSLContext]] = None,
395 ) -> None:
396 r"""All parameters except ``url`` are optional.
397
398 :arg str url: URL to fetch
399 :arg str method: HTTP method, e.g. "GET" or "POST"
400 :arg headers: Additional HTTP headers to pass on the request
401 :type headers: `~tornado.httputil.HTTPHeaders` or `dict`
402 :arg body: HTTP request body as a string (byte or unicode; if unicode
403 the utf-8 encoding will be used)
404 :type body: `str` or `bytes`
405 :arg collections.abc.Callable body_producer: Callable used for
406 lazy/asynchronous request bodies.
407 It is called with one argument, a ``write`` function, and should
408 return a `.Future`. It should call the write function with new
409 data as it becomes available. The write function returns a
410 `.Future` which can be used for flow control.
411 Only one of ``body`` and ``body_producer`` may
412 be specified. ``body_producer`` is not supported on
413 ``curl_httpclient``. When using ``body_producer`` it is recommended
414 to pass a ``Content-Length`` in the headers as otherwise chunked
415 encoding will be used, and many servers do not support chunked
416 encoding on requests. New in Tornado 4.0
417 :arg str auth_username: Username for HTTP authentication
418 :arg str auth_password: Password for HTTP authentication
419 :arg str auth_mode: Authentication mode; default is "basic".
420 Allowed values are implementation-defined; ``curl_httpclient``
421 supports "basic" and "digest"; ``simple_httpclient`` only supports
422 "basic"
423 :arg float connect_timeout: Timeout for initial connection in seconds,
424 default 20 seconds (0 means no timeout)
425 :arg float request_timeout: Timeout for entire request in seconds,
426 default 20 seconds (0 means no timeout)
427 :arg if_modified_since: Timestamp for ``If-Modified-Since`` header
428 :type if_modified_since: `datetime` or `float`
429 :arg bool follow_redirects: Should redirects be followed automatically
430 or return the 3xx response? Default True.
431 :arg int max_redirects: Limit for ``follow_redirects``, default 5.
432 :arg str user_agent: String to send as ``User-Agent`` header
433 :arg bool decompress_response: Request a compressed response from
434 the server and decompress it after downloading. Default is True.
435 New in Tornado 4.0.
436 :arg bool use_gzip: Deprecated alias for ``decompress_response``
437 since Tornado 4.0.
438 :arg str network_interface: Network interface or source IP to use for request.
439 See ``curl_httpclient`` note below.
440 :arg collections.abc.Callable streaming_callback: If set, ``streaming_callback`` will
441 be run with each chunk of data as it is received, and
442 ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
443 the final response.
444 :arg collections.abc.Callable header_callback: If set, ``header_callback`` will
445 be run with each header line as it is received (including the
446 first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
447 containing only ``\r\n``. All lines include the trailing newline
448 characters). ``HTTPResponse.headers`` will be empty in the final
449 response. This is most useful in conjunction with
450 ``streaming_callback``, because it's the only way to get access to
451 header data while the request is in progress.
452 :arg collections.abc.Callable prepare_curl_callback: If set, will be called with
453 a ``pycurl.Curl`` object to allow the application to make additional
454 ``setopt`` calls.
455 :arg str proxy_host: HTTP proxy hostname. To use proxies,
456 ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username``,
457 ``proxy_pass`` and ``proxy_auth_mode`` are optional. Proxies are
458 currently only supported with ``curl_httpclient``.
459 :arg int proxy_port: HTTP proxy port
460 :arg str proxy_username: HTTP proxy username
461 :arg str proxy_password: HTTP proxy password
462 :arg str proxy_auth_mode: HTTP proxy Authentication mode;
463 default is "basic". supports "basic" and "digest"
464 :arg bool allow_nonstandard_methods: Allow unknown values for ``method``
465 argument? Default is False.
466 :arg bool validate_cert: For HTTPS requests, validate the server's
467 certificate? Default is True.
468 :arg str ca_certs: filename of CA certificates in PEM format,
469 or None to use defaults. See note below when used with
470 ``curl_httpclient``.
471 :arg str client_key: Filename for client SSL key, if any. See
472 note below when used with ``curl_httpclient``.
473 :arg str client_cert: Filename for client SSL certificate, if any.
474 See note below when used with ``curl_httpclient``.
475 :arg ssl.SSLContext ssl_options: `ssl.SSLContext` object for use in
476 ``simple_httpclient`` (unsupported by ``curl_httpclient``).
477 Overrides ``validate_cert``, ``ca_certs``, ``client_key``,
478 and ``client_cert``.
479 :arg bool allow_ipv6: Use IPv6 when available? Default is True.
480 :arg bool expect_100_continue: If true, send the
481 ``Expect: 100-continue`` header and wait for a continue response
482 before sending the request body. Only supported with
483 ``simple_httpclient``.
484
485 .. note::
486
487 When using ``curl_httpclient`` certain options may be
488 inherited by subsequent fetches because ``pycurl`` does
489 not allow them to be cleanly reset. This applies to the
490 ``ca_certs``, ``client_key``, ``client_cert``, and
491 ``network_interface`` arguments. If you use these
492 options, you should pass them on every request (you don't
493 have to always use the same values, but it's not possible
494 to mix requests that specify these options with ones that
495 use the defaults).
496
497 .. versionadded:: 3.1
498 The ``auth_mode`` argument.
499
500 .. versionadded:: 4.0
501 The ``body_producer`` and ``expect_100_continue`` arguments.
502
503 .. versionadded:: 4.2
504 The ``ssl_options`` argument.
505
506 .. versionadded:: 4.5
507 The ``proxy_auth_mode`` argument.
508 """
509 # Note that some of these attributes go through property setters
510 # defined below.
511 self.headers = headers # type: ignore
512 if if_modified_since:
513 self.headers["If-Modified-Since"] = httputil.format_timestamp(
514 if_modified_since
515 )
516 self.proxy_host = proxy_host
517 self.proxy_port = proxy_port
518 self.proxy_username = proxy_username
519 self.proxy_password = proxy_password
520 self.proxy_auth_mode = proxy_auth_mode
521 self.url = url
522 self.method = method
523 self.body = body # type: ignore
524 self.body_producer = body_producer
525 self.auth_username = auth_username
526 self.auth_password = auth_password
527 self.auth_mode = auth_mode
528 self.connect_timeout = connect_timeout
529 self.request_timeout = request_timeout
530 self.follow_redirects = follow_redirects
531 self.max_redirects = max_redirects
532 self.user_agent = user_agent
533 if decompress_response is not None:
534 self.decompress_response = decompress_response # type: Optional[bool]
535 else:
536 self.decompress_response = use_gzip
537 self.network_interface = network_interface
538 self.streaming_callback = streaming_callback
539 self.header_callback = header_callback
540 self.prepare_curl_callback = prepare_curl_callback
541 self.allow_nonstandard_methods = allow_nonstandard_methods
542 self.validate_cert = validate_cert
543 self.ca_certs = ca_certs
544 self.allow_ipv6 = allow_ipv6
545 self.client_key = client_key
546 self.client_cert = client_cert
547 self.ssl_options = ssl_options
548 self.expect_100_continue = expect_100_continue
549 self.start_time = time.time()
550
551 @property
552 def headers(self) -> httputil.HTTPHeaders:
553 # TODO: headers may actually be a plain dict until fairly late in
554 # the process (AsyncHTTPClient.fetch), but practically speaking,
555 # whenever the property is used they're already HTTPHeaders.
556 return self._headers # type: ignore
557
558 @headers.setter
559 def headers(self, value: Union[Dict[str, str], httputil.HTTPHeaders]) -> None:
560 if value is None:
561 self._headers = httputil.HTTPHeaders()
562 else:
563 self._headers = value # type: ignore
564
565 @property
566 def body(self) -> bytes:
567 return self._body
568
569 @body.setter
570 def body(self, value: Union[bytes, str]) -> None:
571 self._body = utf8(value)
572
573
574class HTTPResponse:
575 """HTTP Response object.
576
577 Attributes:
578
579 * ``request``: HTTPRequest object
580
581 * ``code``: numeric HTTP status code, e.g. 200 or 404
582
583 * ``reason``: human-readable reason phrase describing the status code
584
585 * ``headers``: `tornado.httputil.HTTPHeaders` object
586
587 * ``effective_url``: final location of the resource after following any
588 redirects
589
590 * ``buffer``: ``cStringIO`` object for response body
591
592 * ``body``: response body as bytes (created on demand from ``self.buffer``)
593
594 * ``error``: Exception object, if any
595
596 * ``request_time``: seconds from request start to finish. Includes all
597 network operations from DNS resolution to receiving the last byte of
598 data. Does not include time spent in the queue (due to the
599 ``max_clients`` option). If redirects were followed, only includes
600 the final request.
601
602 * ``start_time``: Time at which the HTTP operation started, based on
603 `time.time` (not the monotonic clock used by `.IOLoop.time`). May
604 be ``None`` if the request timed out while in the queue.
605
606 * ``time_info``: dictionary of diagnostic timing information from the
607 request. Available data are subject to change, but currently uses timings
608 available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html,
609 plus ``queue``, which is the delay (if any) introduced by waiting for
610 a slot under `AsyncHTTPClient`'s ``max_clients`` setting.
611
612 .. versionadded:: 5.1
613
614 Added the ``start_time`` attribute.
615
616 .. versionchanged:: 5.1
617
618 The ``request_time`` attribute previously included time spent in the queue
619 for ``simple_httpclient``, but not in ``curl_httpclient``. Now queueing time
620 is excluded in both implementations. ``request_time`` is now more accurate for
621 ``curl_httpclient`` because it uses a monotonic clock when available.
622 """
623
624 # I'm not sure why these don't get type-inferred from the references in __init__.
625 error = None # type: Optional[BaseException]
626 _error_is_response_code = False
627 request = None # type: HTTPRequest
628
629 def __init__(
630 self,
631 request: HTTPRequest,
632 code: int,
633 headers: Optional[httputil.HTTPHeaders] = None,
634 buffer: Optional[BytesIO] = None,
635 effective_url: Optional[str] = None,
636 error: Optional[BaseException] = None,
637 request_time: Optional[float] = None,
638 time_info: Optional[Dict[str, float]] = None,
639 reason: Optional[str] = None,
640 start_time: Optional[float] = None,
641 ) -> None:
642 if isinstance(request, _RequestProxy):
643 self.request = request.request
644 else:
645 self.request = request
646 self.code = code
647 self.reason = reason or httputil.responses.get(code, "Unknown")
648 if headers is not None:
649 self.headers = headers
650 else:
651 self.headers = httputil.HTTPHeaders()
652 self.buffer = buffer
653 self._body = None # type: Optional[bytes]
654 if effective_url is None:
655 self.effective_url = request.url
656 else:
657 self.effective_url = effective_url
658 self._error_is_response_code = False
659 if error is None:
660 if self.code < 200 or self.code >= 300:
661 self._error_is_response_code = True
662 self.error = HTTPError(self.code, message=self.reason, response=self)
663 else:
664 self.error = None
665 else:
666 self.error = error
667 self.start_time = start_time
668 self.request_time = request_time
669 self.time_info = time_info or {}
670
671 @property
672 def body(self) -> bytes:
673 if self.buffer is None:
674 return b""
675 elif self._body is None:
676 self._body = self.buffer.getvalue()
677
678 return self._body
679
680 def rethrow(self) -> None:
681 """If there was an error on the request, raise an `HTTPError`."""
682 if self.error:
683 raise self.error
684
685 def __repr__(self) -> str:
686 args = ",".join("%s=%r" % i for i in sorted(self.__dict__.items()))
687 return f"{self.__class__.__name__}({args})"
688
689
690class HTTPClientError(Exception):
691 """Exception thrown for an unsuccessful HTTP request.
692
693 Attributes:
694
695 * ``code`` - HTTP error integer error code, e.g. 404. Error code 599 is
696 used when no HTTP response was received, e.g. for a timeout.
697
698 * ``response`` - `HTTPResponse` object, if any.
699
700 Note that if ``follow_redirects`` is False, redirects become HTTPErrors,
701 and you can look at ``error.response.headers['Location']`` to see the
702 destination of the redirect.
703
704 .. versionchanged:: 5.1
705
706 Renamed from ``HTTPError`` to ``HTTPClientError`` to avoid collisions with
707 `tornado.web.HTTPError`. The name ``tornado.httpclient.HTTPError`` remains
708 as an alias.
709 """
710
711 def __init__(
712 self,
713 code: int,
714 message: Optional[str] = None,
715 response: Optional[HTTPResponse] = None,
716 ) -> None:
717 self.code = code
718 self.message = message or httputil.responses.get(code, "Unknown")
719 self.response = response
720 super().__init__(code, message, response)
721
722 def __str__(self) -> str:
723 return "HTTP %d: %s" % (self.code, self.message)
724
725 # There is a cyclic reference between self and self.response,
726 # which breaks the default __repr__ implementation.
727 # (especially on pypy, which doesn't have the same recursion
728 # detection as cpython).
729 __repr__ = __str__
730
731
732HTTPError = HTTPClientError
733
734
735class _RequestProxy:
736 """Combines an object with a dictionary of defaults.
737
738 Used internally by AsyncHTTPClient implementations.
739 """
740
741 def __init__(
742 self, request: HTTPRequest, defaults: Optional[Dict[str, Any]]
743 ) -> None:
744 self.request = request
745 self.defaults = defaults
746
747 def __getattr__(self, name: str) -> Any:
748 request_attr = getattr(self.request, name)
749 if request_attr is not None:
750 return request_attr
751 elif self.defaults is not None:
752 return self.defaults.get(name, None)
753 else:
754 return None
755
756
757def main() -> None:
758 from tornado.options import define, options, parse_command_line
759
760 define("print_headers", type=bool, default=False)
761 define("print_body", type=bool, default=True)
762 define("follow_redirects", type=bool, default=True)
763 define("validate_cert", type=bool, default=True)
764 define("proxy_host", type=str)
765 define("proxy_port", type=int)
766 args = parse_command_line()
767 client = HTTPClient()
768 for arg in args:
769 try:
770 response = client.fetch(
771 arg,
772 follow_redirects=options.follow_redirects,
773 validate_cert=options.validate_cert,
774 proxy_host=options.proxy_host,
775 proxy_port=options.proxy_port,
776 )
777 except HTTPError as e:
778 if e.response is not None:
779 response = e.response
780 else:
781 raise
782 if options.print_headers:
783 print(response.headers)
784 if options.print_body:
785 print(native_str(response.body))
786 client.close()
787
788
789if __name__ == "__main__":
790 main()
791