codekingpro/portable-devtools
114k
1import base64
2import binascii
3from contextlib import closing
4import copy
5import gzip
6import threading
7import datetime
8from io import BytesIO
9import subprocess
10import sys
11import time
12import typing # noqa: F401
13import unicodedata
14import unittest
15
16from tornado.escape import utf8, native_str, to_unicode
17from tornado import gen
18from tornado.httpclient import (
19 HTTPRequest,
20 HTTPResponse,
21 _RequestProxy,
22 HTTPError,
23 HTTPClient,
24)
25from tornado.httpserver import HTTPServer
26from tornado.ioloop import IOLoop
27from tornado.iostream import IOStream
28from tornado.log import gen_log, app_log
29from tornado import netutil
30from tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog
31from tornado.test.util import ignore_deprecation
32from tornado.web import Application, RequestHandler, url
33from tornado.httputil import format_timestamp, HTTPHeaders
34
35
36class HelloWorldHandler(RequestHandler):
37 def get(self):
38 name = self.get_argument("name", "world")
39 self.set_header("Content-Type", "text/plain")
40 self.finish("Hello %s!" % name)
41
42
43class PostHandler(RequestHandler):
44 def post(self):
45 self.finish(
46 "Post arg1: %s, arg2: %s"
47 % (self.get_argument("arg1"), self.get_argument("arg2"))
48 )
49
50
51class PutHandler(RequestHandler):
52 def put(self):
53 self.write("Put body: ")
54 self.write(self.request.body)
55
56
57class RedirectHandler(RequestHandler):
58 def prepare(self):
59 self.write("redirects can have bodies too")
60 self.redirect(
61 self.get_argument("url"), status=int(self.get_argument("status", "302"))
62 )
63
64
65class RedirectWithoutLocationHandler(RequestHandler):
66 def prepare(self):
67 # For testing error handling of a redirect with no location header.
68 self.set_status(301)
69 self.finish()
70
71
72class ChunkHandler(RequestHandler):
73 @gen.coroutine
74 def get(self):
75 self.write("asdf")
76 self.flush()
77 # Wait a bit to ensure the chunks are sent and received separately.
78 yield gen.sleep(0.01)
79 self.write("qwer")
80
81
82class AuthHandler(RequestHandler):
83 def get(self):
84 self.finish(self.request.headers["Authorization"])
85
86
87class CountdownHandler(RequestHandler):
88 def get(self, count):
89 count = int(count)
90 if count > 0:
91 self.redirect(self.reverse_url("countdown", count - 1))
92 else:
93 self.write("Zero")
94
95
96class EchoPostHandler(RequestHandler):
97 def post(self):
98 self.write(self.request.body)
99
100
101class UserAgentHandler(RequestHandler):
102 def get(self):
103 self.write(self.request.headers.get("User-Agent", "User agent not set"))
104
105
106class ContentLength304Handler(RequestHandler):
107 def get(self):
108 self.set_status(304)
109 self.set_header("Content-Length", 42)
110
111 def _clear_representation_headers(self):
112 # Tornado strips content-length from 304 responses, but here we
113 # want to simulate servers that include the headers anyway.
114 pass
115
116
117class PatchHandler(RequestHandler):
118 def patch(self):
119 "Return the request payload - so we can check it is being kept"
120 self.write(self.request.body)
121
122
123class AllMethodsHandler(RequestHandler):
124 SUPPORTED_METHODS = RequestHandler.SUPPORTED_METHODS + ("OTHER",) # type: ignore
125
126 def method(self):
127 assert self.request.method is not None
128 self.write(self.request.method)
129
130 get = head = post = put = delete = options = patch = other = method # type: ignore
131
132
133class SetHeaderHandler(RequestHandler):
134 def get(self):
135 # Use get_arguments for keys to get strings, but
136 # request.arguments for values to get bytes.
137 for k, v in zip(self.get_arguments("k"), self.request.arguments["v"]):
138 self.set_header(k, v)
139
140
141class InvalidGzipHandler(RequestHandler):
142 def get(self) -> None:
143 # set Content-Encoding manually to avoid automatic gzip encoding
144 self.set_header("Content-Type", "text/plain")
145 self.set_header("Content-Encoding", "gzip")
146 # Triggering the potential bug seems to depend on input length.
147 # This length is taken from the bad-response example reported in
148 # https://github.com/tornadoweb/tornado/pull/2875 (uncompressed).
149 text = "".join(f"Hello World {i}\n" for i in range(9000))[:149051]
150 body = gzip.compress(text.encode(), compresslevel=6) + b"\00"
151 self.write(body)
152
153
154class HeaderEncodingHandler(RequestHandler):
155 def get(self):
156 self.finish(self.request.headers["Foo"].encode("ISO8859-1"))
157
158
159# These tests end up getting run redundantly: once here with the default
160# HTTPClient implementation, and then again in each implementation's own
161# test suite.
162
163
164class HTTPClientCommonTestCase(AsyncHTTPTestCase):
165 def get_app(self):
166 return Application(
167 [
168 url("/hello", HelloWorldHandler),
169 url("/post", PostHandler),
170 url("/put", PutHandler),
171 url("/redirect", RedirectHandler),
172 url("/redirect_without_location", RedirectWithoutLocationHandler),
173 url("/chunk", ChunkHandler),
174 url("/auth", AuthHandler),
175 url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
176 url("/echopost", EchoPostHandler),
177 url("/user_agent", UserAgentHandler),
178 url("/304_with_content_length", ContentLength304Handler),
179 url("/all_methods", AllMethodsHandler),
180 url("/patch", PatchHandler),
181 url("/set_header", SetHeaderHandler),
182 url("/invalid_gzip", InvalidGzipHandler),
183 url("/header-encoding", HeaderEncodingHandler),
184 ],
185 gzip=True,
186 )
187
188 def test_patch_receives_payload(self):
189 body = b"some patch data"
190 response = self.fetch("/patch", method="PATCH", body=body)
191 self.assertEqual(response.code, 200)
192 self.assertEqual(response.body, body)
193
194 def test_hello_world(self):
195 response = self.fetch("/hello")
196 self.assertEqual(response.code, 200)
197 self.assertEqual(response.headers["Content-Type"], "text/plain")
198 self.assertEqual(response.body, b"Hello world!")
199 assert response.request_time is not None
200 self.assertEqual(int(response.request_time), 0)
201
202 response = self.fetch("/hello?name=Ben")
203 self.assertEqual(response.body, b"Hello Ben!")
204
205 def test_streaming_callback(self):
206 # streaming_callback is also tested in test_chunked
207 chunks = [] # type: typing.List[bytes]
208 response = self.fetch("/hello", streaming_callback=chunks.append)
209 # with streaming_callback, data goes to the callback and not response.body
210 self.assertEqual(chunks, [b"Hello world!"])
211 self.assertFalse(response.body)
212
213 def test_post(self):
214 response = self.fetch("/post", method="POST", body="arg1=foo&arg2=bar")
215 self.assertEqual(response.code, 200)
216 self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
217
218 def test_chunked(self):
219 response = self.fetch("/chunk")
220 self.assertEqual(response.body, b"asdfqwer")
221
222 chunks = [] # type: typing.List[bytes]
223 response = self.fetch("/chunk", streaming_callback=chunks.append)
224 self.assertEqual(chunks, [b"asdf", b"qwer"])
225 self.assertFalse(response.body)
226
227 def test_chunked_close(self):
228 # test case in which chunks spread read-callback processing
229 # over several ioloop iterations, but the connection is already closed.
230 sock, port = bind_unused_port()
231 with closing(sock):
232
233 @gen.coroutine
234 def accept_callback(conn, address):
235 # fake an HTTP server using chunked encoding where the final chunks
236 # and connection close all happen at once
237 stream = IOStream(conn)
238 request_data = yield stream.read_until(b"\r\n\r\n")
239 if b"HTTP/1." not in request_data:
240 self.skipTest("requires HTTP/1.x")
241 yield stream.write(
242 b"""\
243HTTP/1.1 200 OK
244Transfer-Encoding: chunked
245
2461
2471
2481
2492
2500
251
252""".replace(
253 b"\n", b"\r\n"
254 )
255 )
256 stream.close()
257
258 netutil.add_accept_handler(sock, accept_callback) # type: ignore
259 resp = self.fetch("http://127.0.0.1:%d/" % port)
260 resp.rethrow()
261 self.assertEqual(resp.body, b"12")
262 self.io_loop.remove_handler(sock.fileno())
263
264 def test_basic_auth(self):
265 # This test data appears in section 2 of RFC 7617.
266 self.assertEqual(
267 self.fetch(
268 "/auth", auth_username="Aladdin", auth_password="open sesame"
269 ).body,
270 b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==",
271 )
272
273 def test_basic_auth_explicit_mode(self):
274 self.assertEqual(
275 self.fetch(
276 "/auth",
277 auth_username="Aladdin",
278 auth_password="open sesame",
279 auth_mode="basic",
280 ).body,
281 b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==",
282 )
283
284 def test_basic_auth_unicode(self):
285 # This test data appears in section 2.1 of RFC 7617.
286 self.assertEqual(
287 self.fetch("/auth", auth_username="test", auth_password="123£").body,
288 b"Basic dGVzdDoxMjPCow==",
289 )
290
291 # The standard mandates NFC. Give it a decomposed username
292 # and ensure it is normalized to composed form.
293 username = unicodedata.normalize("NFD", "josé")
294 self.assertEqual(
295 self.fetch("/auth", auth_username=username, auth_password="səcrət").body,
296 b"Basic am9zw6k6c8mZY3LJmXQ=",
297 )
298
299 def test_unsupported_auth_mode(self):
300 # curl and simple clients handle errors a bit differently; the
301 # important thing is that they don't fall back to basic auth
302 # on an unknown mode.
303 with ExpectLog(gen_log, "uncaught exception", required=False):
304 with self.assertRaises((ValueError, HTTPError)): # type: ignore
305 self.fetch(
306 "/auth",
307 auth_username="Aladdin",
308 auth_password="open sesame",
309 auth_mode="asdf",
310 raise_error=True,
311 )
312
313 def test_follow_redirect(self):
314 response = self.fetch("/countdown/2", follow_redirects=False)
315 self.assertEqual(302, response.code)
316 self.assertTrue(response.headers["Location"].endswith("/countdown/1"))
317
318 response = self.fetch("/countdown/2")
319 self.assertEqual(200, response.code)
320 self.assertTrue(response.effective_url.endswith("/countdown/0"))
321 self.assertEqual(b"Zero", response.body)
322
323 def test_redirect_without_location(self):
324 response = self.fetch("/redirect_without_location", follow_redirects=True)
325 # If there is no location header, the redirect response should
326 # just be returned as-is. (This should arguably raise an
327 # error, but libcurl doesn't treat this as an error, so we
328 # don't either).
329 self.assertEqual(301, response.code)
330
331 def test_redirect_put_with_body(self):
332 response = self.fetch(
333 "/redirect?url=/put&status=307", method="PUT", body="hello"
334 )
335 self.assertEqual(response.body, b"Put body: hello")
336
337 def test_redirect_put_without_body(self):
338 # This "without body" edge case is similar to what happens with body_producer.
339 response = self.fetch(
340 "/redirect?url=/put&status=307",
341 method="PUT",
342 allow_nonstandard_methods=True,
343 )
344 self.assertEqual(response.body, b"Put body: ")
345
346 def test_method_after_redirect(self):
347 # Legacy redirect codes (301, 302) convert POST requests to GET.
348 for status in [301, 302, 303]:
349 url = "/redirect?url=/all_methods&status=%d" % status
350 resp = self.fetch(url, method="POST", body=b"")
351 self.assertEqual(b"GET", resp.body)
352
353 # Other methods are left alone, except for 303 redirect, depending on client
354 for method in ["GET", "OPTIONS", "PUT", "DELETE"]:
355 resp = self.fetch(url, method=method, allow_nonstandard_methods=True)
356 if status in [301, 302]:
357 self.assertEqual(utf8(method), resp.body)
358 else:
359 self.assertIn(resp.body, [utf8(method), b"GET"])
360
361 # HEAD is different so check it separately.
362 resp = self.fetch(url, method="HEAD")
363 self.assertEqual(200, resp.code)
364 self.assertEqual(b"", resp.body)
365
366 # Newer redirects always preserve the original method.
367 for status in [307, 308]:
368 url = "/redirect?url=/all_methods&status=307"
369 for method in ["GET", "OPTIONS", "POST", "PUT", "DELETE"]:
370 resp = self.fetch(url, method=method, allow_nonstandard_methods=True)
371 self.assertEqual(method, to_unicode(resp.body))
372 resp = self.fetch(url, method="HEAD")
373 self.assertEqual(200, resp.code)
374 self.assertEqual(b"", resp.body)
375
376 def test_credentials_in_url(self):
377 url = self.get_url("/auth").replace("http://", "http://me:secret@")
378 response = self.fetch(url)
379 self.assertEqual(b"Basic " + base64.b64encode(b"me:secret"), response.body)
380
381 def test_body_encoding(self):
382 unicode_body = "\xe9"
383 byte_body = binascii.a2b_hex(b"e9")
384
385 # unicode string in body gets converted to utf8
386 response = self.fetch(
387 "/echopost",
388 method="POST",
389 body=unicode_body,
390 headers={"Content-Type": "application/blah"},
391 )
392 self.assertEqual(response.headers["Content-Length"], "2")
393 self.assertEqual(response.body, utf8(unicode_body))
394
395 # byte strings pass through directly
396 response = self.fetch(
397 "/echopost",
398 method="POST",
399 body=byte_body,
400 headers={"Content-Type": "application/blah"},
401 )
402 self.assertEqual(response.headers["Content-Length"], "1")
403 self.assertEqual(response.body, byte_body)
404
405 # Mixing unicode in headers and byte string bodies shouldn't
406 # break anything
407 response = self.fetch(
408 "/echopost",
409 method="POST",
410 body=byte_body,
411 headers={"Content-Type": "application/blah"},
412 user_agent="foo",
413 )
414 self.assertEqual(response.headers["Content-Length"], "1")
415 self.assertEqual(response.body, byte_body)
416
417 def test_types(self):
418 response = self.fetch("/hello")
419 self.assertEqual(type(response.body), bytes)
420 self.assertEqual(type(response.headers["Content-Type"]), str)
421 self.assertEqual(type(response.code), int)
422 self.assertEqual(type(response.effective_url), str)
423
424 def test_gzip(self):
425 # All the tests in this file should be using gzip, but this test
426 # ensures that it is in fact getting compressed, and also tests
427 # the httpclient's decompress=False option.
428 # Setting Accept-Encoding manually bypasses the client's
429 # decompression so we can see the raw data.
430 response = self.fetch(
431 "/chunk", decompress_response=False, headers={"Accept-Encoding": "gzip"}
432 )
433 self.assertEqual(response.headers["Content-Encoding"], "gzip")
434 self.assertNotEqual(response.body, b"asdfqwer")
435 # Our test data gets bigger when gzipped. Oops. :)
436 # Chunked encoding bypasses the MIN_LENGTH check.
437 self.assertEqual(len(response.body), 34)
438 f = gzip.GzipFile(mode="r", fileobj=response.buffer)
439 self.assertEqual(f.read(), b"asdfqwer")
440
441 def test_invalid_gzip(self):
442 # test if client hangs on tricky invalid gzip
443 # curl/simple httpclient have different behavior (exception, logging)
444 with ExpectLog(
445 gen_log, ".*Malformed HTTP message.*unconsumed gzip data", required=False
446 ):
447 try:
448 response = self.fetch("/invalid_gzip")
449 self.assertEqual(response.code, 200)
450 self.assertEqual(response.body[:14], b"Hello World 0\n")
451 except HTTPError:
452 pass # acceptable
453
454 def test_header_callback(self):
455 first_line = []
456 headers = {}
457 chunks = []
458
459 def header_callback(header_line):
460 if header_line.startswith("HTTP/1.1 101"):
461 # Upgrading to HTTP/2
462 pass
463 elif header_line.startswith("HTTP/"):
464 first_line.append(header_line)
465 elif header_line != "\r\n":
466 k, v = header_line.split(":", 1)
467 headers[k.lower()] = v.strip()
468
469 def streaming_callback(chunk):
470 # All header callbacks are run before any streaming callbacks,
471 # so the header data is available to process the data as it
472 # comes in.
473 self.assertEqual(headers["content-type"], "text/html; charset=UTF-8")
474 chunks.append(chunk)
475
476 self.fetch(
477 "/chunk",
478 header_callback=header_callback,
479 streaming_callback=streaming_callback,
480 )
481 self.assertEqual(len(first_line), 1, first_line)
482 self.assertRegex(first_line[0], "HTTP/[0-9]\\.[0-9] 200.*\r\n")
483 self.assertEqual(chunks, [b"asdf", b"qwer"])
484
485 def test_header_callback_to_parse_line(self):
486 # Make a request with header_callback and feed the headers to HTTPHeaders.parse_line.
487 # (Instead of HTTPHeaders.parse which is used in normal cases). Ensure that the resulting
488 # headers are as expected, and in particular do not have trailing whitespace added
489 # due to the final CRLF line.
490 headers = HTTPHeaders()
491
492 def header_callback(line):
493 if line.startswith("HTTP/"):
494 # Ignore the first status line
495 return
496 headers.parse_line(line)
497
498 self.fetch("/hello", header_callback=header_callback)
499 for k, v in headers.get_all():
500 self.assertTrue(v == v.strip(), (k, v))
501
502 @gen_test
503 def test_configure_defaults(self):
504 defaults = dict(user_agent="TestDefaultUserAgent", allow_ipv6=False)
505 # Construct a new instance of the configured client class
506 client = self.http_client.__class__(force_instance=True, defaults=defaults)
507 try:
508 response = yield client.fetch(self.get_url("/user_agent"))
509 self.assertEqual(response.body, b"TestDefaultUserAgent")
510 finally:
511 client.close()
512
513 def test_header_types(self):
514 # Header values may be passed as character or utf8 byte strings,
515 # in a plain dictionary or an HTTPHeaders object.
516 # Keys must always be the native str type.
517 # All combinations should have the same results on the wire.
518 for value in ["MyUserAgent", b"MyUserAgent"]:
519 for container in [dict, HTTPHeaders]:
520 headers = container()
521 headers["User-Agent"] = value
522 resp = self.fetch("/user_agent", headers=headers)
523 self.assertEqual(
524 resp.body,
525 b"MyUserAgent",
526 "response=%r, value=%r, container=%r"
527 % (resp.body, value, container),
528 )
529
530 def test_multi_line_headers(self):
531 # Multi-line http headers are rare but rfc-allowed
532 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
533 sock, port = bind_unused_port()
534 with closing(sock):
535
536 @gen.coroutine
537 def accept_callback(conn, address):
538 stream = IOStream(conn)
539 request_data = yield stream.read_until(b"\r\n\r\n")
540 if b"HTTP/1." not in request_data:
541 self.skipTest("requires HTTP/1.x")
542 yield stream.write(
543 b"""\
544HTTP/1.1 200 OK
545X-XSS-Protection: 1;
546\tmode=block
547
548""".replace(
549 b"\n", b"\r\n"
550 )
551 )
552 stream.close()
553
554 netutil.add_accept_handler(sock, accept_callback) # type: ignore
555 try:
556 resp = self.fetch("http://127.0.0.1:%d/" % port)
557 resp.rethrow()
558 self.assertEqual(resp.headers["X-XSS-Protection"], "1; mode=block")
559 finally:
560 self.io_loop.remove_handler(sock.fileno())
561
562 @gen_test
563 def test_header_encoding(self):
564 response = yield self.http_client.fetch(
565 self.get_url("/header-encoding"),
566 headers={
567 "Foo": "b\xe4r",
568 },
569 )
570 self.assertEqual(response.body, "b\xe4r".encode("ISO8859-1"))
571
572 def test_304_with_content_length(self):
573 # According to the spec 304 responses SHOULD NOT include
574 # Content-Length or other entity headers, but some servers do it
575 # anyway.
576 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
577 response = self.fetch("/304_with_content_length")
578 self.assertEqual(response.code, 304)
579 self.assertEqual(response.headers["Content-Length"], "42")
580
581 @gen_test
582 def test_future_interface(self):
583 response = yield self.http_client.fetch(self.get_url("/hello"))
584 self.assertEqual(response.body, b"Hello world!")
585
586 @gen_test
587 def test_future_http_error(self):
588 with self.assertRaises(HTTPError) as context:
589 yield self.http_client.fetch(self.get_url("/notfound"))
590 assert context.exception is not None
591 assert context.exception.response is not None
592 self.assertEqual(context.exception.code, 404)
593 self.assertEqual(context.exception.response.code, 404)
594
595 @gen_test
596 def test_future_http_error_no_raise(self):
597 response = yield self.http_client.fetch(
598 self.get_url("/notfound"), raise_error=False
599 )
600 self.assertEqual(response.code, 404)
601
602 @gen_test
603 def test_reuse_request_from_response(self):
604 # The response.request attribute should be an HTTPRequest, not
605 # a _RequestProxy.
606 # This test uses self.http_client.fetch because self.fetch calls
607 # self.get_url on the input unconditionally.
608 url = self.get_url("/hello")
609 response = yield self.http_client.fetch(url)
610 self.assertEqual(response.request.url, url)
611 self.assertTrue(isinstance(response.request, HTTPRequest))
612 response2 = yield self.http_client.fetch(response.request)
613 self.assertEqual(response2.body, b"Hello world!")
614
615 @gen_test
616 def test_bind_source_ip(self):
617 url = self.get_url("/hello")
618 request = HTTPRequest(url, network_interface="127.0.0.1")
619 response = yield self.http_client.fetch(request)
620 self.assertEqual(response.code, 200)
621
622 with self.assertRaises((ValueError, HTTPError)) as context: # type: ignore
623 request = HTTPRequest(url, network_interface="not-interface-or-ip")
624 yield self.http_client.fetch(request)
625 self.assertIn("not-interface-or-ip", str(context.exception))
626
627 def test_all_methods(self):
628 for method in ["GET", "DELETE", "OPTIONS"]:
629 response = self.fetch("/all_methods", method=method)
630 self.assertEqual(response.body, utf8(method))
631 for method in ["POST", "PUT", "PATCH"]:
632 response = self.fetch("/all_methods", method=method, body=b"")
633 self.assertEqual(response.body, utf8(method))
634 response = self.fetch("/all_methods", method="HEAD")
635 self.assertEqual(response.body, b"")
636 response = self.fetch(
637 "/all_methods", method="OTHER", allow_nonstandard_methods=True
638 )
639 self.assertEqual(response.body, b"OTHER")
640
641 def test_body_sanity_checks(self):
642 # These methods require a body.
643 for method in ("POST", "PUT", "PATCH"):
644 with self.assertRaises(ValueError) as context:
645 self.fetch("/all_methods", method=method, raise_error=True)
646 self.assertIn("must not be None", str(context.exception))
647
648 resp = self.fetch(
649 "/all_methods", method=method, allow_nonstandard_methods=True
650 )
651 self.assertEqual(resp.code, 200)
652
653 # These methods don't allow a body.
654 for method in ("GET", "DELETE", "OPTIONS"):
655 with self.assertRaises(ValueError) as context:
656 self.fetch(
657 "/all_methods", method=method, body=b"asdf", raise_error=True
658 )
659 self.assertIn("must be None", str(context.exception))
660
661 # In most cases this can be overridden, but curl_httpclient
662 # does not allow body with a GET at all.
663 if method != "GET":
664 self.fetch(
665 "/all_methods",
666 method=method,
667 body=b"asdf",
668 allow_nonstandard_methods=True,
669 raise_error=True,
670 )
671 self.assertEqual(resp.code, 200)
672
673 # This test causes odd failures with the combination of
674 # curl_httpclient (at least with the version of libcurl available
675 # on ubuntu 12.04), TwistedIOLoop, and epoll. For POST (but not PUT),
676 # curl decides the response came back too soon and closes the connection
677 # to start again. It does this *before* telling the socket callback to
678 # unregister the FD. Some IOLoop implementations have special kernel
679 # integration to discover this immediately. Tornado's IOLoops
680 # ignore errors on remove_handler to accommodate this behavior, but
681 # Twisted's reactor does not. The removeReader call fails and so
682 # do all future removeAll calls (which our tests do at cleanup).
683 #
684 # def test_post_307(self):
685 # response = self.fetch("/redirect?status=307&url=/post",
686 # method="POST", body=b"arg1=foo&arg2=bar")
687 # self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
688
689 def test_put_307(self):
690 response = self.fetch(
691 "/redirect?status=307&url=/put", method="PUT", body=b"hello"
692 )
693 response.rethrow()
694 self.assertEqual(response.body, b"Put body: hello")
695
696 def test_non_ascii_header(self):
697 # Non-ascii headers are sent as latin1.
698 response = self.fetch("/set_header?k=foo&v=%E9")
699 response.rethrow()
700 self.assertEqual(response.headers["Foo"], native_str("\u00e9"))
701
702 def test_response_times(self):
703 # A few simple sanity checks of the response time fields to
704 # make sure they're using the right basis (between the
705 # wall-time and monotonic clocks).
706 start_time = time.time()
707 response = self.fetch("/hello")
708 response.rethrow()
709 self.assertIsNotNone(response.request_time)
710 assert response.request_time is not None # for mypy
711 self.assertGreaterEqual(response.request_time, 0)
712 self.assertLess(response.request_time, 1.0)
713 # A very crude check to make sure that start_time is based on
714 # wall time and not the monotonic clock.
715 self.assertIsNotNone(response.start_time)
716 assert response.start_time is not None # for mypy
717 self.assertLess(abs(response.start_time - start_time), 1.0)
718
719 for k, v in response.time_info.items():
720 self.assertTrue(0 <= v < 1.0, f"time_info[{k}] out of bounds: {v}")
721
722 def test_zero_timeout(self):
723 response = self.fetch("/hello", connect_timeout=0)
724 self.assertEqual(response.code, 200)
725
726 response = self.fetch("/hello", request_timeout=0)
727 self.assertEqual(response.code, 200)
728
729 response = self.fetch("/hello", connect_timeout=0, request_timeout=0)
730 self.assertEqual(response.code, 200)
731
732 @gen_test
733 def test_error_after_cancel(self):
734 fut = self.http_client.fetch(self.get_url("/404"))
735 self.assertTrue(fut.cancel())
736 with ExpectLog(app_log, "Exception after Future was cancelled") as el:
737 # We can't wait on the cancelled Future any more, so just
738 # let the IOLoop run until the exception gets logged (or
739 # not, in which case we exit the loop and ExpectLog will
740 # raise).
741 for i in range(100):
742 yield gen.sleep(0.01)
743 if el.logged_stack:
744 break
745
746 def test_header_crlf(self):
747 # Ensure that the client doesn't allow CRLF injection in headers. RFC 9112 section 2.2
748 # prohibits a bare CR specifically and "a recipient MAY recognize a single LF as a line
749 # terminator" so we check each character separately as well as the (redundant) CRLF pair.
750 for header, name in [
751 ("foo\rbar:", "cr"),
752 ("foo\nbar:", "lf"),
753 ("foo\r\nbar:", "crlf"),
754 ]:
755 with self.subTest(name=name, position="value"):
756 with self.assertRaises(ValueError):
757 self.fetch("/hello", headers={"foo": header})
758 with self.subTest(name=name, position="key"):
759 with self.assertRaises(ValueError):
760 self.fetch("/hello", headers={header: "foo"})
761
762
763class RequestProxyTest(unittest.TestCase):
764 def test_request_set(self):
765 proxy = _RequestProxy(
766 HTTPRequest("http://example.com/", user_agent="foo"), dict()
767 )
768 self.assertEqual(proxy.user_agent, "foo")
769
770 def test_default_set(self):
771 proxy = _RequestProxy(
772 HTTPRequest("http://example.com/"), dict(network_interface="foo")
773 )
774 self.assertEqual(proxy.network_interface, "foo")
775
776 def test_both_set(self):
777 proxy = _RequestProxy(
778 HTTPRequest("http://example.com/", proxy_host="foo"), dict(proxy_host="bar")
779 )
780 self.assertEqual(proxy.proxy_host, "foo")
781
782 def test_neither_set(self):
783 proxy = _RequestProxy(HTTPRequest("http://example.com/"), dict())
784 self.assertIsNone(proxy.auth_username)
785
786 def test_bad_attribute(self):
787 proxy = _RequestProxy(HTTPRequest("http://example.com/"), dict())
788 with self.assertRaises(AttributeError):
789 proxy.foo
790
791 def test_defaults_none(self):
792 proxy = _RequestProxy(HTTPRequest("http://example.com/"), None)
793 self.assertIsNone(proxy.auth_username)
794
795
796class HTTPResponseTestCase(unittest.TestCase):
797 def test_str(self):
798 response = HTTPResponse( # type: ignore
799 HTTPRequest("http://example.com"), 200, buffer=BytesIO()
800 )
801 s = str(response)
802 self.assertTrue(s.startswith("HTTPResponse("))
803 self.assertIn("code=200", s)
804
805
806class SyncHTTPClientTest(unittest.TestCase):
807 def setUp(self):
808 self.server_ioloop = IOLoop(make_current=False)
809 event = threading.Event()
810
811 @gen.coroutine
812 def init_server():
813 sock, self.port = bind_unused_port()
814 app = Application([("/", HelloWorldHandler)])
815 self.server = HTTPServer(app)
816 self.server.add_socket(sock)
817 event.set()
818
819 def start():
820 self.server_ioloop.run_sync(init_server)
821 self.server_ioloop.start()
822
823 self.server_thread = threading.Thread(target=start)
824 self.server_thread.start()
825 event.wait()
826
827 self.http_client = HTTPClient()
828
829 def tearDown(self):
830 def stop_server():
831 self.server.stop()
832 # Delay the shutdown of the IOLoop by several iterations because
833 # the server may still have some cleanup work left when
834 # the client finishes with the response (this is noticeable
835 # with http/2, which leaves a Future with an unexamined
836 # StreamClosedError on the loop).
837
838 @gen.coroutine
839 def slow_stop():
840 yield self.server.close_all_connections()
841 # The number of iterations is difficult to predict. Typically,
842 # one is sufficient, although sometimes it needs more.
843 for i in range(5):
844 yield
845 self.server_ioloop.stop()
846
847 self.server_ioloop.add_callback(slow_stop)
848
849 self.server_ioloop.add_callback(stop_server)
850 self.server_thread.join()
851 self.http_client.close()
852 self.server_ioloop.close(all_fds=True)
853
854 def get_url(self, path):
855 return "http://127.0.0.1:%d%s" % (self.port, path)
856
857 def test_sync_client(self):
858 response = self.http_client.fetch(self.get_url("/"))
859 self.assertEqual(b"Hello world!", response.body)
860
861 def test_sync_client_error(self):
862 # Synchronous HTTPClient raises errors directly; no need for
863 # response.rethrow()
864 with self.assertRaises(HTTPError) as assertion:
865 self.http_client.fetch(self.get_url("/notfound"))
866 self.assertEqual(assertion.exception.code, 404)
867
868
869class SyncHTTPClientSubprocessTest(unittest.TestCase):
870 def test_destructor_log(self):
871 # Regression test for
872 # https://github.com/tornadoweb/tornado/issues/2539
873 #
874 # In the past, the following program would log an
875 # "inconsistent AsyncHTTPClient cache" error from a destructor
876 # when the process is shutting down. The shutdown process is
877 # subtle and I don't fully understand it; the failure does not
878 # manifest if that lambda isn't there or is a simpler object
879 # like an int (nor does it manifest in the tornado test suite
880 # as a whole, which is why we use this subprocess).
881 proc = subprocess.run(
882 [
883 sys.executable,
884 "-c",
885 "from tornado.httpclient import HTTPClient; f = lambda: None; c = HTTPClient()",
886 ],
887 stdout=subprocess.PIPE,
888 stderr=subprocess.STDOUT,
889 check=True,
890 timeout=15,
891 )
892 if proc.stdout:
893 print("STDOUT:")
894 print(to_unicode(proc.stdout))
895 if proc.stdout:
896 self.fail("subprocess produced unexpected output")
897
898
899class HTTPRequestTestCase(unittest.TestCase):
900 def test_headers(self):
901 request = HTTPRequest("http://example.com", headers={"foo": "bar"})
902 self.assertEqual(request.headers, {"foo": "bar"})
903
904 def test_headers_setter(self):
905 request = HTTPRequest("http://example.com")
906 request.headers = {"bar": "baz"} # type: ignore
907 self.assertEqual(request.headers, {"bar": "baz"})
908
909 def test_null_headers_setter(self):
910 request = HTTPRequest("http://example.com")
911 request.headers = None # type: ignore
912 self.assertEqual(request.headers, {})
913
914 def test_body(self):
915 request = HTTPRequest("http://example.com", body="foo")
916 self.assertEqual(request.body, utf8("foo"))
917
918 def test_body_setter(self):
919 request = HTTPRequest("http://example.com")
920 request.body = "foo" # type: ignore
921 self.assertEqual(request.body, utf8("foo"))
922
923 def test_if_modified_since(self):
924 http_date = datetime.datetime.now(datetime.timezone.utc)
925 request = HTTPRequest("http://example.com", if_modified_since=http_date)
926 self.assertEqual(
927 request.headers, {"If-Modified-Since": format_timestamp(http_date)}
928 )
929
930 def test_if_modified_since_naive_deprecated(self):
931 with ignore_deprecation():
932 http_date = datetime.datetime.utcnow()
933 request = HTTPRequest("http://example.com", if_modified_since=http_date)
934 self.assertEqual(
935 request.headers, {"If-Modified-Since": format_timestamp(http_date)}
936 )
937
938
939class HTTPErrorTestCase(unittest.TestCase):
940 def test_copy(self):
941 e = HTTPError(403)
942 e2 = copy.copy(e)
943 self.assertIsNot(e, e2)
944 self.assertEqual(e.code, e2.code)
945
946 def test_plain_error(self):
947 e = HTTPError(403)
948 self.assertEqual(str(e), "HTTP 403: Forbidden")
949 self.assertEqual(repr(e), "HTTP 403: Forbidden")
950
951 def test_error_with_response(self):
952 resp = HTTPResponse(HTTPRequest("http://example.com/"), 403)
953 with self.assertRaises(HTTPError) as cm:
954 resp.rethrow()
955 e = cm.exception
956 self.assertEqual(str(e), "HTTP 403: Forbidden")
957 self.assertEqual(repr(e), "HTTP 403: Forbidden")
958