codekingpro/portable-devtools
115k
1import asyncio
2import io
3import os
4import pathlib
5import sys
6from contextlib import suppress
7from enum import Enum, auto
8from mimetypes import MimeTypes
9from stat import S_ISREG
10from types import MappingProxyType
11from typing import ( # noqa
12 IO,
13 TYPE_CHECKING,
14 Any,
15 Awaitable,
16 Callable,
17 Final,
18 Iterator,
19 List,
20 Optional,
21 Set,
22 Tuple,
23 Union,
24 cast,
25)
26
27from . import hdrs
28from .abc import AbstractStreamWriter
29from .helpers import ETAG_ANY, ETag, must_be_empty_body
30from .typedefs import LooseHeaders, PathLike
31from .web_exceptions import (
32 HTTPForbidden,
33 HTTPNotFound,
34 HTTPNotModified,
35 HTTPPartialContent,
36 HTTPPreconditionFailed,
37 HTTPRequestRangeNotSatisfiable,
38)
39from .web_response import StreamResponse
40
41__all__ = ("FileResponse",)
42
43if TYPE_CHECKING:
44 from .web_request import BaseRequest
45
46
47_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]]
48
49
50NOSENDFILE: Final[bool] = bool(os.environ.get("AIOHTTP_NOSENDFILE"))
51
52CONTENT_TYPES: Final[MimeTypes] = MimeTypes()
53
54# File extension to IANA encodings map that will be checked in the order defined.
55ENCODING_EXTENSIONS = MappingProxyType(
56 {ext: CONTENT_TYPES.encodings_map[ext] for ext in (".br", ".gz")}
57)
58
59FALLBACK_CONTENT_TYPE = "application/octet-stream"
60
61# Provide additional MIME type/extension pairs to be recognized.
62# https://en.wikipedia.org/wiki/List_of_archive_formats#Compression_only
63ADDITIONAL_CONTENT_TYPES = MappingProxyType(
64 {
65 "application/gzip": ".gz",
66 "application/x-brotli": ".br",
67 "application/x-bzip2": ".bz2",
68 "application/x-compress": ".Z",
69 "application/x-xz": ".xz",
70 }
71)
72
73
74class _FileResponseResult(Enum):
75 """The result of the file response."""
76
77 SEND_FILE = auto() # Ie a regular file to send
78 NOT_ACCEPTABLE = auto() # Ie a socket, or non-regular file
79 PRE_CONDITION_FAILED = auto() # Ie If-Match or If-None-Match failed
80 NOT_MODIFIED = auto() # 304 Not Modified
81
82
83# Add custom pairs and clear the encodings map so guess_type ignores them.
84CONTENT_TYPES.encodings_map.clear()
85for content_type, extension in ADDITIONAL_CONTENT_TYPES.items():
86 CONTENT_TYPES.add_type(content_type, extension)
87
88
89_CLOSE_FUTURES: Set[asyncio.Future[None]] = set()
90
91
92class FileResponse(StreamResponse):
93 """A response object can be used to send files."""
94
95 def __init__(
96 self,
97 path: PathLike,
98 chunk_size: int = 256 * 1024,
99 status: int = 200,
100 reason: Optional[str] = None,
101 headers: Optional[LooseHeaders] = None,
102 ) -> None:
103 super().__init__(status=status, reason=reason, headers=headers)
104
105 self._path = pathlib.Path(path)
106 self._chunk_size = chunk_size
107
108 def _seek_and_read(self, fobj: IO[Any], offset: int, chunk_size: int) -> bytes:
109 fobj.seek(offset)
110 return fobj.read(chunk_size) # type: ignore[no-any-return]
111
112 async def _sendfile_fallback(
113 self, writer: AbstractStreamWriter, fobj: IO[Any], offset: int, count: int
114 ) -> AbstractStreamWriter:
115 # To keep memory usage low,fobj is transferred in chunks
116 # controlled by the constructor's chunk_size argument.
117
118 chunk_size = self._chunk_size
119 loop = asyncio.get_event_loop()
120 chunk = await loop.run_in_executor(
121 None, self._seek_and_read, fobj, offset, min(chunk_size, count)
122 )
123 while chunk:
124 await writer.write(chunk)
125 count = count - len(chunk)
126 if count <= 0:
127 break
128 chunk = await loop.run_in_executor(None, fobj.read, min(chunk_size, count))
129
130 await writer.drain()
131 return writer
132
133 async def _sendfile(
134 self, request: "BaseRequest", fobj: IO[Any], offset: int, count: int
135 ) -> AbstractStreamWriter:
136 writer = await super().prepare(request)
137 assert writer is not None
138
139 if NOSENDFILE or self.compression:
140 return await self._sendfile_fallback(writer, fobj, offset, count)
141
142 loop = request._loop
143 transport = request.transport
144 assert transport is not None
145
146 try:
147 await loop.sendfile(transport, fobj, offset, count)
148 except NotImplementedError:
149 return await self._sendfile_fallback(writer, fobj, offset, count)
150
151 await super().write_eof()
152 return writer
153
154 @staticmethod
155 def _etag_match(etag_value: str, etags: Tuple[ETag, ...], *, weak: bool) -> bool:
156 if len(etags) == 1 and etags[0].value == ETAG_ANY:
157 return True
158 return any(
159 etag.value == etag_value for etag in etags if weak or not etag.is_weak
160 )
161
162 async def _not_modified(
163 self, request: "BaseRequest", etag_value: str, last_modified: float
164 ) -> Optional[AbstractStreamWriter]:
165 self.set_status(HTTPNotModified.status_code)
166 self._length_check = False
167 self.etag = etag_value
168 self.last_modified = last_modified
169 # Delete any Content-Length headers provided by user. HTTP 304
170 # should always have empty response body
171 return await super().prepare(request)
172
173 async def _precondition_failed(
174 self, request: "BaseRequest"
175 ) -> Optional[AbstractStreamWriter]:
176 self.set_status(HTTPPreconditionFailed.status_code)
177 self.content_length = 0
178 return await super().prepare(request)
179
180 def _make_response(
181 self, request: "BaseRequest", accept_encoding: str
182 ) -> Tuple[
183 _FileResponseResult, Optional[io.BufferedReader], os.stat_result, Optional[str]
184 ]:
185 """Return the response result, io object, stat result, and encoding.
186
187 If an uncompressed file is returned, the encoding is set to
188 :py:data:`None`.
189
190 This method should be called from a thread executor
191 since it calls os.stat which may block.
192 """
193 file_path, st, file_encoding = self._get_file_path_stat_encoding(
194 accept_encoding
195 )
196 if not file_path:
197 return _FileResponseResult.NOT_ACCEPTABLE, None, st, None
198
199 etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}"
200
201 # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.1-2
202 if (ifmatch := request.if_match) is not None and not self._etag_match(
203 etag_value, ifmatch, weak=False
204 ):
205 return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding
206
207 if (
208 (unmodsince := request.if_unmodified_since) is not None
209 and ifmatch is None
210 and st.st_mtime > unmodsince.timestamp()
211 ):
212 return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding
213
214 # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.2-2
215 if (ifnonematch := request.if_none_match) is not None and self._etag_match(
216 etag_value, ifnonematch, weak=True
217 ):
218 return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding
219
220 if (
221 (modsince := request.if_modified_since) is not None
222 and ifnonematch is None
223 and st.st_mtime <= modsince.timestamp()
224 ):
225 return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding
226
227 fobj = file_path.open("rb")
228 with suppress(OSError):
229 # fstat() may not be available on all platforms
230 # Once we open the file, we want the fstat() to ensure
231 # the file has not changed between the first stat()
232 # and the open().
233 st = os.stat(fobj.fileno())
234 return _FileResponseResult.SEND_FILE, fobj, st, file_encoding
235
236 def _get_file_path_stat_encoding(
237 self, accept_encoding: str
238 ) -> Tuple[Optional[pathlib.Path], os.stat_result, Optional[str]]:
239 file_path = self._path
240 for file_extension, file_encoding in ENCODING_EXTENSIONS.items():
241 if file_encoding not in accept_encoding:
242 continue
243
244 compressed_path = file_path.with_suffix(file_path.suffix + file_extension)
245 with suppress(OSError):
246 # Do not follow symlinks and ignore any non-regular files.
247 st = compressed_path.lstat()
248 if S_ISREG(st.st_mode):
249 return compressed_path, st, file_encoding
250
251 # Fallback to the uncompressed file
252 st = file_path.stat()
253 return file_path if S_ISREG(st.st_mode) else None, st, None
254
255 async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]:
256 loop = asyncio.get_running_loop()
257 # Encoding comparisons should be case-insensitive
258 # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1
259 accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower()
260 try:
261 response_result, fobj, st, file_encoding = await loop.run_in_executor(
262 None, self._make_response, request, accept_encoding
263 )
264 except PermissionError:
265 self.set_status(HTTPForbidden.status_code)
266 return await super().prepare(request)
267 except OSError:
268 # Most likely to be FileNotFoundError or OSError for circular
269 # symlinks in python >= 3.13, so respond with 404.
270 self.set_status(HTTPNotFound.status_code)
271 return await super().prepare(request)
272
273 # Forbid special files like sockets, pipes, devices, etc.
274 if response_result is _FileResponseResult.NOT_ACCEPTABLE:
275 self.set_status(HTTPForbidden.status_code)
276 return await super().prepare(request)
277
278 if response_result is _FileResponseResult.PRE_CONDITION_FAILED:
279 return await self._precondition_failed(request)
280
281 if response_result is _FileResponseResult.NOT_MODIFIED:
282 etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}"
283 last_modified = st.st_mtime
284 return await self._not_modified(request, etag_value, last_modified)
285
286 assert fobj is not None
287 try:
288 return await self._prepare_open_file(request, fobj, st, file_encoding)
289 finally:
290 # We do not await here because we do not want to wait
291 # for the executor to finish before returning the response
292 # so the connection can begin servicing another request
293 # as soon as possible.
294 close_future = loop.run_in_executor(None, fobj.close)
295 # Hold a strong reference to the future to prevent it from being
296 # garbage collected before it completes.
297 _CLOSE_FUTURES.add(close_future)
298 close_future.add_done_callback(_CLOSE_FUTURES.remove)
299
300 async def _prepare_open_file(
301 self,
302 request: "BaseRequest",
303 fobj: io.BufferedReader,
304 st: os.stat_result,
305 file_encoding: Optional[str],
306 ) -> Optional[AbstractStreamWriter]:
307 status = self._status
308 file_size: int = st.st_size
309 file_mtime: float = st.st_mtime
310 count: int = file_size
311 start: Optional[int] = None
312
313 if (ifrange := request.if_range) is None or file_mtime <= ifrange.timestamp():
314 # If-Range header check:
315 # condition = cached date >= last modification date
316 # return 206 if True else 200.
317 # if False:
318 # Range header would not be processed, return 200
319 # if True but Range header missing
320 # return 200
321 try:
322 rng = request.http_range
323 start = rng.start
324 end: Optional[int] = rng.stop
325 except ValueError:
326 # https://tools.ietf.org/html/rfc7233:
327 # A server generating a 416 (Range Not Satisfiable) response to
328 # a byte-range request SHOULD send a Content-Range header field
329 # with an unsatisfied-range value.
330 # The complete-length in a 416 response indicates the current
331 # length of the selected representation.
332 #
333 # Will do the same below. Many servers ignore this and do not
334 # send a Content-Range header with HTTP 416
335 self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}"
336 self.set_status(HTTPRequestRangeNotSatisfiable.status_code)
337 return await super().prepare(request)
338
339 # If a range request has been made, convert start, end slice
340 # notation into file pointer offset and count
341 if start is not None:
342 if start < 0 and end is None: # return tail of file
343 start += file_size
344 if start < 0:
345 # if Range:bytes=-1000 in request header but file size
346 # is only 200, there would be trouble without this
347 start = 0
348 count = file_size - start
349 else:
350 # rfc7233:If the last-byte-pos value is
351 # absent, or if the value is greater than or equal to
352 # the current length of the representation data,
353 # the byte range is interpreted as the remainder
354 # of the representation (i.e., the server replaces the
355 # value of last-byte-pos with a value that is one less than
356 # the current length of the selected representation).
357 count = (
358 min(end if end is not None else file_size, file_size) - start
359 )
360
361 if start >= file_size:
362 # HTTP 416 should be returned in this case.
363 #
364 # According to https://tools.ietf.org/html/rfc7233:
365 # If a valid byte-range-set includes at least one
366 # byte-range-spec with a first-byte-pos that is less than
367 # the current length of the representation, or at least one
368 # suffix-byte-range-spec with a non-zero suffix-length,
369 # then the byte-range-set is satisfiable. Otherwise, the
370 # byte-range-set is unsatisfiable.
371 self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}"
372 self.set_status(HTTPRequestRangeNotSatisfiable.status_code)
373 return await super().prepare(request)
374
375 status = HTTPPartialContent.status_code
376 # Even though you are sending the whole file, you should still
377 # return a HTTP 206 for a Range request.
378 self.set_status(status)
379
380 # If the Content-Type header is not already set, guess it based on the
381 # extension of the request path. The encoding returned by guess_type
382 # can be ignored since the map was cleared above.
383 if hdrs.CONTENT_TYPE not in self._headers:
384 if sys.version_info >= (3, 13):
385 guesser = CONTENT_TYPES.guess_file_type
386 else:
387 guesser = CONTENT_TYPES.guess_type
388 self.content_type = guesser(self._path)[0] or FALLBACK_CONTENT_TYPE
389
390 if file_encoding:
391 self._headers[hdrs.CONTENT_ENCODING] = file_encoding
392 self._headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING
393 # Disable compression if we are already sending
394 # a compressed file since we don't want to double
395 # compress.
396 self._compression = False
397
398 self.etag = f"{st.st_mtime_ns:x}-{st.st_size:x}"
399 self.last_modified = file_mtime
400 self.content_length = count
401
402 self._headers[hdrs.ACCEPT_RANGES] = "bytes"
403
404 if status == HTTPPartialContent.status_code:
405 real_start = start
406 assert real_start is not None
407 self._headers[hdrs.CONTENT_RANGE] = "bytes {}-{}/{}".format(
408 real_start, real_start + count - 1, file_size
409 )
410
411 # If we are sending 0 bytes calling sendfile() will throw a ValueError
412 if count == 0 or must_be_empty_body(request.method, status):
413 return await super().prepare(request)
414
415 # be aware that start could be None or int=0 here.
416 offset = start or 0
417
418 return await self._sendfile(request, fobj, offset, count)
419