codekingpro/portable-devtools
114k
1"""2Python implementation of the io module.3"""4 5import os6import abc7import codecs8import errno9import stat10import sys11# Import _thread instead of threading to reduce startup cost12from _thread import allocate_lock as Lock13if sys.platform in {'win32', 'cygwin'}:14 from msvcrt import setmode as _setmode15else:16 _setmode = None17 18import io19from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END, Reader, Writer) # noqa: F40120 21valid_seek_flags = {0, 1, 2} # Hardwired values22if hasattr(os, 'SEEK_HOLE') :23 valid_seek_flags.add(os.SEEK_HOLE)24 valid_seek_flags.add(os.SEEK_DATA)25 26# open() uses max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)27# when the device block size is available.28DEFAULT_BUFFER_SIZE = 128 * 1024 # bytes29 30# NOTE: Base classes defined here are registered with the "official" ABCs31# defined in io.py. We don't use real inheritance though, because we don't want32# to inherit the C implementations.33 34# Rebind for compatibility35BlockingIOError = BlockingIOError36 37# Does open() check its 'errors' argument?38_CHECK_ERRORS = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)39 40 41def text_encoding(encoding, stacklevel=2):42 """43 A helper function to choose the text encoding.44 45 When encoding is not None, this function returns it.46 Otherwise, this function returns the default text encoding47 (i.e. "locale" or "utf-8" depends on UTF-8 mode).48 49 This function emits an EncodingWarning if *encoding* is None and50 sys.flags.warn_default_encoding is true.51 52 This can be used in APIs with an encoding=None parameter53 that pass it to TextIOWrapper or open.54 However, please consider using encoding="utf-8" for new APIs.55 """56 if encoding is None:57 if sys.flags.utf8_mode:58 encoding = "utf-8"59 else:60 encoding = "locale"61 if sys.flags.warn_default_encoding:62 import warnings63 warnings.warn("'encoding' argument not specified.",64 EncodingWarning, stacklevel + 1)65 return encoding66 67 68# Wrapper for builtins.open69#70# Trick so that open() won't become a bound method when stored71# as a class variable (as dbm.dumb does).72#73# See init_set_builtins_open() in Python/pylifecycle.c.74@staticmethod75def open(file, mode="r", buffering=-1, encoding=None, errors=None,76 newline=None, closefd=True, opener=None):77 78 r"""Open file and return a stream. Raise OSError upon failure.79 80 file is either a text or byte string giving the name (and the path81 if the file isn't in the current working directory) of the file to82 be opened or an integer file descriptor of the file to be83 wrapped. (If a file descriptor is given, it is closed when the84 returned I/O object is closed, unless closefd is set to False.)85 86 mode is an optional string that specifies the mode in which the file is87 opened. It defaults to 'r' which means open for reading in text mode. Other88 common values are 'w' for writing (truncating the file if it already89 exists), 'x' for exclusive creation of a new file, and 'a' for appending90 (which on some Unix systems, means that all writes append to the end of the91 file regardless of the current seek position). In text mode, if encoding is92 not specified the encoding used is platform dependent. (For reading and93 writing raw bytes use binary mode and leave encoding unspecified.) The94 available modes are:95 96 ========= ===============================================================97 Character Meaning98 --------- ---------------------------------------------------------------99 'r' open for reading (default)100 'w' open for writing, truncating the file first101 'x' create a new file and open it for writing102 'a' open for writing, appending to the end of the file if it exists103 'b' binary mode104 't' text mode (default)105 '+' open a disk file for updating (reading and writing)106 ========= ===============================================================107 108 The default mode is 'rt' (open for reading text). For binary random109 access, the mode 'w+b' opens and truncates the file to 0 bytes, while110 'r+b' opens the file without truncation. The 'x' mode implies 'w' and111 raises an `FileExistsError` if the file already exists.112 113 Python distinguishes between files opened in binary and text modes,114 even when the underlying operating system doesn't. Files opened in115 binary mode (appending 'b' to the mode argument) return contents as116 bytes objects without any decoding. In text mode (the default, or when117 't' is appended to the mode argument), the contents of the file are118 returned as strings, the bytes having been first decoded using a119 platform-dependent encoding or using the specified encoding if given.120 121 buffering is an optional integer used to set the buffering policy.122 Pass 0 to switch buffering off (only allowed in binary mode), 1 to select123 line buffering (only usable in text mode), and an integer > 1 to indicate124 the size of a fixed-size chunk buffer. When no buffering argument is125 given, the default buffering policy works as follows:126 127 * Binary files are buffered in fixed-size chunks; the size of the buffer128 is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)129 when the device block size is available.130 On most systems, the buffer will typically be 128 kilobytes long.131 132 * "Interactive" text files (files for which isatty() returns True)133 use line buffering. Other text files use the policy described above134 for binary files.135 136 encoding is the str name of the encoding used to decode or encode the137 file. This should only be used in text mode. The default encoding is138 platform dependent, but any encoding supported by Python can be139 passed. See the codecs module for the list of supported encodings.140 141 errors is an optional string that specifies how encoding errors are to142 be handled---this argument should not be used in binary mode. Pass143 'strict' to raise a ValueError exception if there is an encoding error144 (the default of None has the same effect), or pass 'ignore' to ignore145 errors. (Note that ignoring encoding errors can lead to data loss.)146 See the documentation for codecs.register for a list of the permitted147 encoding error strings.148 149 newline is a string controlling how universal newlines works (it only150 applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works151 as follows:152 153 * On input, if newline is None, universal newlines mode is154 enabled. Lines in the input can end in '\n', '\r', or '\r\n', and155 these are translated into '\n' before being returned to the156 caller. If it is '', universal newline mode is enabled, but line157 endings are returned to the caller untranslated. If it has any of158 the other legal values, input lines are only terminated by the given159 string, and the line ending is returned to the caller untranslated.160 161 * On output, if newline is None, any '\n' characters written are162 translated to the system default line separator, os.linesep. If163 newline is '', no translation takes place. If newline is any of the164 other legal values, any '\n' characters written are translated to165 the given string.166 167 closedfd is a bool. If closefd is False, the underlying file descriptor will168 be kept open when the file is closed. This does not work when a file name is169 given and must be True in that case.170 171 The newly created file is non-inheritable.172 173 A custom opener can be used by passing a callable as *opener*. The174 underlying file descriptor for the file object is then obtained by calling175 *opener* with (*file*, *flags*). *opener* must return an open file176 descriptor (passing os.open as *opener* results in functionality similar to177 passing None).178 179 open() returns a file object whose type depends on the mode, and180 through which the standard file operations such as reading and writing181 are performed. When open() is used to open a file in a text mode ('w',182 'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open183 a file in a binary mode, the returned class varies: in read binary184 mode, it returns a BufferedReader; in write binary and append binary185 modes, it returns a BufferedWriter, and in read/write mode, it returns186 a BufferedRandom.187 188 It is also possible to use a string or bytearray as a file for both189 reading and writing. For strings StringIO can be used like a file190 opened in a text mode, and for bytes a BytesIO can be used like a file191 opened in a binary mode.192 """193 if not isinstance(file, int):194 file = os.fspath(file)195 if not isinstance(file, (str, bytes, int)):196 raise TypeError("invalid file: %r" % file)197 if not isinstance(mode, str):198 raise TypeError("invalid mode: %r" % mode)199 if not isinstance(buffering, int):200 raise TypeError("invalid buffering: %r" % buffering)201 if encoding is not None and not isinstance(encoding, str):202 raise TypeError("invalid encoding: %r" % encoding)203 if errors is not None and not isinstance(errors, str):204 raise TypeError("invalid errors: %r" % errors)205 modes = set(mode)206 if modes - set("axrwb+t") or len(mode) > len(modes):207 raise ValueError("invalid mode: %r" % mode)208 creating = "x" in modes209 reading = "r" in modes210 writing = "w" in modes211 appending = "a" in modes212 updating = "+" in modes213 text = "t" in modes214 binary = "b" in modes215 if text and binary:216 raise ValueError("can't have text and binary mode at once")217 if creating + reading + writing + appending > 1:218 raise ValueError("can't have read/write/append mode at once")219 if not (creating or reading or writing or appending):220 raise ValueError("must have exactly one of read/write/append mode")221 if binary and encoding is not None:222 raise ValueError("binary mode doesn't take an encoding argument")223 if binary and errors is not None:224 raise ValueError("binary mode doesn't take an errors argument")225 if binary and newline is not None:226 raise ValueError("binary mode doesn't take a newline argument")227 if binary and buffering == 1:228 import warnings229 warnings.warn("line buffering (buffering=1) isn't supported in binary "230 "mode, the default buffer size will be used",231 RuntimeWarning, 2)232 raw = FileIO(file,233 (creating and "x" or "") +234 (reading and "r" or "") +235 (writing and "w" or "") +236 (appending and "a" or "") +237 (updating and "+" or ""),238 closefd, opener=opener)239 result = raw240 try:241 line_buffering = False242 if buffering == 1 or buffering < 0 and raw._isatty_open_only():243 buffering = -1244 line_buffering = True245 if buffering < 0:246 buffering = max(min(raw._blksize, 8192 * 1024), DEFAULT_BUFFER_SIZE)247 if buffering < 0:248 raise ValueError("invalid buffering size")249 if buffering == 0:250 if binary:251 return result252 raise ValueError("can't have unbuffered text I/O")253 if updating:254 buffer = BufferedRandom(raw, buffering)255 elif creating or writing or appending:256 buffer = BufferedWriter(raw, buffering)257 elif reading:258 buffer = BufferedReader(raw, buffering)259 else:260 raise ValueError("unknown mode: %r" % mode)261 result = buffer262 if binary:263 return result264 encoding = text_encoding(encoding)265 text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)266 result = text267 text.mode = mode268 return result269 except:270 result.close()271 raise272 273# Define a default pure-Python implementation for open_code()274# that does not allow hooks. Warn on first use. Defined for tests.275def _open_code_with_warning(path):276 """Opens the provided file with mode ``'rb'``. This function277 should be used when the intent is to treat the contents as278 executable code.279 280 ``path`` should be an absolute path.281 282 When supported by the runtime, this function can be hooked283 in order to allow embedders more control over code files.284 This functionality is not supported on the current runtime.285 """286 import warnings287 warnings.warn("_pyio.open_code() may not be using hooks",288 RuntimeWarning, 2)289 return open(path, "rb")290 291try:292 open_code = io.open_code293except AttributeError:294 open_code = _open_code_with_warning295 296 297# In normal operation, both `UnsupportedOperation`s should be bound to the298# same object.299try:300 UnsupportedOperation = io.UnsupportedOperation301except AttributeError:302 class UnsupportedOperation(OSError, ValueError):303 pass304 305 306class IOBase(metaclass=abc.ABCMeta):307 308 """The abstract base class for all I/O classes.309 310 This class provides dummy implementations for many methods that311 derived classes can override selectively; the default implementations312 represent a file that cannot be read, written or seeked.313 314 Even though IOBase does not declare read or write because315 their signatures will vary, implementations and clients should316 consider those methods part of the interface. Also, implementations317 may raise UnsupportedOperation when operations they do not support are318 called.319 320 The basic type used for binary data read from or written to a file is321 bytes. Other bytes-like objects are accepted as method arguments too.322 Text I/O classes work with str data.323 324 Note that calling any method (even inquiries) on a closed stream is325 undefined. Implementations may raise OSError in this case.326 327 IOBase (and its subclasses) support the iterator protocol, meaning328 that an IOBase object can be iterated over yielding the lines in a329 stream.330 331 IOBase also supports the :keyword:`with` statement. In this example,332 fp is closed after the suite of the with statement is complete:333 334 with open('spam.txt', 'r') as fp:335 fp.write('Spam and eggs!')336 """337 338 ### Internal ###339 340 def _unsupported(self, name):341 """Internal: raise an OSError exception for unsupported operations."""342 raise UnsupportedOperation("%s.%s() not supported" %343 (self.__class__.__name__, name))344 345 ### Positioning ###346 347 def seek(self, pos, whence=0):348 """Change stream position.349 350 Change the stream position to byte offset pos. Argument pos is351 interpreted relative to the position indicated by whence. Values352 for whence are ints:353 354 * 0 -- start of stream (the default); offset should be zero or positive355 * 1 -- current stream position; offset may be negative356 * 2 -- end of stream; offset is usually negative357 Some operating systems / file systems could provide additional values.358 359 Return an int indicating the new absolute position.360 """361 self._unsupported("seek")362 363 def tell(self):364 """Return an int indicating the current stream position."""365 return self.seek(0, 1)366 367 def truncate(self, pos=None):368 """Truncate file to size bytes.369 370 Size defaults to the current IO position as reported by tell(). Return371 the new size.372 """373 self._unsupported("truncate")374 375 ### Flush and close ###376 377 def flush(self):378 """Flush write buffers, if applicable.379 380 This is not implemented for read-only and non-blocking streams.381 """382 self._checkClosed()383 # XXX Should this return the number of bytes written???384 385 __closed = False386 387 def close(self):388 """Flush and close the IO object.389 390 This method has no effect if the file is already closed.391 """392 if not self.__closed:393 try:394 self.flush()395 finally:396 self.__closed = True397 398 def __del__(self):399 """Destructor. Calls close()."""400 try:401 closed = self.closed402 except AttributeError:403 # If getting closed fails, then the object is probably404 # in an unusable state, so ignore.405 return406 407 if closed:408 return409 410 if dealloc_warn := getattr(self, "_dealloc_warn", None):411 dealloc_warn(self)412 413 # If close() fails, the caller logs the exception with414 # sys.unraisablehook. close() must be called at the end at __del__().415 self.close()416 417 ### Inquiries ###418 419 def seekable(self):420 """Return a bool indicating whether object supports random access.421 422 If False, seek(), tell() and truncate() will raise OSError.423 This method may need to do a test seek().424 """425 return False426 427 def _checkSeekable(self, msg=None):428 """Internal: raise UnsupportedOperation if file is not seekable429 """430 if not self.seekable():431 raise UnsupportedOperation("File or stream is not seekable."432 if msg is None else msg)433 434 def readable(self):435 """Return a bool indicating whether object was opened for reading.436 437 If False, read() will raise OSError.438 """439 return False440 441 def _checkReadable(self, msg=None):442 """Internal: raise UnsupportedOperation if file is not readable443 """444 if not self.readable():445 raise UnsupportedOperation("File or stream is not readable."446 if msg is None else msg)447 448 def writable(self):449 """Return a bool indicating whether object was opened for writing.450 451 If False, write() and truncate() will raise OSError.452 """453 return False454 455 def _checkWritable(self, msg=None):456 """Internal: raise UnsupportedOperation if file is not writable457 """458 if not self.writable():459 raise UnsupportedOperation("File or stream is not writable."460 if msg is None else msg)461 462 @property463 def closed(self):464 """closed: bool. True iff the file has been closed.465 466 For backwards compatibility, this is a property, not a predicate.467 """468 return self.__closed469 470 def _checkClosed(self, msg=None):471 """Internal: raise a ValueError if file is closed472 """473 if self.closed:474 raise ValueError("I/O operation on closed file."475 if msg is None else msg)476 477 ### Context manager ###478 479 def __enter__(self): # That's a forward reference480 """Context management protocol. Returns self (an instance of IOBase)."""481 self._checkClosed()482 return self483 484 def __exit__(self, *args):485 """Context management protocol. Calls close()"""486 self.close()487 488 ### Lower-level APIs ###489 490 # XXX Should these be present even if unimplemented?491 492 def fileno(self):493 """Returns underlying file descriptor (an int) if one exists.494 495 An OSError is raised if the IO object does not use a file descriptor.496 """497 self._unsupported("fileno")498 499 def isatty(self):500 """Return a bool indicating whether this is an 'interactive' stream.501 502 Return False if it can't be determined.503 """504 self._checkClosed()505 return False506 507 ### Readline[s] and writelines ###508 509 def readline(self, size=-1):510 r"""Read and return a line of bytes from the stream.511 512 If size is specified, at most size bytes will be read.513 Size should be an int.514 515 The line terminator is always b'\n' for binary files; for text516 files, the newlines argument to open can be used to select the line517 terminator(s) recognized.518 """519 # For backwards compatibility, a (slowish) readline().520 if hasattr(self, "peek"):521 def nreadahead():522 readahead = self.peek(1)523 if not readahead:524 return 1525 n = (readahead.find(b"\n") + 1) or len(readahead)526 if size >= 0:527 n = min(n, size)528 return n529 else:530 def nreadahead():531 return 1532 if size is None:533 size = -1534 else:535 try:536 size_index = size.__index__537 except AttributeError:538 raise TypeError(f"{size!r} is not an integer")539 else:540 size = size_index()541 res = bytearray()542 while size < 0 or len(res) < size:543 b = self.read(nreadahead())544 if not b:545 break546 res += b547 if res.endswith(b"\n"):548 break549 return bytes(res)550 551 def __iter__(self):552 self._checkClosed()553 return self554 555 def __next__(self):556 line = self.readline()557 if not line:558 raise StopIteration559 return line560 561 def readlines(self, hint=None):562 """Return a list of lines from the stream.563 564 hint can be specified to control the number of lines read: no more565 lines will be read if the total size (in bytes/characters) of all566 lines so far exceeds hint.567 """568 if hint is None or hint <= 0:569 return list(self)570 n = 0571 lines = []572 for line in self:573 lines.append(line)574 n += len(line)575 if n >= hint:576 break577 return lines578 579 def writelines(self, lines):580 """Write a list of lines to the stream.581 582 Line separators are not added, so it is usual for each of the lines583 provided to have a line separator at the end.584 """585 self._checkClosed()586 for line in lines:587 self.write(line)588 589io.IOBase.register(IOBase)590 591 592class RawIOBase(IOBase):593 594 """Base class for raw binary I/O."""595 596 # The read() method is implemented by calling readinto(); derived597 # classes that want to support read() only need to implement598 # readinto() as a primitive operation. In general, readinto() can be599 # more efficient than read().600 601 # (It would be tempting to also provide an implementation of602 # readinto() in terms of read(), in case the latter is a more suitable603 # primitive operation, but that would lead to nasty recursion in case604 # a subclass doesn't implement either.)605 606 def read(self, size=-1):607 """Read and return up to size bytes, where size is an int.608 609 Returns an empty bytes object on EOF, or None if the object is610 set not to block and has no data to read.611 """612 if size is None:613 size = -1614 if size < 0:615 return self.readall()616 b = bytearray(size.__index__())617 n = self.readinto(b)618 if n is None:619 return None620 if n < 0 or n > len(b):621 raise ValueError(f"readinto returned {n} outside buffer size {len(b)}")622 del b[n:]623 return bytes(b)624 625 def readall(self):626 """Read until EOF, using multiple read() call."""627 res = bytearray()628 while data := self.read(DEFAULT_BUFFER_SIZE):629 res += data630 if res:631 return bytes(res)632 else:633 # b'' or None634 return data635 636 def readinto(self, b):637 """Read bytes into a pre-allocated bytes-like object b.638 639 Returns an int representing the number of bytes read (0 for EOF), or640 None if the object is set not to block and has no data to read.641 """642 self._unsupported("readinto")643 644 def write(self, b):645 """Write the given buffer to the IO stream.646 647 Returns the number of bytes written, which may be less than the648 length of b in bytes.649 """650 self._unsupported("write")651 652io.RawIOBase.register(RawIOBase)653 654 655class BufferedIOBase(IOBase):656 657 """Base class for buffered IO objects.658 659 The main difference with RawIOBase is that the read() method660 supports omitting the size argument, and does not have a default661 implementation that defers to readinto().662 663 In addition, read(), readinto() and write() may raise664 BlockingIOError if the underlying raw stream is in non-blocking665 mode and not ready; unlike their raw counterparts, they will never666 return None.667 668 A typical implementation should not inherit from a RawIOBase669 implementation, but wrap one.670 """671 672 def read(self, size=-1):673 """Read and return up to size bytes, where size is an int.674 675 If the argument is omitted, None, or negative, reads and676 returns all data until EOF.677 678 If the argument is positive, and the underlying raw stream is679 not 'interactive', multiple raw reads may be issued to satisfy680 the byte count (unless EOF is reached first). But for681 interactive raw streams (XXX and for pipes?), at most one raw682 read will be issued, and a short result does not imply that683 EOF is imminent.684 685 Returns an empty bytes array on EOF.686 687 Raises BlockingIOError if the underlying raw stream has no688 data at the moment.689 """690 self._unsupported("read")691 692 def read1(self, size=-1):693 """Read up to size bytes with at most one read() system call,694 where size is an int.695 """696 self._unsupported("read1")697 698 def readinto(self, b):699 """Read bytes into a pre-allocated bytes-like object b.700 701 Like read(), this may issue multiple reads to the underlying raw702 stream, unless the latter is 'interactive'.703 704 Returns an int representing the number of bytes read (0 for EOF).705 706 Raises BlockingIOError if the underlying raw stream has no707 data at the moment.708 """709 710 return self._readinto(b, read1=False)711 712 def readinto1(self, b):713 """Read bytes into buffer *b*, using at most one system call714 715 Returns an int representing the number of bytes read (0 for EOF).716 717 Raises BlockingIOError if the underlying raw stream has no718 data at the moment.719 """720 721 return self._readinto(b, read1=True)722 723 def _readinto(self, b, read1):724 if not isinstance(b, memoryview):725 b = memoryview(b)726 b = b.cast('B')727 728 if read1:729 data = self.read1(len(b))730 else:731 data = self.read(len(b))732 n = len(data)733 734 b[:n] = data735 736 return n737 738 def write(self, b):739 """Write the given bytes buffer to the IO stream.740 741 Return the number of bytes written, which is always the length of b742 in bytes.743 744 Raises BlockingIOError if the buffer is full and the745 underlying raw stream cannot accept more data at the moment.746 """747 self._unsupported("write")748 749 def detach(self):750 """751 Separate the underlying raw stream from the buffer and return it.752 753 After the raw stream has been detached, the buffer is in an unusable754 state.755 """756 self._unsupported("detach")757 758io.BufferedIOBase.register(BufferedIOBase)759 760 761class _BufferedIOMixin(BufferedIOBase):762 763 """A mixin implementation of BufferedIOBase with an underlying raw stream.764 765 This passes most requests on to the underlying raw stream. It766 does *not* provide implementations of read(), readinto() or767 write().768 """769 770 def __init__(self, raw):771 self._raw = raw772 773 ### Positioning ###774 775 def seek(self, pos, whence=0):776 new_position = self.raw.seek(pos, whence)777 if new_position < 0:778 raise OSError("seek() returned an invalid position")779 return new_position780 781 def tell(self):782 pos = self.raw.tell()783 if pos < 0:784 raise OSError("tell() returned an invalid position")785 return pos786 787 def truncate(self, pos=None):788 self._checkClosed()789 self._checkWritable()790 791 # Flush the stream. We're mixing buffered I/O with lower-level I/O,792 # and a flush may be necessary to synch both views of the current793 # file state.794 self.flush()795 796 if pos is None:797 pos = self.tell()798 # XXX: Should seek() be used, instead of passing the position799 # XXX directly to truncate?800 return self.raw.truncate(pos)801 802 ### Flush and close ###803 804 def flush(self):805 if self.closed:806 raise ValueError("flush on closed file")807 self.raw.flush()808 809 def close(self):810 if self.raw is not None and not self.closed:811 try:812 # may raise BlockingIOError or BrokenPipeError etc813 self.flush()814 finally:815 self.raw.close()816 817 def detach(self):818 if self.raw is None:819 raise ValueError("raw stream already detached")820 self.flush()821 raw = self._raw822 self._raw = None823 return raw824 825 ### Inquiries ###826 827 def seekable(self):828 return self.raw.seekable()829 830 @property831 def raw(self):832 return self._raw833 834 @property835 def closed(self):836 return self.raw.closed837 838 @property839 def name(self):840 return self.raw.name841 842 @property843 def mode(self):844 return self.raw.mode845 846 def __getstate__(self):847 raise TypeError(f"cannot pickle {self.__class__.__name__!r} object")848 849 def __repr__(self):850 modname = self.__class__.__module__851 clsname = self.__class__.__qualname__852 try:853 name = self.name854 except AttributeError:855 return "<{}.{}>".format(modname, clsname)856 else:857 return "<{}.{} name={!r}>".format(modname, clsname, name)858 859 def _dealloc_warn(self, source):860 if dealloc_warn := getattr(self.raw, "_dealloc_warn", None):861 dealloc_warn(source)862 863 ### Lower-level APIs ###864 865 def fileno(self):866 return self.raw.fileno()867 868 def isatty(self):869 return self.raw.isatty()870 871 872class BytesIO(BufferedIOBase):873 874 """Buffered I/O implementation using an in-memory bytes buffer."""875 876 # Initialize _buffer as soon as possible since it's used by __del__()877 # which calls close()878 _buffer = None879 880 def __init__(self, initial_bytes=None):881 buf = bytearray()882 if initial_bytes is not None:883 buf += initial_bytes884 self._buffer = buf885 self._pos = 0886 887 def __getstate__(self):888 if self.closed:889 raise ValueError("__getstate__ on closed file")890 return self.__dict__.copy()891 892 def getvalue(self):893 """Return the bytes value (contents) of the buffer894 """895 if self.closed:896 raise ValueError("getvalue on closed file")897 return bytes(self._buffer)898 899 def getbuffer(self):900 """Return a readable and writable view of the buffer.901 """902 if self.closed:903 raise ValueError("getbuffer on closed file")904 return memoryview(self._buffer)905 906 def close(self):907 if self._buffer is not None:908 self._buffer.clear()909 super().close()910 911 def read(self, size=-1):912 if self.closed:913 raise ValueError("read from closed file")914 if size is None:915 size = -1916 else:917 try:918 size_index = size.__index__919 except AttributeError:920 raise TypeError(f"{size!r} is not an integer")921 else:922 size = size_index()923 if size < 0:924 size = len(self._buffer)925 if len(self._buffer) <= self._pos:926 return b""927 newpos = min(len(self._buffer), self._pos + size)928 b = self._buffer[self._pos : newpos]929 self._pos = newpos930 return bytes(b)931 932 def read1(self, size=-1):933 """This is the same as read.934 """935 return self.read(size)936 937 def write(self, b):938 if isinstance(b, str):939 raise TypeError("can't write str to binary stream")940 with memoryview(b) as view:941 if self.closed:942 raise ValueError("write to closed file")943 944 n = view.nbytes # Size of any bytes-like object945 if n == 0:946 return 0947 948 pos = self._pos949 if pos > len(self._buffer):950 # Pad buffer to pos with null bytes.951 self._buffer.resize(pos)952 self._buffer[pos:pos + n] = view953 self._pos += n954 return n955 956 def seek(self, pos, whence=0):957 if self.closed:958 raise ValueError("seek on closed file")959 try:960 pos_index = pos.__index__961 except AttributeError:962 raise TypeError(f"{pos!r} is not an integer")963 else:964 pos = pos_index()965 if whence == 0:966 if pos < 0:967 raise ValueError("negative seek position %r" % (pos,))968 self._pos = pos969 elif whence == 1:970 self._pos = max(0, self._pos + pos)971 elif whence == 2:972 self._pos = max(0, len(self._buffer) + pos)973 else:974 raise ValueError("unsupported whence value")975 return self._pos976 977 def tell(self):978 if self.closed:979 raise ValueError("tell on closed file")980 return self._pos981 982 def truncate(self, pos=None):983 if self.closed:984 raise ValueError("truncate on closed file")985 if pos is None:986 pos = self._pos987 else:988 try:989 pos_index = pos.__index__990 except AttributeError:991 raise TypeError(f"{pos!r} is not an integer")992 else:993 pos = pos_index()994 if pos < 0:995 raise ValueError("negative truncate position %r" % (pos,))996 del self._buffer[pos:]997 return pos998 999 def readable(self):1000 if self.closed:1001 raise ValueError("I/O operation on closed file.")1002 return True1003 1004 def writable(self):1005 if self.closed:1006 raise ValueError("I/O operation on closed file.")1007 return True1008 1009 def seekable(self):1010 if self.closed:1011 raise ValueError("I/O operation on closed file.")1012 return True1013 1014 1015class BufferedReader(_BufferedIOMixin):1016 1017 """BufferedReader(raw[, buffer_size])1018 1019 A buffer for a readable, sequential BaseRawIO object.1020 1021 The constructor creates a BufferedReader for the given readable raw1022 stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE1023 is used.1024 """1025 1026 def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):1027 """Create a new buffered reader using the given readable raw IO object.1028 """1029 if not raw.readable():1030 raise OSError('"raw" argument must be readable.')1031 1032 _BufferedIOMixin.__init__(self, raw)1033 if buffer_size <= 0:1034 raise ValueError("invalid buffer size")1035 self.buffer_size = buffer_size1036 self._reset_read_buf()1037 self._read_lock = Lock()1038 1039 def readable(self):1040 return self.raw.readable()1041 1042 def _reset_read_buf(self):1043 self._read_buf = b""1044 self._read_pos = 01045 1046 def read(self, size=None):1047 """Read size bytes.1048 1049 Returns exactly size bytes of data unless the underlying raw IO1050 stream reaches EOF or if the call would block in non-blocking1051 mode. If size is negative, read until EOF or until read() would1052 block.1053 """1054 if size is not None and size < -1:1055 raise ValueError("invalid number of bytes to read")1056 with self._read_lock:1057 return self._read_unlocked(size)1058 1059 def _read_unlocked(self, n=None):1060 nodata_val = b""1061 empty_values = (b"", None)1062 buf = self._read_buf1063 pos = self._read_pos1064 1065 # Special case for when the number of bytes to read is unspecified.1066 if n is None or n == -1:1067 self._reset_read_buf()1068 if hasattr(self.raw, 'readall'):1069 chunk = self.raw.readall()1070 if chunk is None:1071 return buf[pos:] or None1072 else:1073 return buf[pos:] + chunk1074 chunks = [buf[pos:]] # Strip the consumed bytes.1075 current_size = 01076 while True:1077 # Read until EOF or until read() would block.1078 chunk = self.raw.read()1079 if chunk in empty_values:1080 nodata_val = chunk1081 break1082 current_size += len(chunk)1083 chunks.append(chunk)1084 return b"".join(chunks) or nodata_val1085 1086 # The number of bytes to read is specified, return at most n bytes.1087 avail = len(buf) - pos # Length of the available buffered data.1088 if n <= avail:1089 # Fast path: the data to read is fully buffered.1090 self._read_pos += n1091 return buf[pos:pos+n]1092 # Slow path: read from the stream until enough bytes are read,1093 # or until an EOF occurs or until read() would block.1094 chunks = [buf[pos:]]1095 wanted = max(self.buffer_size, n)1096 while avail < n:1097 chunk = self.raw.read(wanted)1098 if chunk in empty_values:1099 nodata_val = chunk1100 break1101 avail += len(chunk)1102 chunks.append(chunk)1103 # n is more than avail only when an EOF occurred or when1104 # read() would have blocked.1105 n = min(n, avail)1106 out = b"".join(chunks)1107 self._read_buf = out[n:] # Save the extra data in the buffer.1108 self._read_pos = 01109 return out[:n] if out else nodata_val1110 1111 def peek(self, size=0):1112 """Returns buffered bytes without advancing the position.1113 1114 The argument indicates a desired minimal number of bytes; we1115 do at most one raw read to satisfy it. We never return more1116 than self.buffer_size.1117 """1118 self._checkClosed("peek of closed file")1119 with self._read_lock:1120 return self._peek_unlocked(size)1121 1122 def _peek_unlocked(self, n=0):1123 want = min(n, self.buffer_size)1124 have = len(self._read_buf) - self._read_pos1125 if have < want or have <= 0:1126 to_read = self.buffer_size - have1127 current = self.raw.read(to_read)1128 if current:1129 self._read_buf = self._read_buf[self._read_pos:] + current1130 self._read_pos = 01131 return self._read_buf[self._read_pos:]1132 1133 def read1(self, size=-1):1134 """Reads up to size bytes, with at most one read() system call."""1135 # Returns up to size bytes. If at least one byte is buffered, we1136 # only return buffered bytes. Otherwise, we do one raw read.1137 self._checkClosed("read of closed file")1138 if size < 0:1139 size = self.buffer_size1140 if size == 0:1141 return b""1142 with self._read_lock:1143 self._peek_unlocked(1)1144 return self._read_unlocked(1145 min(size, len(self._read_buf) - self._read_pos))1146 1147 # Implementing readinto() and readinto1() is not strictly necessary (we1148 # could rely on the base class that provides an implementation in terms of1149 # read() and read1()). We do it anyway to keep the _pyio implementation1150 # similar to the io implementation (which implements the methods for1151 # performance reasons).1152 def _readinto(self, buf, read1):1153 """Read data into *buf* with at most one system call."""1154 1155 self._checkClosed("readinto of closed file")1156 1157 # Need to create a memoryview object of type 'b', otherwise1158 # we may not be able to assign bytes to it, and slicing it1159 # would create a new object.1160 if not isinstance(buf, memoryview):1161 buf = memoryview(buf)1162 if buf.nbytes == 0:1163 return 01164 buf = buf.cast('B')1165 1166 written = 01167 with self._read_lock:1168 while written < len(buf):1169 1170 # First try to read from internal buffer1171 avail = min(len(self._read_buf) - self._read_pos, len(buf))1172 if avail:1173 buf[written:written+avail] = \1174 self._read_buf[self._read_pos:self._read_pos+avail]1175 self._read_pos += avail1176 written += avail1177 if written == len(buf):1178 break1179 1180 # If remaining space in callers buffer is larger than1181 # internal buffer, read directly into callers buffer1182 if len(buf) - written > self.buffer_size:1183 n = self.raw.readinto(buf[written:])1184 if not n:1185 break # eof1186 written += n1187 1188 # Otherwise refill internal buffer - unless we're1189 # in read1 mode and already got some data1190 elif not (read1 and written):1191 if not self._peek_unlocked(1):1192 break # eof1193 1194 # In readinto1 mode, return as soon as we have some data1195 if read1 and written:1196 break1197 1198 return written1199 1200 def tell(self):