codekingpro/portable-devtools

1#ifndef Py_INTERNAL_UNICODEOBJECT_H2#define Py_INTERNAL_UNICODEOBJECT_H3#ifdef __cplusplus4extern "C" {5#endif6 7#ifndef Py_BUILD_CORE8#  error "this header requires Py_BUILD_CORE define"9#endif10 11#include "pycore_fileutils.h"     // _Py_error_handler12#include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI13 14/* --- Characters Type APIs ----------------------------------------------- */15 16extern int _PyUnicode_IsXidStart(Py_UCS4 ch);17extern int _PyUnicode_IsXidContinue(Py_UCS4 ch);18extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res);19extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res);20extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res);21extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res);22extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch);23extern int _PyUnicode_IsCased(Py_UCS4 ch);24 25/* --- Unicode API -------------------------------------------------------- */26 27// Export for '_json' shared extension28PyAPI_FUNC(int) _PyUnicode_CheckConsistency(29    PyObject *op,30    int check_content);31 32PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op);33extern Py_ssize_t _PyUnicode_InternedSize(void);34extern Py_ssize_t _PyUnicode_InternedSize_Immortal(void);35 36// Get a copy of a Unicode string.37// Export for '_datetime' shared extension.38PyAPI_FUNC(PyObject*) _PyUnicode_Copy(39    PyObject *unicode);40 41/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash42   if parameters are invalid (e.g. if length is longer than the string). */43extern void _PyUnicode_FastFill(44    PyObject *unicode,45    Py_ssize_t start,46    Py_ssize_t length,47    Py_UCS4 fill_char48    );49 50/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so51   may crash if parameters are invalid (e.g. if the output string52   is too short). */53extern void _PyUnicode_FastCopyCharacters(54    PyObject *to,55    Py_ssize_t to_start,56    PyObject *from,57    Py_ssize_t from_start,58    Py_ssize_t how_many59    );60 61/* Create a new string from a buffer of ASCII characters.62   WARNING: Don't check if the string contains any non-ASCII character. */63extern PyObject* _PyUnicode_FromASCII(64    const char *buffer,65    Py_ssize_t size);66 67/* Compute the maximum character of the substring unicode[start:end].68   Return 127 for an empty string. */69extern Py_UCS4 _PyUnicode_FindMaxChar (70    PyObject *unicode,71    Py_ssize_t start,72    Py_ssize_t end);73 74/* --- _PyUnicodeWriter API ----------------------------------------------- */75 76/* Format the object based on the format_spec, as defined in PEP 310177   (Advanced String Formatting). */78extern int _PyUnicode_FormatAdvancedWriter(79    _PyUnicodeWriter *writer,80    PyObject *obj,81    PyObject *format_spec,82    Py_ssize_t start,83    Py_ssize_t end);84 85/* --- UTF-7 Codecs ------------------------------------------------------- */86 87extern PyObject* _PyUnicode_EncodeUTF7(88    PyObject *unicode,          /* Unicode object */89    int base64SetO,             /* Encode RFC2152 Set O characters in base64 */90    int base64WhiteSpace,       /* Encode whitespace (sp, ht, nl, cr) in base64 */91    const char *errors);        /* error handling */92 93/* --- UTF-8 Codecs ------------------------------------------------------- */94 95// Export for '_tkinter' shared extension.96PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String(97    PyObject *unicode,98    const char *errors);99 100/* --- UTF-32 Codecs ------------------------------------------------------ */101 102// Export for '_tkinter' shared extension103PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(104    PyObject *object,           /* Unicode object */105    const char *errors,         /* error handling */106    int byteorder);             /* byteorder to use 0=BOM+native;-1=LE,1=BE */107 108/* --- UTF-16 Codecs ------------------------------------------------------ */109 110// Returns a Python string object holding the UTF-16 encoded value of111// the Unicode data.112//113// If byteorder is not 0, output is written according to the following114// byte order:115//116// byteorder == -1: little endian117// byteorder == 0:  native byte order (writes a BOM mark)118// byteorder == 1:  big endian119//120// If byteorder is 0, the output string will always start with the121// Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is122// prepended.123//124// Export for '_tkinter' shared extension125PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16(126    PyObject* unicode,          /* Unicode object */127    const char *errors,         /* error handling */128    int byteorder);             /* byteorder to use 0=BOM+native;-1=LE,1=BE */129 130/* --- Unicode-Escape Codecs ---------------------------------------------- */131 132/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */133extern PyObject* _PyUnicode_DecodeUnicodeEscapeStateful(134    const char *string,     /* Unicode-Escape encoded string */135    Py_ssize_t length,      /* size of string */136    const char *errors,     /* error handling */137    Py_ssize_t *consumed);  /* bytes consumed */138 139// Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape140// chars.141// Export for test_peg_generator.142PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal2(143    const char *string,     /* Unicode-Escape encoded string */144    Py_ssize_t length,      /* size of string */145    const char *errors,     /* error handling */146    Py_ssize_t *consumed,   /* bytes consumed */147    int *first_invalid_escape_char, /* on return, if not -1, contain the first148                                       invalid escaped char (<= 0xff) or invalid149                                       octal escape (> 0xff) in string. */150    const char **first_invalid_escape_ptr); /* on return, if not NULL, may151                                        point to the first invalid escaped152                                        char in string.153                                        May be NULL if errors is not NULL. */154 155/* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */156 157/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */158extern PyObject* _PyUnicode_DecodeRawUnicodeEscapeStateful(159    const char *string,     /* Unicode-Escape encoded string */160    Py_ssize_t length,      /* size of string */161    const char *errors,     /* error handling */162    Py_ssize_t *consumed);  /* bytes consumed */163 164/* --- Latin-1 Codecs ----------------------------------------------------- */165 166extern PyObject* _PyUnicode_AsLatin1String(167    PyObject* unicode,168    const char* errors);169 170/* --- ASCII Codecs ------------------------------------------------------- */171 172extern PyObject* _PyUnicode_AsASCIIString(173    PyObject* unicode,174    const char* errors);175 176/* --- Character Map Codecs ----------------------------------------------- */177 178/* Translate an Unicode object by applying a character mapping table to179   it and return the resulting Unicode object.180 181   The mapping table must map Unicode ordinal integers to Unicode strings,182   Unicode ordinal integers or None (causing deletion of the character).183 184   Mapping tables may be dictionaries or sequences. Unmapped character185   ordinals (ones which cause a LookupError) are left untouched and186   are copied as-is.187*/188extern PyObject* _PyUnicode_EncodeCharmap(189    PyObject *unicode,          /* Unicode object */190    PyObject *mapping,          /* encoding mapping */191    const char *errors);        /* error handling */192 193/* --- Decimal Encoder ---------------------------------------------------- */194 195// Converts a Unicode object holding a decimal value to an ASCII string196// for using in int, float and complex parsers.197// Transforms code points that have decimal digit property to the198// corresponding ASCII digit code points.  Transforms spaces to ASCII.199// Transforms code points starting from the first non-ASCII code point that200// is neither a decimal digit nor a space to the end into '?'.201//202// Export for '_testinternalcapi' shared extension.203PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(204    PyObject *unicode);         /* Unicode object */205 206/* --- Methods & Slots ---------------------------------------------------- */207 208PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray(209    PyObject *separator,210    PyObject *const *items,211    Py_ssize_t seqlen212    );213 214/* Test whether a unicode is equal to ASCII identifier.  Return 1 if true,215   0 otherwise.  The right argument must be ASCII identifier.216   Any error occurs inside will be cleared before return. */217extern int _PyUnicode_EqualToASCIIId(218    PyObject *left,             /* Left string */219    _Py_Identifier *right       /* Right identifier */220    );221 222// Test whether a unicode is equal to ASCII string.  Return 1 if true,223// 0 otherwise.  The right argument must be ASCII-encoded string.224// Any error occurs inside will be cleared before return.225// Export for '_ctypes' shared extension226PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString(227    PyObject *left,228    const char *right           /* ASCII-encoded string */229    );230 231/* Externally visible for str.strip(unicode) */232extern PyObject* _PyUnicode_XStrip(233    PyObject *self,234    int striptype,235    PyObject *sepobj236    );237 238 239/* Using explicit passed-in values, insert the thousands grouping240   into the string pointed to by buffer.  For the argument descriptions,241   see Objects/stringlib/localeutil.h */242extern Py_ssize_t _PyUnicode_InsertThousandsGrouping(243    _PyUnicodeWriter *writer,244    Py_ssize_t n_buffer,245    PyObject *digits,246    Py_ssize_t d_pos,247    Py_ssize_t n_digits,248    Py_ssize_t min_width,249    const char *grouping,250    PyObject *thousands_sep,251    Py_UCS4 *maxchar,252    int forward);253 254/* Dedent a string.255   Behaviour is expected to be an exact match of `textwrap.dedent`.256   Return a new reference on success, NULL with exception set on error.257   */258extern PyObject* _PyUnicode_Dedent(PyObject *unicode);259 260/* --- Misc functions ----------------------------------------------------- */261 262extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int);263 264// Fast equality check when the inputs are known to be exact unicode types.265// Export for '_pickle' shared extension.266PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *);267 268extern int _PyUnicode_WideCharString_Converter(PyObject *, void *);269extern int _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *);270 271// Export for test_peg_generator272PyAPI_FUNC(Py_ssize_t) _PyUnicode_ScanIdentifier(PyObject *);273 274/* --- Runtime lifecycle -------------------------------------------------- */275 276extern void _PyUnicode_InitState(PyInterpreterState *);277extern PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *);278extern PyStatus _PyUnicode_InitTypes(PyInterpreterState *);279extern void _PyUnicode_Fini(PyInterpreterState *);280extern void _PyUnicode_FiniTypes(PyInterpreterState *);281 282extern PyTypeObject _PyUnicodeASCIIIter_Type;283 284/* --- Interning ---------------------------------------------------------- */285 286// All these are "ref-neutral", like the public PyUnicode_InternInPlace.287 288// Explicit interning routines:289PyAPI_FUNC(void) _PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **);290PyAPI_FUNC(void) _PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **);291// Left here to help backporting:292PyAPI_FUNC(void) _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p);293// Only for singletons in the _PyRuntime struct:294extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **);295 296/* --- Other API ---------------------------------------------------------- */297 298extern void _PyUnicode_ClearInterned(PyInterpreterState *interp);299 300// Like PyUnicode_AsUTF8(), but check for embedded null characters.301// Export for '_sqlite3' shared extension.302PyAPI_FUNC(const char *) _PyUnicode_AsUTF8NoNUL(PyObject *);303 304 305#ifdef __cplusplus306}307#endif308#endif /* !Py_INTERNAL_UNICODEOBJECT_H */309