codekingpro/portable-devtools

1#ifndef Py_INTERNAL_CODE_H2#define Py_INTERNAL_CODE_H3#ifdef __cplusplus4extern "C" {5#endif6 7#ifndef Py_BUILD_CORE8#  error "this header requires Py_BUILD_CORE define"9#endif10 11#include "pycore_backoff.h"     // _Py_BackoffCounter12#include "pycore_structs.h"     // _Py_CODEUNIT13#include "pycore_tstate.h"      // _PyThreadStateImpl14 15 16#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive)17#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))18 19 20/* These macros only remain defined for compatibility. */21#define _Py_OPCODE(word) ((word).op.code)22#define _Py_OPARG(word) ((word).op.arg)23 24static inline _Py_CODEUNIT25_py_make_codeunit(uint8_t opcode, uint8_t oparg)26{27    // No designated initialisers because of C++ compat28    _Py_CODEUNIT word;29    word.op.code = opcode;30    word.op.arg = oparg;31    return word;32}33 34static inline void35_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode)36{37    word->op.code = opcode;38}39 40#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg))41#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode))42 43 44// We hide some of the newer PyCodeObject fields behind macros.45// This helps with backporting certain changes to 3.12.46#define _PyCode_HAS_EXECUTORS(CODE) \47    (CODE->co_executors != NULL)48#define _PyCode_HAS_INSTRUMENTATION(CODE) \49    (CODE->_co_instrumentation_version > 0)50 51 52extern PyStatus _PyCode_Init(PyInterpreterState *interp);53extern void _PyCode_Fini(PyInterpreterState *interp);54 55 56/* PEP 65957 * Specialization and quickening structs and helper functions58 */59 60 61// Inline caches. If you change the number of cache entries for an instruction,62// you must *also* update the number of cache entries in Lib/opcode.py and bump63// the magic number in Lib/importlib/_bootstrap_external.py!64 65#define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT))66 67typedef struct {68    _Py_BackoffCounter counter;69    uint16_t module_keys_version;70    uint16_t builtin_keys_version;71    uint16_t index;72} _PyLoadGlobalCache;73 74#define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache)75 76typedef struct {77    _Py_BackoffCounter counter;78    uint16_t external_cache[4];79} _PyBinaryOpCache;80 81#define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache)82 83typedef struct {84    _Py_BackoffCounter counter;85} _PyUnpackSequenceCache;86 87#define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \88    CACHE_ENTRIES(_PyUnpackSequenceCache)89 90typedef struct {91    _Py_BackoffCounter counter;92} _PyCompareOpCache;93 94#define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache)95 96typedef struct {97    _Py_BackoffCounter counter;98} _PySuperAttrCache;99 100#define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache)101 102typedef struct {103    _Py_BackoffCounter counter;104    uint16_t version[2];105    uint16_t index;106} _PyAttrCache;107 108typedef struct {109    _Py_BackoffCounter counter;110    uint16_t type_version[2];111    union {112        uint16_t keys_version[2];113        uint16_t dict_offset;114    };115    uint16_t descr[4];116} _PyLoadMethodCache;117 118 119// MUST be the max(_PyAttrCache, _PyLoadMethodCache)120#define INLINE_CACHE_ENTRIES_LOAD_ATTR CACHE_ENTRIES(_PyLoadMethodCache)121 122#define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache)123 124typedef struct {125    _Py_BackoffCounter counter;126    uint16_t func_version[2];127} _PyCallCache;128 129#define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache)130#define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache)131 132typedef struct {133    _Py_BackoffCounter counter;134} _PyStoreSubscrCache;135 136#define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)137 138typedef struct {139    _Py_BackoffCounter counter;140} _PyForIterCache;141 142#define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)143 144typedef struct {145    _Py_BackoffCounter counter;146} _PySendCache;147 148#define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache)149 150typedef struct {151    _Py_BackoffCounter counter;152    uint16_t version[2];153} _PyToBoolCache;154 155#define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache)156 157typedef struct {158    _Py_BackoffCounter counter;159} _PyContainsOpCache;160 161#define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache)162 163/* "Locals plus" for a code object is the set of locals + cell vars +164 * free vars.  This relates to variable names as well as offsets into165 * the "fast locals" storage array of execution frames.  The compiler166 * builds the list of names, their offsets, and the corresponding167 * kind of local.168 *169 * Those kinds represent the source of the initial value and the170 * variable's scope (as related to closures).  A "local" is an171 * argument or other variable defined in the current scope.  A "free"172 * variable is one that is defined in an outer scope and comes from173 * the function's closure.  A "cell" variable is a local that escapes174 * into an inner function as part of a closure, and thus must be175 * wrapped in a cell.  Any "local" can also be a "cell", but the176 * "free" kind is mutually exclusive with both.177 */178 179// Note that these all fit within a byte, as do combinations.180#define CO_FAST_ARG_POS (0x02)  // pos-only, pos-or-kw, varargs181#define CO_FAST_ARG_KW  (0x04)  // kw-only, pos-or-kw, varkwargs182#define CO_FAST_ARG_VAR (0x08)  // varargs, varkwargs183#define CO_FAST_ARG     (CO_FAST_ARG_POS | CO_FAST_ARG_KW | CO_FAST_ARG_VAR)184#define CO_FAST_HIDDEN  (0x10)185#define CO_FAST_LOCAL   (0x20)186#define CO_FAST_CELL    (0x40)187#define CO_FAST_FREE    (0x80)188 189typedef unsigned char _PyLocals_Kind;190 191static inline _PyLocals_Kind192_PyLocals_GetKind(PyObject *kinds, int i)193{194    assert(PyBytes_Check(kinds));195    assert(0 <= i && i < PyBytes_GET_SIZE(kinds));196    char *ptr = PyBytes_AS_STRING(kinds);197    return (_PyLocals_Kind)(ptr[i]);198}199 200static inline void201_PyLocals_SetKind(PyObject *kinds, int i, _PyLocals_Kind kind)202{203    assert(PyBytes_Check(kinds));204    assert(0 <= i && i < PyBytes_GET_SIZE(kinds));205    char *ptr = PyBytes_AS_STRING(kinds);206    ptr[i] = (char) kind;207}208 209 210struct _PyCodeConstructor {211    /* metadata */212    PyObject *filename;213    PyObject *name;214    PyObject *qualname;215    int flags;216 217    /* the code */218    PyObject *code;219    int firstlineno;220    PyObject *linetable;221 222    /* used by the code */223    PyObject *consts;224    PyObject *names;225 226    /* mapping frame offsets to information */227    PyObject *localsplusnames;  // Tuple of strings228    PyObject *localspluskinds;  // Bytes object, one byte per variable229 230    /* args (within varnames) */231    int argcount;232    int posonlyargcount;233    // XXX Replace argcount with posorkwargcount (argcount - posonlyargcount).234    int kwonlyargcount;235 236    /* needed to create the frame */237    int stacksize;238 239    /* used by the eval loop */240    PyObject *exceptiontable;241};242 243// Using an "arguments struct" like this is helpful for maintainability244// in a case such as this with many parameters.  It does bear a risk:245// if the struct changes and callers are not updated properly then the246// compiler will not catch problems (like a missing argument).  This can247// cause hard-to-debug problems.  The risk is mitigated by the use of248// check_code() in codeobject.c.  However, we may decide to switch249// back to a regular function signature.  Regardless, this approach250// wouldn't be appropriate if this weren't a strictly internal API.251// (See the comments in https://github.com/python/cpython/pull/26258.)252extern int _PyCode_Validate(struct _PyCodeConstructor *);253extern PyCodeObject* _PyCode_New(struct _PyCodeConstructor *);254 255 256/* Private API */257 258/* Getters for internal PyCodeObject data. */259extern PyObject* _PyCode_GetVarnames(PyCodeObject *);260extern PyObject* _PyCode_GetCellvars(PyCodeObject *);261extern PyObject* _PyCode_GetFreevars(PyCodeObject *);262extern PyObject* _PyCode_GetCode(PyCodeObject *);263 264/** API for initializing the line number tables. */265extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);266 267/** Out of process API for initializing the location table. */268extern void _PyLineTable_InitAddressRange(269    const char *linetable,270    Py_ssize_t length,271    int firstlineno,272    PyCodeAddressRange *range);273 274/** API for traversing the line number table. */275extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range);276extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range);277 278// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid279// and can be called without an attached tstate. Used by dump_frame() in280// Python/traceback.c. The function uses heuristics to detect freed memory,281// it's not 100% reliable.282extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr);283 284 285/** API for executors */286extern void _PyCode_Clear_Executors(PyCodeObject *code);287 288 289#ifdef Py_GIL_DISABLED290// gh-115999 tracks progress on addressing this.291#define ENABLE_SPECIALIZATION 0292// Use this to enable specialization families once they are thread-safe. All293// uses will be replaced with ENABLE_SPECIALIZATION once all families are294// thread-safe.295#define ENABLE_SPECIALIZATION_FT 1296#else297#define ENABLE_SPECIALIZATION 1298#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION299#endif300 301/* Specialization functions */302 303extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls,304                                         _Py_CODEUNIT *instr, int load_method);305extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr,306                                    PyObject *name);307extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr,308                                     PyObject *name);309extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins,310                                      _Py_CODEUNIT *instr, PyObject *name);311extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub,312                                       _Py_CODEUNIT *instr);313extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr,314                                int nargs);315extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr,316                                  int nargs);317extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr,318                                    int oparg, _PyStackRef *locals);319extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs,320                                     _Py_CODEUNIT *instr, int oparg);321extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr,322                                          int oparg);323extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg);324extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr);325extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr);326extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr);327extern void _Py_GatherStats_GetIter(_PyStackRef iterable);328 329// Utility functions for reading/writing 32/64-bit values in the inline caches.330// Great care should be taken to ensure that these functions remain correct and331// performant! They should compile to just "move" instructions on all supported332// compilers and platforms.333 334// We use memcpy to let the C compiler handle unaligned accesses and endianness335// issues for us. It also seems to produce better code than manual copying for336// most compilers (see https://blog.regehr.org/archives/959 for more info).337 338static inline void339write_u32(uint16_t *p, uint32_t val)340{341    memcpy(p, &val, sizeof(val));342}343 344static inline void345write_u64(uint16_t *p, uint64_t val)346{347    memcpy(p, &val, sizeof(val));348}349 350static inline void351write_ptr(uint16_t *p, void *val)352{353    memcpy(p, &val, sizeof(val));354}355 356static inline uint16_t357read_u16(uint16_t *p)358{359    return *p;360}361 362static inline uint32_t363read_u32(uint16_t *p)364{365    uint32_t val;366    memcpy(&val, p, sizeof(val));367    return val;368}369 370static inline uint64_t371read_u64(uint16_t *p)372{373    uint64_t val;374    memcpy(&val, p, sizeof(val));375    return val;376}377 378static inline PyObject *379read_obj(uint16_t *p)380{381    PyObject *val;382    memcpy(&val, p, sizeof(val));383    return val;384}385 386/* See InternalDocs/exception_handling.md for details.387 */388static inline unsigned char *389parse_varint(unsigned char *p, int *result) {390    int val = p[0] & 63;391    while (p[0] & 64) {392        p++;393        val = (val << 6) | (p[0] & 63);394    }395    *result = val;396    return p+1;397}398 399static inline int400write_varint(uint8_t *ptr, unsigned int val)401{402    int written = 1;403    while (val >= 64) {404        *ptr++ = 64 | (val & 63);405        val >>= 6;406        written++;407    }408    *ptr = (uint8_t)val;409    return written;410}411 412static inline int413write_signed_varint(uint8_t *ptr, int val)414{415    unsigned int uval;416    if (val < 0) {417        // (unsigned int)(-val) has an undefined behavior for INT_MIN418        uval = ((0 - (unsigned int)val) << 1) | 1;419    }420    else {421        uval = (unsigned int)val << 1;422    }423    return write_varint(ptr, uval);424}425 426static inline int427write_location_entry_start(uint8_t *ptr, int code, int length)428{429    assert((code & 15) == code);430    *ptr = 128 | (uint8_t)(code << 3) | (uint8_t)(length - 1);431    return 1;432}433 434 435/** Counters436 * The first 16-bit value in each inline cache is a counter.437 *438 * When counting executions until the next specialization attempt,439 * exponential backoff is used to reduce the number of specialization failures.440 * See pycore_backoff.h for more details.441 * On a specialization failure, the backoff counter is restarted.442 */443 444// A value of 1 means that we attempt to specialize the *second* time each445// instruction is executed. Executing twice is a much better indicator of446// "hotness" than executing once, but additional warmup delays only prevent447// specialization. Most types stabilize by the second execution, too:448#define ADAPTIVE_WARMUP_VALUE 1449#define ADAPTIVE_WARMUP_BACKOFF 1450 451// A value of 52 means that we attempt to re-specialize after 53 misses (a prime452// number, useful for avoiding artifacts if every nth value is a different type453// or something). Setting the backoff to 0 means that the counter is reset to454// the same state as a warming-up instruction (value == 1, backoff == 1) after455// deoptimization. This isn't strictly necessary, but it is bit easier to reason456// about when thinking about the opcode transitions as a state machine:457#define ADAPTIVE_COOLDOWN_VALUE 52458#define ADAPTIVE_COOLDOWN_BACKOFF 0459 460// Can't assert this in pycore_backoff.h because of header order dependencies461#if SIDE_EXIT_INITIAL_VALUE <= ADAPTIVE_COOLDOWN_VALUE462#  error  "Cold exit value should be larger than adaptive cooldown value"463#endif464 465static inline _Py_BackoffCounter466adaptive_counter_bits(uint16_t value, uint16_t backoff) {467    return make_backoff_counter(value, backoff);468}469 470static inline _Py_BackoffCounter471adaptive_counter_warmup(void) {472    return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE,473                                 ADAPTIVE_WARMUP_BACKOFF);474}475 476static inline _Py_BackoffCounter477adaptive_counter_cooldown(void) {478    return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE,479                                 ADAPTIVE_COOLDOWN_BACKOFF);480}481 482static inline _Py_BackoffCounter483adaptive_counter_backoff(_Py_BackoffCounter counter) {484    return restart_backoff_counter(counter);485}486 487/* Specialization Extensions */488 489/* callbacks for an external specialization */490typedef int (*binaryopguardfunc)(PyObject *lhs, PyObject *rhs);491typedef PyObject *(*binaryopactionfunc)(PyObject *lhs, PyObject *rhs);492 493typedef struct {494    int oparg;495    binaryopguardfunc guard;496    binaryopactionfunc action;497} _PyBinaryOpSpecializationDescr;498 499/* Comparison bit masks. */500 501/* Note this evaluates its arguments twice each */502#define COMPARISON_BIT(x, y) (1 << (2 * ((x) >= (y)) + ((x) <= (y))))503 504/*505 * The following bits are chosen so that the value of506 * COMPARSION_BIT(left, right)507 * masked by the values below will be non-zero if the508 * comparison is true, and zero if it is false */509 510/* This is for values that are unordered, ie. NaN, not types that are unordered, e.g. sets */511#define COMPARISON_UNORDERED 1512 513#define COMPARISON_LESS_THAN 2514#define COMPARISON_GREATER_THAN 4515#define COMPARISON_EQUALS 8516 517#define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN)518 519extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp);520 521extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset);522 523extern int _PyInstruction_GetLength(PyCodeObject *code, int offset);524 525extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code);526 527struct _PyCode8 _PyCode_DEF(8);528 529PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup;530 531#ifdef Py_GIL_DISABLED532 533static inline _PyCodeArray *534_PyCode_GetTLBCArray(PyCodeObject *co)535{536    return _Py_STATIC_CAST(_PyCodeArray *,537                           _Py_atomic_load_ptr_acquire(&co->co_tlbc));538}539 540// Return a pointer to the thread-local bytecode for the current thread, if it541// exists.542static inline _Py_CODEUNIT *543_PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co)544{545    _PyCodeArray *code = _PyCode_GetTLBCArray(co);546    int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index;547    if (idx < code->size && code->entries[idx] != NULL) {548        return (_Py_CODEUNIT *) code->entries[idx];549    }550    return NULL;551}552 553// Return a pointer to the thread-local bytecode for the current thread,554// creating it if necessary.555extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co);556 557// Reserve an index for the current thread into thread-local bytecode558// arrays559//560// Returns the reserved index or -1 on error.561extern int32_t _Py_ReserveTLBCIndex(PyInterpreterState *interp);562 563// Release the current thread's index into thread-local bytecode arrays564extern void _Py_ClearTLBCIndex(_PyThreadStateImpl *tstate);565 566// Free all TLBC copies not associated with live threads.567//568// Returns 0 on success or -1 on error.569extern int _Py_ClearUnusedTLBC(PyInterpreterState *interp);570#endif571 572 573typedef struct {574    int total;575    struct co_locals_counts {576        int total;577        struct {578            int total;579            int numposonly;580            int numposorkw;581            int numkwonly;582            int varargs;583            int varkwargs;584        } args;585        int numpure;586        struct {587            int total;588            // numargs does not contribute to locals.total.589            int numargs;590            int numothers;591        } cells;592        struct {593            int total;594            int numpure;595            int numcells;596        } hidden;597    } locals;598    int numfree;  // nonlocal599    struct co_unbound_counts {600        int total;601        struct {602            int total;603            int numglobal;604            int numbuiltin;605            int numunknown;606        } globals;607        int numattrs;608        int numunknown;609    } unbound;610} _PyCode_var_counts_t;611 612PyAPI_FUNC(void) _PyCode_GetVarCounts(613        PyCodeObject *,614        _PyCode_var_counts_t *);615PyAPI_FUNC(int) _PyCode_SetUnboundVarCounts(616        PyThreadState *,617        PyCodeObject *,618        _PyCode_var_counts_t *,619        PyObject *globalnames,620        PyObject *attrnames,621        PyObject *globalsns,622        PyObject *builtinsns);623 624 625/* "Stateless" code is a function or code object which does not rely on626 * external state or internal state.  It may rely on arguments and627 * builtins, but not globals or a closure.  Thus it does not rely628 * on __globals__ or __closure__, and a stateless function629 * is equivalent to its code object.630 *631 * Stateless code also does not keep any persistent state632 * of its own, so it can't have any executors, monitoring,633 * instrumentation, or "extras" (i.e. co_extra).634 *635 * Stateless code may create nested functions, including closures.636 * However, nested functions must themselves be stateless, except they637 * *can* close on the enclosing locals.638 *639 * Stateless code may return any value, including nested functions and closures.640 *641 * Stateless code that takes no arguments and doesn't return anything642 * may be treated like a script.643 *644 * We consider stateless code to be "portable" if it does not return645 * any object that holds a reference to any of the code's locals.  Thus646 * generators and coroutines are not portable.  Likewise a function647 * that returns a closure is not portable.  The concept of648 * portability is useful in cases where the code is run649 * in a different execution context than where650 * the return value will be used. */651 652PyAPI_FUNC(int) _PyCode_CheckNoInternalState(PyCodeObject *, const char **);653PyAPI_FUNC(int) _PyCode_CheckNoExternalState(654        PyCodeObject *,655        _PyCode_var_counts_t *,656        const char **);657PyAPI_FUNC(int) _PyCode_VerifyStateless(658        PyThreadState *,659        PyCodeObject *,660        PyObject *globalnames,661        PyObject *globalsns,662        PyObject *builtinsns);663 664PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **);665PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *);666 667 668#ifdef __cplusplus669}670#endif671#endif /* !Py_INTERNAL_CODE_H */672