codekingpro/portable-devtools

1#ifndef Py_INTERNAL_OBJECT_H2#define Py_INTERNAL_OBJECT_H3#ifdef __cplusplus4extern "C" {5#endif6 7#ifndef Py_BUILD_CORE8#  error "this header requires Py_BUILD_CORE define"9#endif10 11#include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall()12#include "pycore_gc.h"            // _PyObject_GC_TRACK()13#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE()14#include "pycore_pystate.h"       // _PyInterpreterState_GET()15#include "pycore_runtime.h"       // _PyRuntime16#include "pycore_typeobject.h"    // _PyStaticType_GetState()17#include "pycore_uniqueid.h"      // _PyObject_ThreadIncrefSlow()18 19#include <stdbool.h>              // bool20 21 22// This value is added to `ob_ref_shared` for objects that use deferred23// reference counting so that they are not immediately deallocated when the24// non-deferred reference count drops to zero.25//26// The value is half the maximum shared refcount because the low two bits of27// `ob_ref_shared` are used for flags.28#define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8)29 30/* For backwards compatibility -- Do not use this */31#define _Py_IsImmortalLoose(op) _Py_IsImmortal32 33 34/* Check if an object is consistent. For example, ensure that the reference35   counter is greater than or equal to 1, and ensure that ob_type is not NULL.36 37   Call _PyObject_AssertFailed() if the object is inconsistent.38 39   If check_content is zero, only check header fields: reduce the overhead.40 41   The function always return 1. The return value is just here to be able to42   write:43 44   assert(_PyObject_CheckConsistency(obj, 1)); */45extern int _PyObject_CheckConsistency(PyObject *op, int check_content);46 47extern void _PyDebugAllocatorStats(FILE *out, const char *block_name,48                                   int num_blocks, size_t sizeof_block);49 50extern void _PyObject_DebugTypeStats(FILE *out);51 52#ifdef Py_TRACE_REFS53// Forget a reference registered by _Py_NewReference(). Function called by54// _Py_Dealloc().55//56// On a free list, the function can be used before modifying an object to57// remove the object from traced objects. Then _Py_NewReference() or58// _Py_NewReferenceNoTotal() should be called again on the object to trace59// it again.60extern void _Py_ForgetReference(PyObject *);61#endif62 63// Export for shared _testinternalcapi extension64PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *);65 66/* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid67   designated initializer conflicts in C++20. If we use the definition in68   object.h, we will be mixing designated and non-designated initializers in69   pycore objects which is forbiddent in C++20. However, if we then use70   designated initializers in object.h then Extensions without designated break.71   Furthermore, we can't use designated initializers in Extensions since these72   are not supported pre-C++20. Thus, keeping an internal copy here is the most73   backwards compatible solution */74#if defined(Py_GIL_DISABLED)75#define _PyObject_HEAD_INIT(type)                   \76    {                                               \77        .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL,  \78        .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG,  \79        .ob_gc_bits = _PyGC_BITS_DEFERRED,          \80        .ob_type = (type)                           \81    }82#else83#if SIZEOF_VOID_P > 484#define _PyObject_HEAD_INIT(type)         \85    {                                     \86        .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT,  \87        .ob_flags = _Py_STATIC_FLAG_BITS, \88        .ob_type = (type)                 \89    }90#else91#define _PyObject_HEAD_INIT(type)         \92    {                                     \93        .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \94        .ob_type = (type)                 \95    }96#endif97#endif98#define _PyVarObject_HEAD_INIT(type, size)    \99    {                                         \100        .ob_base = _PyObject_HEAD_INIT(type), \101        .ob_size = size                       \102    }103 104PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(105    const char *func,106    const char *message);107 108#define _Py_FatalRefcountError(message) \109    _Py_FatalRefcountErrorFunc(__func__, (message))110 111#define _PyReftracerTrack(obj, operation) \112    do { \113        struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \114        if (tracer->tracer_func != NULL) { \115            void *data = tracer->tracer_data; \116            tracer->tracer_func((obj), (operation), data); \117        } \118    } while(0)119 120#ifdef Py_REF_DEBUG121/* The symbol is only exposed in the API for the sake of extensions122   built against the pre-3.12 stable ABI. */123PyAPI_DATA(Py_ssize_t) _Py_RefTotal;124 125extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t);126extern PyAPI_FUNC(void) _Py_IncRefTotal(PyThreadState *);127extern PyAPI_FUNC(void) _Py_DecRefTotal(PyThreadState *);128 129#  define _Py_DEC_REFTOTAL(interp) \130    interp->object_state.reftotal--131#endif132 133// Increment reference count by n134static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n)135{136    if (_Py_IsImmortal(op)) {137        _Py_INCREF_IMMORTAL_STAT_INC();138        return;139    }140#ifndef Py_GIL_DISABLED141    Py_ssize_t refcnt = _Py_REFCNT(op);142    Py_ssize_t new_refcnt = refcnt + n;143    if (new_refcnt >= (Py_ssize_t)_Py_IMMORTAL_MINIMUM_REFCNT) {144        new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT;145    }146#  if SIZEOF_VOID_P > 4147    op->ob_refcnt = (PY_UINT32_T)new_refcnt;148#  else149    op->ob_refcnt = new_refcnt;150#  endif151#  ifdef Py_REF_DEBUG152    _Py_AddRefTotal(_PyThreadState_GET(), new_refcnt - refcnt);153#  endif154#else155    if (_Py_IsOwnedByCurrentThread(op)) {156        uint32_t local = op->ob_ref_local;157        Py_ssize_t refcnt = (Py_ssize_t)local + n;158#  if PY_SSIZE_T_MAX > UINT32_MAX159        if (refcnt > (Py_ssize_t)UINT32_MAX) {160            // Make the object immortal if the 32-bit local reference count161            // would overflow.162            refcnt = _Py_IMMORTAL_REFCNT_LOCAL;163        }164#  endif165        _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, (uint32_t)refcnt);166    }167    else {168        _Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT));169    }170#  ifdef Py_REF_DEBUG171    _Py_AddRefTotal(_PyThreadState_GET(), n);172#  endif173#endif174    // Although the ref count was increased by `n` (which may be greater than 1)175    // it is only a single increment (i.e. addition) operation, so only 1 refcnt176    // increment operation is counted.177    _Py_INCREF_STAT_INC();178}179#define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n)180 181// Checks if an object has a single, unique reference. If the caller holds a182// unique reference, it may be able to safely modify the object in-place.183static inline int184_PyObject_IsUniquelyReferenced(PyObject *ob)185{186#if !defined(Py_GIL_DISABLED)187    return Py_REFCNT(ob) == 1;188#else189    // NOTE: the entire ob_ref_shared field must be zero, including flags, to190    // ensure that other threads cannot concurrently create new references to191    // this object.192    return (_Py_IsOwnedByCurrentThread(ob) &&193            _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 &&194            _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0);195#endif196}197 198PyAPI_FUNC(void) _Py_SetImmortal(PyObject *op);199PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op);200 201// Makes an immortal object mortal again with the specified refcnt. Should only202// be used during runtime finalization.203static inline void _Py_SetMortal(PyObject *op, short refcnt)204{205    if (op) {206        assert(_Py_IsImmortal(op));207#ifdef Py_GIL_DISABLED208        op->ob_tid = _Py_UNOWNED_TID;209        op->ob_ref_local = 0;210        op->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED);211#else212        op->ob_refcnt = refcnt;213#endif214    }215}216 217/* _Py_ClearImmortal() should only be used during runtime finalization. */218static inline void _Py_ClearImmortal(PyObject *op)219{220    if (op) {221        _Py_SetMortal(op, 1);222        Py_DECREF(op);223    }224}225#define _Py_ClearImmortal(op) \226    do { \227        _Py_ClearImmortal(_PyObject_CAST(op)); \228        op = NULL; \229    } while (0)230 231#if !defined(Py_GIL_DISABLED)232static inline void233_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)234{235    if (_Py_IsImmortal(op)) {236        _Py_DECREF_IMMORTAL_STAT_INC();237        return;238    }239    _Py_DECREF_STAT_INC();240#ifdef Py_REF_DEBUG241    _Py_DEC_REFTOTAL(PyInterpreterState_Get());242#endif243    if (--op->ob_refcnt != 0) {244        assert(op->ob_refcnt > 0);245    }246    else {247#ifdef Py_TRACE_REFS248        _Py_ForgetReference(op);249#endif250        _PyReftracerTrack(op, PyRefTracer_DESTROY);251        destruct(op);252    }253}254 255static inline void256_Py_DECREF_NO_DEALLOC(PyObject *op)257{258    if (_Py_IsImmortal(op)) {259        _Py_DECREF_IMMORTAL_STAT_INC();260        return;261    }262    _Py_DECREF_STAT_INC();263#ifdef Py_REF_DEBUG264    _Py_DEC_REFTOTAL(PyInterpreterState_Get());265#endif266    op->ob_refcnt--;267#ifdef Py_DEBUG268    if (op->ob_refcnt <= 0) {269        _Py_FatalRefcountError("Expected a positive remaining refcount");270    }271#endif272}273 274#else275// TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build276static inline void277_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)278{279    Py_DECREF(op);280}281 282static inline void283_Py_DECREF_NO_DEALLOC(PyObject *op)284{285    Py_DECREF(op);286}287 288static inline int289_Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared)290{291    return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED;292}293 294static inline int295_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared)296{297    return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED;298}299 300// Merge the local and shared reference count fields and add `extra` to the301// refcount when merging.302Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra);303#endif // !defined(Py_GIL_DISABLED)304 305#ifdef Py_REF_DEBUG306#  undef _Py_DEC_REFTOTAL307#endif308 309 310extern int _PyType_CheckConsistency(PyTypeObject *type);311extern int _PyDict_CheckConsistency(PyObject *mp, int check_content);312 313// Fast inlined version of PyType_HasFeature()314static inline int315_PyType_HasFeature(PyTypeObject *type, unsigned long feature) {316    return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0);317}318 319extern void _PyType_InitCache(PyInterpreterState *interp);320 321extern PyStatus _PyObject_InitState(PyInterpreterState *interp);322extern void _PyObject_FiniState(PyInterpreterState *interp);323extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);324 325// Macros used for per-thread reference counting in the free threading build.326// They resolve to normal Py_INCREF/DECREF calls in the default build.327//328// The macros are used for only a few references that would otherwise cause329// scaling bottlenecks in the free threading build:330// - The reference from an object to `ob_type`.331// - The reference from a function to `func_code`.332// - The reference from a function to `func_globals` and `func_builtins`.333//334// It's safe, but not performant or necessary, to use these macros for other335// references to code, type, or dict objects. It's also safe to mix their336// usage with normal Py_INCREF/DECREF calls.337//338// See also Include/internal/pycore_dict.h for _Py_INCREF_DICT/_Py_DECREF_DICT.339#ifndef Py_GIL_DISABLED340#  define _Py_INCREF_TYPE Py_INCREF341#  define _Py_DECREF_TYPE Py_DECREF342#  define _Py_INCREF_CODE Py_INCREF343#  define _Py_DECREF_CODE Py_DECREF344#else345static inline void346_Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)347{348    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();349 350    // The table index is `unique_id - 1` because 0 is not a valid unique id.351    // Unsigned comparison so that `idx=-1` is handled by the "else".352    size_t idx = (size_t)(unique_id - 1);353    if (idx < (size_t)tstate->refcounts.size) {354#  ifdef Py_REF_DEBUG355        _Py_INCREF_IncRefTotal();356#  endif357        _Py_INCREF_STAT_INC();358        tstate->refcounts.values[idx]++;359    }360    else {361        // The slow path resizes the per-thread refcount array if necessary.362        // It handles the unique_id=0 case to keep the inlinable function smaller.363        _PyObject_ThreadIncrefSlow(obj, idx);364    }365}366 367static inline void368_Py_INCREF_TYPE(PyTypeObject *type)369{370    if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {371        assert(_Py_IsImmortal(type));372        _Py_INCREF_IMMORTAL_STAT_INC();373        return;374    }375 376    // gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when377    // _Py_INCREF_TYPE() is called on a statically allocated type.  The378    // _PyType_HasFeature check above ensures that the type is a heap type.379#if defined(__GNUC__) && __GNUC__ >= 11380#  pragma GCC diagnostic push381#  pragma GCC diagnostic ignored "-Warray-bounds"382#endif383    _Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id);384#if defined(__GNUC__) && __GNUC__ >= 11385#  pragma GCC diagnostic pop386#endif387}388 389static inline void390_Py_INCREF_CODE(PyCodeObject *co)391{392    _Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id);393}394 395static inline void396_Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)397{398    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();399 400    // The table index is `unique_id - 1` because 0 is not a valid unique id.401    // Unsigned comparison so that `idx=-1` is handled by the "else".402    size_t idx = (size_t)(unique_id - 1);403    if (idx < (size_t)tstate->refcounts.size) {404#  ifdef Py_REF_DEBUG405        _Py_DECREF_DecRefTotal();406#  endif407        _Py_DECREF_STAT_INC();408        tstate->refcounts.values[idx]--;409    }410    else {411        // Directly decref the object if the id is not assigned or if412        // per-thread refcounting has been disabled on this object.413        Py_DECREF(obj);414    }415}416 417static inline void418_Py_DECREF_TYPE(PyTypeObject *type)419{420    if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {421        assert(_Py_IsImmortal(type));422        _Py_DECREF_IMMORTAL_STAT_INC();423        return;424    }425    PyHeapTypeObject *ht = (PyHeapTypeObject *)type;426    _Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id);427}428 429static inline void430_Py_DECREF_CODE(PyCodeObject *co)431{432    _Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id);433}434#endif435 436#ifndef Py_GIL_DISABLED437#ifdef Py_REF_DEBUG438 439static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *op)440{441    if (op->ob_refcnt <= 0) {442        _Py_NegativeRefcount(filename, lineno, op);443    }444    _Py_DECREF_STAT_INC();445    assert(!_Py_IsStaticImmortal(op));446    if (!_Py_IsImmortal(op)) {447        _Py_DECREF_DecRefTotal();448    }449    if (--op->ob_refcnt == 0) {450        _Py_Dealloc(op);451    }452}453#define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(__FILE__, __LINE__, _PyObject_CAST(op))454 455static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int lineno, PyObject *op, destructor destruct)456{457    if (op->ob_refcnt <= 0) {458        _Py_NegativeRefcount(filename, lineno, op);459    }460    _Py_DECREF_STAT_INC();461    assert(!_Py_IsStaticImmortal(op));462    if (!_Py_IsImmortal(op)) {463        _Py_DECREF_DecRefTotal();464    }465    if (--op->ob_refcnt == 0) {466#ifdef Py_TRACE_REFS467        _Py_ForgetReference(op);468#endif469        _PyReftracerTrack(op, PyRefTracer_DESTROY);470        destruct(op);471    }472}473#define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) _Py_DECREF_MORTAL_SPECIALIZED(__FILE__, __LINE__, op, destruct)474 475#else476 477static inline void Py_DECREF_MORTAL(PyObject *op)478{479    assert(!_Py_IsStaticImmortal(op));480    _Py_DECREF_STAT_INC();481    if (--op->ob_refcnt == 0) {482        _Py_Dealloc(op);483    }484}485#define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(_PyObject_CAST(op))486 487static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruct)488{489    assert(!_Py_IsStaticImmortal(op));490    _Py_DECREF_STAT_INC();491    if (--op->ob_refcnt == 0) {492        _PyReftracerTrack(op, PyRefTracer_DESTROY);493        destruct(op);494    }495}496#define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct)497 498#endif499#endif500 501/* Inline functions trading binary compatibility for speed:502   _PyObject_Init() is the fast version of PyObject_Init(), and503   _PyObject_InitVar() is the fast version of PyObject_InitVar().504 505   These inline functions must not be called with op=NULL. */506static inline void507_PyObject_Init(PyObject *op, PyTypeObject *typeobj)508{509    assert(op != NULL);510    Py_SET_TYPE(op, typeobj);511    assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));512    _Py_INCREF_TYPE(typeobj);513    _Py_NewReference(op);514}515 516static inline void517_PyObject_InitVar(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size)518{519    assert(op != NULL);520    assert(typeobj != &PyLong_Type);521    _PyObject_Init((PyObject *)op, typeobj);522    Py_SET_SIZE(op, size);523}524 525// Macros to accept any type for the parameter, and to automatically pass526// the filename and the filename (if NDEBUG is not defined) where the macro527// is called.528#ifdef NDEBUG529#  define _PyObject_GC_TRACK(op) \530        _PyObject_GC_TRACK(_PyObject_CAST(op))531#  define _PyObject_GC_UNTRACK(op) \532        _PyObject_GC_UNTRACK(_PyObject_CAST(op))533#else534#  define _PyObject_GC_TRACK(op) \535        _PyObject_GC_TRACK(__FILE__, __LINE__, _PyObject_CAST(op))536#  define _PyObject_GC_UNTRACK(op) \537        _PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op))538#endif539 540#ifdef Py_GIL_DISABLED541 542/* Tries to increment an object's reference count543 *544 * This is a specialized version of _Py_TryIncref that only succeeds if the545 * object is immortal or local to this thread. It does not handle the case546 * where the  reference count modification requires an atomic operation. This547 * allows call sites to specialize for the immortal/local case.548 */549static inline int550_Py_TryIncrefFast(PyObject *op) {551    uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);552    local += 1;553    if (local == 0) {554        // immortal555        _Py_INCREF_IMMORTAL_STAT_INC();556        return 1;557    }558    if (_Py_IsOwnedByCurrentThread(op)) {559        _Py_INCREF_STAT_INC();560        _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);561#ifdef Py_REF_DEBUG562        _Py_IncRefTotal(_PyThreadState_GET());563#endif564        return 1;565    }566    return 0;567}568 569static inline int570_Py_TryIncRefShared(PyObject *op)571{572    Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);573    for (;;) {574        // If the shared refcount is zero and the object is either merged575        // or may not have weak references, then we cannot incref it.576        if (shared == 0 || shared == _Py_REF_MERGED) {577            return 0;578        }579 580        if (_Py_atomic_compare_exchange_ssize(581                &op->ob_ref_shared,582                &shared,583                shared + (1 << _Py_REF_SHARED_SHIFT))) {584#ifdef Py_REF_DEBUG585            _Py_IncRefTotal(_PyThreadState_GET());586#endif587            _Py_INCREF_STAT_INC();588            return 1;589        }590    }591}592 593/* Tries to incref the object op and ensures that *src still points to it. */594static inline int595_Py_TryIncrefCompare(PyObject **src, PyObject *op)596{597    if (_Py_TryIncrefFast(op)) {598        return 1;599    }600    if (!_Py_TryIncRefShared(op)) {601        return 0;602    }603    if (op != _Py_atomic_load_ptr(src)) {604        Py_DECREF(op);605        return 0;606    }607    return 1;608}609 610/* Loads and increfs an object from ptr, which may contain a NULL value.611   Safe with concurrent (atomic) updates to ptr.612   NOTE: The writer must set maybe-weakref on the stored object! */613static inline PyObject *614_Py_XGetRef(PyObject **ptr)615{616    for (;;) {617        PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr));618        if (value == NULL) {619            return value;620        }621        if (_Py_TryIncrefCompare(ptr, value)) {622            return value;623        }624    }625}626 627/* Attempts to loads and increfs an object from ptr. Returns NULL628   on failure, which may be due to a NULL value or a concurrent update. */629static inline PyObject *630_Py_TryXGetRef(PyObject **ptr)631{632    PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr));633    if (value == NULL) {634        return value;635    }636    if (_Py_TryIncrefCompare(ptr, value)) {637        return value;638    }639    return NULL;640}641 642/* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF643   on objects owned by a different thread. */644static inline PyObject *645_Py_NewRefWithLock(PyObject *op)646{647    if (_Py_TryIncrefFast(op)) {648        return op;649    }650#ifdef Py_REF_DEBUG651    _Py_IncRefTotal(_PyThreadState_GET());652#endif653    _Py_INCREF_STAT_INC();654    for (;;) {655        Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);656        Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT);657        if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) {658            new_shared |= _Py_REF_MAYBE_WEAKREF;659        }660        if (_Py_atomic_compare_exchange_ssize(661                &op->ob_ref_shared,662                &shared,663                new_shared)) {664            return op;665        }666    }667}668 669static inline PyObject *670_Py_XNewRefWithLock(PyObject *obj)671{672    if (obj == NULL) {673        return NULL;674    }675    return _Py_NewRefWithLock(obj);676}677 678static inline void679_PyObject_SetMaybeWeakref(PyObject *op)680{681    if (_Py_IsImmortal(op)) {682        return;683    }684    for (;;) {685        Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);686        if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) {687            // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states.688            return;689        }690        if (_Py_atomic_compare_exchange_ssize(691                &op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) {692            return;693        }694    }695}696 697extern PyAPI_FUNC(int) _PyObject_ResurrectEndSlow(PyObject *op);698#endif699 700// Temporarily resurrects an object during deallocation. The refcount is set701// to one.702static inline void703_PyObject_ResurrectStart(PyObject *op)704{705    assert(Py_REFCNT(op) == 0);706#ifdef Py_REF_DEBUG707    _Py_IncRefTotal(_PyThreadState_GET());708#endif709#ifdef Py_GIL_DISABLED710    _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_ThreadId());711    _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 1);712    _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0);713#else714    Py_SET_REFCNT(op, 1);715#endif716#ifdef Py_TRACE_REFS717    _Py_ResurrectReference(op);718#endif719}720 721// Undoes an object resurrection by decrementing the refcount without calling722// _Py_Dealloc(). Returns 0 if the object is dead (the normal case), and723// deallocation should continue. Returns 1 if the object is still alive.724static inline int725_PyObject_ResurrectEnd(PyObject *op)726{727#ifdef Py_REF_DEBUG728    _Py_DecRefTotal(_PyThreadState_GET());729#endif730#ifndef Py_GIL_DISABLED731    Py_SET_REFCNT(op, Py_REFCNT(op) - 1);732    if (Py_REFCNT(op) == 0) {733# ifdef Py_TRACE_REFS734        _Py_ForgetReference(op);735# endif736        return 0;737    }738    return 1;739#else740    uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);741    Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared);742    if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) {743        // Fast-path: object has a single refcount and is owned by this thread744        _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0);745# ifdef Py_TRACE_REFS746        _Py_ForgetReference(op);747# endif748        return 0;749    }750    // Slow-path: object has a shared refcount or is not owned by this thread751    return _PyObject_ResurrectEndSlow(op);752#endif753}754 755/* Tries to incref op and returns 1 if successful or 0 otherwise. */756static inline int757_Py_TryIncref(PyObject *op)758{759#ifdef Py_GIL_DISABLED760    return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op);761#else762    if (Py_REFCNT(op) > 0) {763        Py_INCREF(op);764        return 1;765    }766    return 0;767#endif768}769 770#ifdef Py_REF_DEBUG771extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *);772extern void _Py_FinalizeRefTotal(_PyRuntimeState *);773extern void _PyDebug_PrintTotalRefs(void);774#endif775 776#ifdef Py_TRACE_REFS777extern void _Py_AddToAllObjects(PyObject *op);778extern void _Py_PrintReferences(PyInterpreterState *, FILE *);779extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *);780#endif781 782 783/* Return the *address* of the object's weaklist.  The address may be784 * dereferenced to get the current head of the weaklist.  This is useful785 * for iterating over the linked list of weakrefs, especially when the786 * list is being modified externally (e.g. refs getting removed).787 *788 * The returned pointer should not be used to change the head of the list789 * nor should it be used to add, remove, or swap any refs in the list.790 * That is the sole responsibility of the code in weakrefobject.c.791 */792static inline PyObject **793_PyObject_GET_WEAKREFS_LISTPTR(PyObject *op)794{795    if (PyType_Check(op) &&796            ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) {797        PyInterpreterState *interp = _PyInterpreterState_GET();798        managed_static_type_state *state = _PyStaticType_GetState(799                                                interp, (PyTypeObject *)op);800        return _PyStaticType_GET_WEAKREFS_LISTPTR(state);801    }802    // Essentially _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET():803    Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset;804    return (PyObject **)((char *)op + offset);805}806 807/* This is a special case of _PyObject_GET_WEAKREFS_LISTPTR().808 * Only the most fundamental lookup path is used.809 * Consequently, static types should not be used.810 *811 * For static builtin types the returned pointer will always point812 * to a NULL tp_weaklist.  This is fine for any deallocation cases,813 * since static types are never deallocated and static builtin types814 * are only finalized at the end of runtime finalization.815 *816 * If the weaklist for static types is actually needed then use817 * _PyObject_GET_WEAKREFS_LISTPTR().818 */819static inline PyWeakReference **820_PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op)821{822    assert(!PyType_Check(op) ||823            ((PyTypeObject *)op)->tp_flags & Py_TPFLAGS_HEAPTYPE);824    Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset;825    return (PyWeakReference **)((char *)op + offset);826}827 828// Fast inlined version of PyType_IS_GC()829#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)830 831// Fast inlined version of PyObject_IS_GC()832static inline int833_PyObject_IS_GC(PyObject *obj)834{835    PyTypeObject *type = Py_TYPE(obj);836    return (_PyType_IS_GC(type)837            && (type->tp_is_gc == NULL || type->tp_is_gc(obj)));838}839 840// Fast inlined version of PyObject_Hash()841static inline Py_hash_t842_PyObject_HashFast(PyObject *op)843{844    if (PyUnicode_CheckExact(op)) {845        Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(846                             _PyASCIIObject_CAST(op)->hash);847        if (hash != -1) {848            return hash;849        }850    }851    return PyObject_Hash(op);852}853 854static inline size_t855_PyType_PreHeaderSize(PyTypeObject *tp)856{857    return (858#ifndef Py_GIL_DISABLED859        (size_t)_PyType_IS_GC(tp) * sizeof(PyGC_Head) +860#endif861        (size_t)_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *)862    );863}864 865void _PyObject_GC_Link(PyObject *op);866 867// Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL));868extern int _Py_CheckSlotResult(869    PyObject *obj,870    const char *slot_name,871    int success);872 873// Test if a type supports weak references874static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) {875    return (type->tp_weaklistoffset != 0);876}877 878extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems);879PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type);880 881extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);882extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);883extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);884extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name);885 886void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);887extern int _PyObject_StoreInstanceAttribute(PyObject *obj,888                                            PyObject *name, PyObject *value);889extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name,890                                              PyObject **attr);891extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *,892                                             unsigned int *);893 894// Internal API to look for a name through the MRO.895// This stores a stack reference in out and returns the value of896// type->tp_version or zero if name is missing. It doesn't set an exception!897extern unsigned int898_PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out);899 900extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self,901                                       PyObject *name, _PyStackRef *method);902 903// Cache the provided init method in the specialization cache of type if the904// provided type version matches the current version of the type.905//906// The cached value is borrowed and is only valid if guarded by a type907// version check. In free-threaded builds the init method must also use908// deferred reference counting.909//910// Returns 1 if the value was cached or 0 otherwise.911extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type,912                                              PyObject *init,913                                              unsigned int tp_version);914 915#ifdef Py_GIL_DISABLED916#  define MANAGED_DICT_OFFSET    (((Py_ssize_t)sizeof(PyObject *))*-1)917#  define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2)918#else919#  define MANAGED_DICT_OFFSET    (((Py_ssize_t)sizeof(PyObject *))*-3)920#  define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4)921#endif922 923typedef union {924    PyDictObject *dict;925} PyManagedDictPointer;926 927static inline PyManagedDictPointer *928_PyObject_ManagedDictPointer(PyObject *obj)929{930    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);931    return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);932}933 934static inline PyDictObject *935_PyObject_GetManagedDict(PyObject *obj)936{937    PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj);938    return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict);939}940 941static inline PyDictValues *942_PyObject_InlineValues(PyObject *obj)943{944    PyTypeObject *tp = Py_TYPE(obj);945    assert(tp->tp_basicsize > 0 && (size_t)tp->tp_basicsize % sizeof(PyObject *) == 0);946    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);947    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);948    return (PyDictValues *)((char *)obj + tp->tp_basicsize);949}950 951extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);952extern int _PyObject_IsInstanceDictEmpty(PyObject *);953 954// Export for 'math' shared extension955PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *);956PyAPI_FUNC(int) _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self);957 958// Calls the method named `attr` on `self`, but does not set an exception if959// the attribute does not exist.960PyAPI_FUNC(PyObject *)961_PyObject_MaybeCallSpecialNoArgs(PyObject *self, PyObject *attr);962 963PyAPI_FUNC(PyObject *)964_PyObject_MaybeCallSpecialOneArg(PyObject *self, PyObject *attr, PyObject *arg);965 966extern int _PyObject_IsAbstract(PyObject *);967 968PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);969extern PyObject* _PyObject_NextNotImplemented(PyObject *);970 971// Pickle support.972// Export for '_datetime' shared extension973PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *);974 975/* C function call trampolines to mitigate bad function pointer casts.976 *977 * Typical native ABIs ignore additional arguments or fill in missing978 * values with 0/NULL in function pointer cast. Compilers do not show979 * warnings when a function pointer is explicitly casted to an980 * incompatible type.981 *982 * Bad fpcasts are an issue in WebAssembly. WASM's indirect_call has strict983 * function signature checks. Argument count, types, and return type must984 * match.985 *986 * Third party code unintentionally rely on problematic fpcasts. The call987 * trampoline mitigates common occurrences of bad fpcasts on Emscripten.988 */989#if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE))990#define _PyCFunction_TrampolineCall(meth, self, args) \991    (meth)((self), (args))992#define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \993    (meth)((self), (args), (kw))994#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE995 996// Export these 2 symbols for '_pickle' shared extension997PyAPI_DATA(PyTypeObject) _PyNone_Type;998PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type;999 1000// Maps Py_LT to Py_GT, ..., Py_GE to Py_LE.1001// Export for the stable ABI.1002PyAPI_DATA(int) _Py_SwappedOp[];1003 1004extern void _Py_GetConstant_Init(void);1005 1006enum _PyAnnotateFormat {1007    _Py_ANNOTATE_FORMAT_VALUE = 1,1008    _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS = 2,1009    _Py_ANNOTATE_FORMAT_FORWARDREF = 3,1010    _Py_ANNOTATE_FORMAT_STRING = 4,1011};1012 1013int _PyObject_SetDict(PyObject *obj, PyObject *value);1014 1015#ifndef Py_GIL_DISABLED1016static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op)1017{1018    assert(!_Py_IsStaticImmortal(op));1019    op->ob_refcnt++;1020    _Py_INCREF_STAT_INC();1021#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API)1022    if (!_Py_IsImmortal(op)) {1023        _Py_INCREF_IncRefTotal();1024    }1025#endif1026}1027#endif1028 1029#ifdef __cplusplus1030}1031#endif1032#endif /* !Py_INTERNAL_OBJECT_H */1033