codekingpro/portable-devtools

1#ifndef _Py_REFCOUNT_H2#define _Py_REFCOUNT_H3#ifdef __cplusplus4extern "C" {5#endif6 7 8/*9Immortalization:10 11The following indicates the immortalization strategy depending on the amount12of available bits in the reference count field. All strategies are backwards13compatible but the specific reference count value or immortalization check14might change depending on the specializations for the underlying system.15 16Proper deallocation of immortal instances requires distinguishing between17statically allocated immortal instances vs those promoted by the runtime to be18immortal. The latter should be the only instances that require19cleanup during runtime finalization.20*/21 22#define _Py_STATICALLY_ALLOCATED_FLAG 423#define _Py_IMMORTAL_FLAGS 124 25#if SIZEOF_VOID_P > 426/*27In 64+ bit systems, any object whose 32 bit reference count is >= 2**3128will be treated as immortal.29 30Using the lower 32 bits makes the value backwards compatible by allowing31C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely32increase and decrease the objects reference count.33 34In order to offer sufficient resilience to C extensions using the stable ABI35compiled against 3.11 or earlier, we set the initial value near the36middle of the range (2**31, 2**32). That way the refcount can be37off by ~1 billion without affecting immortality.38 39Reference count increases will use saturated arithmetic, taking advantage of40having all the lower 32 bits set, which will avoid the reference count to go41beyond the refcount limit. Immortality checks for reference count decreases will42be done by checking the bit sign flag in the lower 32 bits.43 44To ensure that once an object becomes immortal, it remains immortal, the threshold45for omitting increfs is much higher than for omitting decrefs. Consequently, once46the refcount for an object exceeds _Py_IMMORTAL_MINIMUM_REFCNT it will gradually47increase over time until it reaches _Py_IMMORTAL_INITIAL_REFCNT.48*/49#define _Py_IMMORTAL_INITIAL_REFCNT (3ULL << 30)50#define _Py_IMMORTAL_MINIMUM_REFCNT (1ULL << 31)51#define _Py_STATIC_FLAG_BITS ((Py_ssize_t)(_Py_STATICALLY_ALLOCATED_FLAG | _Py_IMMORTAL_FLAGS))52#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT (((Py_ssize_t)_Py_IMMORTAL_INITIAL_REFCNT) | (_Py_STATIC_FLAG_BITS << 48))53 54#else55/*56In 32 bit systems, an object will be treated as immortal if its reference57count equals or exceeds _Py_IMMORTAL_MINIMUM_REFCNT (2**30).58 59Using the lower 30 bits makes the value backwards compatible by allowing60C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely61increase and decrease the objects reference count. The object would lose its62immortality, but the execution would still be correct.63 64Reference count increases and decreases will first go through an immortality65check by comparing the reference count field to the minimum immortality refcount.66*/67#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(5L << 28))68#define _Py_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(1L << 30))69#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(7L << 28))70#define _Py_STATIC_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(6L << 28))71#endif72 73// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is74// always 32-bits.75#ifdef Py_GIL_DISABLED76#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX77#endif78 79 80#ifdef Py_GIL_DISABLED81   // The shared reference count uses the two least-significant bits to store82   // flags. The remaining bits are used to store the reference count.83#  define _Py_REF_SHARED_SHIFT        284#  define _Py_REF_SHARED_FLAG_MASK    0x385 86   // The shared flags are initialized to zero.87#  define _Py_REF_SHARED_INIT         0x088#  define _Py_REF_MAYBE_WEAKREF       0x189#  define _Py_REF_QUEUED              0x290#  define _Py_REF_MERGED              0x391 92   // Create a shared field from a refcnt and desired flags93#  define _Py_REF_SHARED(refcnt, flags) \94              (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags))95#endif  // Py_GIL_DISABLED96 97 98// Py_REFCNT() implementation for the stable ABI99PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob);100 101#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000102    // Stable ABI implements Py_REFCNT() as a function call103    // on limited C API version 3.14 and newer.104#else105    static inline Py_ssize_t _Py_REFCNT(PyObject *ob) {106    #if !defined(Py_GIL_DISABLED)107        return ob->ob_refcnt;108    #else109        uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local);110        if (local == _Py_IMMORTAL_REFCNT_LOCAL) {111            return _Py_IMMORTAL_INITIAL_REFCNT;112        }113        Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared);114        return _Py_STATIC_CAST(Py_ssize_t, local) +115               Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT);116    #endif117    }118    #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000119    #  define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob))120    #else121    #  define Py_REFCNT(ob) _Py_REFCNT(ob)122    #endif123#endif124 125static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op)126{127#if defined(Py_GIL_DISABLED)128    return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) ==129            _Py_IMMORTAL_REFCNT_LOCAL);130#elif SIZEOF_VOID_P > 4131    return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0;132#else133    return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT;134#endif135}136#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op))137 138 139static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op)140{141#if defined(Py_GIL_DISABLED) || SIZEOF_VOID_P > 4142    return (op->ob_flags & _Py_STATICALLY_ALLOCATED_FLAG) != 0;143#else144    return op->ob_refcnt >= _Py_STATIC_IMMORTAL_MINIMUM_REFCNT;145#endif146}147#define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op))148 149// Py_SET_REFCNT() implementation for stable ABI150PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt);151 152static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) {153    assert(refcnt >= 0);154#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000155    // Stable ABI implements Py_SET_REFCNT() as a function call156    // on limited C API version 3.13 and newer.157    _Py_SetRefcnt(ob, refcnt);158#else159    // This immortal check is for code that is unaware of immortal objects.160    // The runtime tracks these objects and we should avoid as much161    // as possible having extensions inadvertently change the refcnt162    // of an immortalized object.163    if (_Py_IsImmortal(ob)) {164        return;165    }166#ifndef Py_GIL_DISABLED167#if SIZEOF_VOID_P > 4168    ob->ob_refcnt = (PY_UINT32_T)refcnt;169#else170    ob->ob_refcnt = refcnt;171#endif172#else173    if (_Py_IsOwnedByCurrentThread(ob)) {174        if ((size_t)refcnt > (size_t)UINT32_MAX) {175            // On overflow, make the object immortal176            ob->ob_tid = _Py_UNOWNED_TID;177            ob->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL;178            ob->ob_ref_shared = 0;179        }180        else {181            // Set local refcount to desired refcount and shared refcount182            // to zero, but preserve the shared refcount flags.183            ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt);184            ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK;185        }186    }187    else {188        // Set local refcount to zero and shared refcount to desired refcount.189        // Mark the object as merged.190        ob->ob_tid = _Py_UNOWNED_TID;191        ob->ob_ref_local = 0;192        ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED);193    }194#endif  // Py_GIL_DISABLED195#endif  // Py_LIMITED_API+0 < 0x030d0000196}197#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000198#  define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt))199#endif200 201 202/*203The macros Py_INCREF(op) and Py_DECREF(op) are used to increment or decrement204reference counts.  Py_DECREF calls the object's deallocator function when205the refcount falls to 0; for206objects that don't contain references to other objects or heap memory207this can be the standard function free().  Both macros can be used208wherever a void expression is allowed.  The argument must not be a209NULL pointer.  If it may be NULL, use Py_XINCREF/Py_XDECREF instead.210The macro _Py_NewReference(op) initialize reference counts to 1, and211in special builds (Py_REF_DEBUG, Py_TRACE_REFS) performs additional212bookkeeping appropriate to the special build.213 214We assume that the reference count field can never overflow; this can215be proven when the size of the field is the same as the pointer size, so216we ignore the possibility.  Provided a C int is at least 32 bits (which217is implicitly assumed in many parts of this code), that's enough for218about 2**31 references to an object.219 220XXX The following became out of date in Python 2.2, but I'm not sure221XXX what the full truth is now.  Certainly, heap-allocated type objects222XXX can and should be deallocated.223Type objects should never be deallocated; the type pointer in an object224is not considered to be a reference to the type object, to save225complications in the deallocation function.  (This is actually a226decision that's up to the implementer of each new type so if you want,227you can count such references to the type object.)228*/229 230#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API)231PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno,232                                      PyObject *op);233PyAPI_FUNC(void) _Py_INCREF_IncRefTotal(void);234PyAPI_FUNC(void) _Py_DECREF_DecRefTotal(void);235#endif  // Py_REF_DEBUG && !Py_LIMITED_API236 237PyAPI_FUNC(void) _Py_Dealloc(PyObject *);238 239 240/*241These are provided as conveniences to Python runtime embedders, so that242they can have object code that is not dependent on Python compilation flags.243*/244PyAPI_FUNC(void) Py_IncRef(PyObject *);245PyAPI_FUNC(void) Py_DecRef(PyObject *);246 247// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL.248// Private functions used by Py_INCREF() and Py_DECREF().249PyAPI_FUNC(void) _Py_IncRef(PyObject *);250PyAPI_FUNC(void) _Py_DecRef(PyObject *);251 252static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)253{254#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))255    // Stable ABI implements Py_INCREF() as a function call on limited C API256    // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef()257    // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions.258    // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't.259#  if Py_LIMITED_API+0 >= 0x030a00A7260    _Py_IncRef(op);261#  else262    Py_IncRef(op);263#  endif264#else265    // Non-limited C API and limited C API for Python 3.9 and older access266    // directly PyObject.ob_refcnt.267#if defined(Py_GIL_DISABLED)268    uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);269    uint32_t new_local = local + 1;270    if (new_local == 0) {271        _Py_INCREF_IMMORTAL_STAT_INC();272        // local is equal to _Py_IMMORTAL_REFCNT_LOCAL: do nothing273        return;274    }275    if (_Py_IsOwnedByCurrentThread(op)) {276        _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local);277    }278    else {279        _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT));280    }281#elif SIZEOF_VOID_P > 4282    PY_UINT32_T cur_refcnt = op->ob_refcnt;283    if (cur_refcnt >= _Py_IMMORTAL_INITIAL_REFCNT) {284        // the object is immortal285        _Py_INCREF_IMMORTAL_STAT_INC();286        return;287    }288    op->ob_refcnt = cur_refcnt + 1;289#else290    if (_Py_IsImmortal(op)) {291        _Py_INCREF_IMMORTAL_STAT_INC();292        return;293    }294    op->ob_refcnt++;295#endif296    _Py_INCREF_STAT_INC();297#ifdef Py_REF_DEBUG298    // Don't count the incref if the object is immortal.299    if (!_Py_IsImmortal(op)) {300        _Py_INCREF_IncRefTotal();301    }302#endif303#endif304}305#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000306#  define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))307#endif308 309 310#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED)311// Implements Py_DECREF on objects not owned by the current thread.312PyAPI_FUNC(void) _Py_DecRefShared(PyObject *);313PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int);314 315// Called from Py_DECREF by the owning thread when the local refcount reaches316// zero. The call will deallocate the object if the shared refcount is also317// zero. Otherwise, the thread gives up ownership and merges the reference318// count fields.319PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *);320#endif321 322#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))323// Stable ABI implements Py_DECREF() as a function call on limited C API324// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was325// added to Python 3.10.0a7, use Py_DecRef() on older Python versions.326// Py_DecRef() accepts NULL whereas _Py_DecRef() doesn't.327static inline void Py_DECREF(PyObject *op) {328#  if Py_LIMITED_API+0 >= 0x030a00A7329    _Py_DecRef(op);330#  else331    Py_DecRef(op);332#  endif333}334#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))335 336#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG)337static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)338{339    uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);340    if (local == _Py_IMMORTAL_REFCNT_LOCAL) {341        _Py_DECREF_IMMORTAL_STAT_INC();342        return;343    }344    _Py_DECREF_STAT_INC();345    _Py_DECREF_DecRefTotal();346    if (_Py_IsOwnedByCurrentThread(op)) {347        if (local == 0) {348            _Py_NegativeRefcount(filename, lineno, op);349        }350        local--;351        _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);352        if (local == 0) {353            _Py_MergeZeroLocalRefcount(op);354        }355    }356    else {357        _Py_DecRefSharedDebug(op, filename, lineno);358    }359}360#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))361 362#elif defined(Py_GIL_DISABLED)363static inline void Py_DECREF(PyObject *op)364{365    uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);366    if (local == _Py_IMMORTAL_REFCNT_LOCAL) {367        _Py_DECREF_IMMORTAL_STAT_INC();368        return;369    }370    _Py_DECREF_STAT_INC();371    if (_Py_IsOwnedByCurrentThread(op)) {372        local--;373        _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);374        if (local == 0) {375            _Py_MergeZeroLocalRefcount(op);376        }377    }378    else {379        _Py_DecRefShared(op);380    }381}382#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))383 384#elif defined(Py_REF_DEBUG)385 386static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)387{388#if SIZEOF_VOID_P > 4389    /* If an object has been freed, it will have a negative full refcnt390     * If it has not it been freed, will have a very large refcnt */391    if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) {392#else393    if (op->ob_refcnt <= 0) {394#endif395        _Py_NegativeRefcount(filename, lineno, op);396    }397    if (_Py_IsImmortal(op)) {398        _Py_DECREF_IMMORTAL_STAT_INC();399        return;400    }401    _Py_DECREF_STAT_INC();402    _Py_DECREF_DecRefTotal();403    if (--op->ob_refcnt == 0) {404        _Py_Dealloc(op);405    }406}407#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))408 409#else410 411static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)412{413    // Non-limited C API and limited C API for Python 3.9 and older access414    // directly PyObject.ob_refcnt.415    if (_Py_IsImmortal(op)) {416        _Py_DECREF_IMMORTAL_STAT_INC();417        return;418    }419    _Py_DECREF_STAT_INC();420    if (--op->ob_refcnt == 0) {421        _Py_Dealloc(op);422    }423}424#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))425#endif426 427 428/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear429 * and tp_dealloc implementations.430 *431 * Note that "the obvious" code can be deadly:432 *433 *     Py_XDECREF(op);434 *     op = NULL;435 *436 * Typically, `op` is something like self->containee, and `self` is done437 * using its `containee` member.  In the code sequence above, suppose438 * `containee` is non-NULL with a refcount of 1.  Its refcount falls to439 * 0 on the first line, which can trigger an arbitrary amount of code,440 * possibly including finalizers (like __del__ methods or weakref callbacks)441 * coded in Python, which in turn can release the GIL and allow other threads442 * to run, etc.  Such code may even invoke methods of `self` again, or cause443 * cyclic gc to trigger, but-- oops! --self->containee still points to the444 * object being torn down, and it may be in an insane state while being torn445 * down.  This has in fact been a rich historic source of miserable (rare &446 * hard-to-diagnose) segfaulting (and other) bugs.447 *448 * The safe way is:449 *450 *      Py_CLEAR(op);451 *452 * That arranges to set `op` to NULL _before_ decref'ing, so that any code453 * triggered as a side-effect of `op` getting torn down no longer believes454 * `op` points to a valid object.455 *456 * There are cases where it's safe to use the naive code, but they're brittle.457 * For example, if `op` points to a Python integer, you know that destroying458 * one of those can't cause problems -- but in part that relies on that459 * Python integers aren't currently weakly referencable.  Best practice is460 * to use Py_CLEAR() even if you can't think of a reason for why you need to.461 *462 * gh-98724: Use a temporary variable to only evaluate the macro argument once,463 * to avoid the duplication of side effects if the argument has side effects.464 *465 * gh-99701: If the PyObject* type is used with casting arguments to PyObject*,466 * the code can be miscompiled with strict aliasing because of type punning.467 * With strict aliasing, a compiler considers that two pointers of different468 * types cannot read or write the same memory which enables optimization469 * opportunities.470 *471 * If available, use _Py_TYPEOF() to use the 'op' type for temporary variables,472 * and so avoid type punning. Otherwise, use memcpy() which causes type erasure473 * and so prevents the compiler to reuse an old cached 'op' value after474 * Py_CLEAR().475 */476#ifdef _Py_TYPEOF477#define Py_CLEAR(op) \478    do { \479        _Py_TYPEOF(op)* _tmp_op_ptr = &(op); \480        _Py_TYPEOF(op) _tmp_old_op = (*_tmp_op_ptr); \481        if (_tmp_old_op != NULL) { \482            *_tmp_op_ptr = _Py_NULL; \483            Py_DECREF(_tmp_old_op); \484        } \485    } while (0)486#else487#define Py_CLEAR(op) \488    do { \489        PyObject **_tmp_op_ptr = _Py_CAST(PyObject**, &(op)); \490        PyObject *_tmp_old_op = (*_tmp_op_ptr); \491        if (_tmp_old_op != NULL) { \492            PyObject *_null_ptr = _Py_NULL; \493            memcpy(_tmp_op_ptr, &_null_ptr, sizeof(PyObject*)); \494            Py_DECREF(_tmp_old_op); \495        } \496    } while (0)497#endif498 499 500/* Function to use in case the object pointer can be NULL: */501static inline void Py_XINCREF(PyObject *op)502{503    if (op != _Py_NULL) {504        Py_INCREF(op);505    }506}507#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000508#  define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op))509#endif510 511static inline void Py_XDECREF(PyObject *op)512{513    if (op != _Py_NULL) {514        Py_DECREF(op);515    }516}517#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000518#  define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op))519#endif520 521// Create a new strong reference to an object:522// increment the reference count of the object and return the object.523PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj);524 525// Similar to Py_NewRef(), but the object can be NULL.526PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj);527 528static inline PyObject* _Py_NewRef(PyObject *obj)529{530    Py_INCREF(obj);531    return obj;532}533 534static inline PyObject* _Py_XNewRef(PyObject *obj)535{536    Py_XINCREF(obj);537    return obj;538}539 540// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI.541// Names overridden with macros by static inline functions for best542// performances.543#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000544#  define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj))545#  define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj))546#else547#  define Py_NewRef(obj) _Py_NewRef(obj)548#  define Py_XNewRef(obj) _Py_XNewRef(obj)549#endif550 551 552#ifdef __cplusplus553}554#endif555#endif   // !_Py_REFCOUNT_H556