codekingpro/portable-devtools

1#ifndef Py_INTERNAL_STACKREF_H2#define Py_INTERNAL_STACKREF_H3#ifdef __cplusplus4extern "C" {5#endif6 7#ifndef Py_BUILD_CORE8#  error "this header requires Py_BUILD_CORE define"9#endif10 11#include "pycore_object.h"        // Py_DECREF_MORTAL12#include "pycore_object_deferred.h" // _PyObject_HasDeferredRefcount()13 14#include <stdbool.h>              // bool15 16 17/*18  This file introduces a new API for handling references on the stack, called19  _PyStackRef. This API is inspired by HPy.20 21  There are 3 main operations, that convert _PyStackRef to PyObject* and22  vice versa:23 24    1. Borrow (discouraged)25    2. Steal26    3. New27 28  Borrow means that the reference is converted without any change in ownership.29  This is discouraged because it makes verification much harder. It also makes30  unboxed integers harder in the future.31 32  Steal means that ownership is transferred to something else. The total33  number of references to the object stays the same.  The old reference is no34  longer valid.35 36  New creates a new reference from the old reference. The old reference37  is still valid.38 39  All _PyStackRef must be operated on by the new reference operations:40 41    1. DUP42    2. CLOSE43 44   DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old45   reference. The old reference remains unchanged.46 47   CLOSE is roughly equivalent to Py_DECREF. It destroys a reference.48 49   Note that it is unsafe to borrow a _PyStackRef and then do normal50   CPython refcounting operations on it!51*/52 53 54#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)55 56#define Py_TAG_BITS 057 58PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref);59PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber);60PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber);61PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber);62extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref);63 64static const _PyStackRef PyStackRef_NULL = { .index = 0 };65 66// Use the first 3 even numbers for None, True and False.67// Odd numbers are reserved for (tagged) integers68#define PyStackRef_None ((_PyStackRef){ .index = 2 } )69#define PyStackRef_False ((_PyStackRef){ .index = 4 })70#define PyStackRef_True ((_PyStackRef){ .index = 6 })71 72#define INITIAL_STACKREF_INDEX 873 74static inline int75PyStackRef_IsNull(_PyStackRef ref)76{77    return ref.index == 0;78}79 80static inline int81PyStackRef_IsTrue(_PyStackRef ref)82{83    return _Py_stackref_get_object(ref) == Py_True;84}85 86static inline int87PyStackRef_IsFalse(_PyStackRef ref)88{89    return _Py_stackref_get_object(ref) == Py_False;90}91 92static inline int93PyStackRef_IsNone(_PyStackRef ref)94{95    return _Py_stackref_get_object(ref) == Py_None;96}97 98static inline PyObject *99_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)100{101    assert((ref.index & 1) == 0);102    _Py_stackref_record_borrow(ref, filename, linenumber);103    return _Py_stackref_get_object(ref);104}105 106#define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__)107 108static inline PyObject *109_PyStackRef_AsPyObjectSteal(_PyStackRef ref, const char *filename, int linenumber)110{111    return _Py_stackref_close(ref, filename, linenumber);112}113#define PyStackRef_AsPyObjectSteal(REF) _PyStackRef_AsPyObjectSteal((REF), __FILE__, __LINE__)114 115static inline _PyStackRef116_PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber)117{118    Py_INCREF(obj);119    return _Py_stackref_create(obj, filename, linenumber);120}121#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__)122 123static inline _PyStackRef124_PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber)125{126    return _Py_stackref_create(obj, filename, linenumber);127}128#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__)129 130static inline _PyStackRef131_PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber)132{133    return _Py_stackref_create(obj, filename, linenumber);134}135#define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__)136 137static inline _PyStackRef138_PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber)139{140    assert(_Py_IsImmortal(obj));141    return _Py_stackref_create(obj, filename, linenumber);142}143#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__)144 145static inline bool146PyStackRef_IsTaggedInt(_PyStackRef ref)147{148    return (ref.index & 1) == 1;149}150 151static inline void152_PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)153{154    if (PyStackRef_IsTaggedInt(ref)) {155        return;156    }157    PyObject *obj = _Py_stackref_close(ref, filename, linenumber);158    Py_DECREF(obj);159}160#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)161 162 163static inline void164_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)165{166    if (PyStackRef_IsNull(ref)) {167        return;168    }169    _PyStackRef_CLOSE(ref, filename, linenumber);170}171#define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)172 173static inline _PyStackRef174_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)175{176    if (PyStackRef_IsTaggedInt(ref)) {177        return ref;178    }179    else {180        PyObject *obj = _Py_stackref_get_object(ref);181        Py_INCREF(obj);182        return _Py_stackref_create(obj, filename, linenumber);183    }184}185#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)186 187extern void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber);188#define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__)189 190static inline _PyStackRef191PyStackRef_MakeHeapSafe(_PyStackRef ref)192{193    return ref;194}195 196static inline _PyStackRef197PyStackRef_Borrow(_PyStackRef ref)198{199    return PyStackRef_DUP(ref);200}201 202#define PyStackRef_CLEAR(REF) \203    do { \204        _PyStackRef *_tmp_op_ptr = &(REF); \205        _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \206        *_tmp_op_ptr = PyStackRef_NULL; \207        PyStackRef_XCLOSE(_tmp_old_op); \208    } while (0)209 210static inline _PyStackRef211_PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber)212{213    assert(!_Py_IsImmortal(obj));214    return _Py_stackref_create(obj, filename, linenumber);215}216#define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__)217 218static inline bool219PyStackRef_IsHeapSafe(_PyStackRef ref)220{221    return true;222}223 224static inline _PyStackRef225_PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linenumber)226{227    assert(!_Py_IsStaticImmortal(obj));228    Py_INCREF(obj);229    return _Py_stackref_create(obj, filename, linenumber);230}231#define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj), __FILE__, __LINE__)232 233#define PyStackRef_RefcountOnObject(REF) 1234 235extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b);236 237extern bool PyStackRef_IsTaggedInt(_PyStackRef ref);238 239extern intptr_t PyStackRef_UntagInt(_PyStackRef ref);240 241extern _PyStackRef PyStackRef_TagInt(intptr_t i);242 243extern bool244PyStackRef_IsNullOrInt(_PyStackRef ref);245 246#else247 248#define Py_INT_TAG 3249#define Py_TAG_REFCNT 1250 251static inline bool252PyStackRef_IsTaggedInt(_PyStackRef i)253{254    return (i.bits & Py_INT_TAG) == Py_INT_TAG;255}256 257static inline _PyStackRef258PyStackRef_TagInt(intptr_t i)259{260    assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i);261    return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) };262}263 264static inline intptr_t265PyStackRef_UntagInt(_PyStackRef i)266{267    assert((i.bits & Py_INT_TAG) == Py_INT_TAG);268    intptr_t val = (intptr_t)i.bits;269    return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2);270}271 272 273#ifdef Py_GIL_DISABLED274 275#define Py_TAG_DEFERRED Py_TAG_REFCNT276 277#define Py_TAG_PTR      ((uintptr_t)0)278#define Py_TAG_BITS     ((uintptr_t)1)279 280 281static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};282#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)283#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })284#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })285#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })286 287// Checks that mask out the deferred bit in the free threading build.288#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None)289#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)290#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)291 292#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref))293 294static inline PyObject *295PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)296{297    PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS)));298    return cleared;299}300 301#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED)302 303static inline PyObject *304PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref)305{306    assert(!PyStackRef_IsDeferred(stackref));307    return (PyObject *)stackref.bits;308}309 310static inline PyObject *311PyStackRef_AsPyObjectSteal(_PyStackRef stackref)312{313    assert(!PyStackRef_IsNull(stackref));314    if (PyStackRef_IsDeferred(stackref)) {315        return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));316    }317    return PyStackRef_AsPyObjectBorrow(stackref);318}319 320static inline _PyStackRef321_PyStackRef_FromPyObjectSteal(PyObject *obj)322{323    assert(obj != NULL);324    // Make sure we don't take an already tagged value.325    assert(((uintptr_t)obj & Py_TAG_BITS) == 0);326    return (_PyStackRef){ .bits = (uintptr_t)obj };327}328#   define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))329 330static inline _PyStackRef331PyStackRef_FromPyObjectBorrow(PyObject *obj)332{333    assert(obj != NULL);334    assert(((uintptr_t)obj & Py_TAG_BITS) == 0);335    return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };336}337 338static inline bool339PyStackRef_IsHeapSafe(_PyStackRef stackref)340{341    if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsTaggedInt(stackref)) {342        PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);343        return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj);344    }345    return true;346}347 348static inline _PyStackRef349PyStackRef_MakeHeapSafe(_PyStackRef stackref)350{351    if (PyStackRef_IsHeapSafe(stackref)) {352        return stackref;353    }354    PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);355    return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };356}357 358static inline _PyStackRef359PyStackRef_FromPyObjectStealMortal(PyObject *obj)360{361    assert(obj != NULL);362    assert(!_Py_IsImmortal(obj));363    // Make sure we don't take an already tagged value.364    assert(((uintptr_t)obj & Py_TAG_BITS) == 0);365    return (_PyStackRef){ .bits = (uintptr_t)obj };366}367 368static inline _PyStackRef369PyStackRef_FromPyObjectNew(PyObject *obj)370{371    // Make sure we don't take an already tagged value.372    assert(((uintptr_t)obj & Py_TAG_BITS) == 0);373    assert(obj != NULL);374    if (_PyObject_HasDeferredRefcount(obj)) {375        return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };376    }377    else {378        return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };379    }380}381#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))382 383static inline _PyStackRef384PyStackRef_FromPyObjectImmortal(PyObject *obj)385{386    // Make sure we don't take an already tagged value.387    assert(((uintptr_t)obj & Py_TAG_BITS) == 0);388    assert(obj != NULL);389    assert(_Py_IsImmortal(obj));390    return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };391}392#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj))393 394#define PyStackRef_CLOSE(REF)                                        \395        do {                                                            \396            _PyStackRef _close_tmp = (REF);                             \397            assert(!PyStackRef_IsNull(_close_tmp));                     \398            if (!PyStackRef_IsDeferred(_close_tmp)) {                   \399                Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp));     \400            }                                                           \401        } while (0)402 403static inline void404PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)405{406    (void)destruct;407    PyStackRef_CLOSE(ref);408}409 410static inline _PyStackRef411PyStackRef_DUP(_PyStackRef stackref)412{413    assert(!PyStackRef_IsNull(stackref));414    if (PyStackRef_IsDeferred(stackref)) {415        return stackref;416    }417    Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));418    return stackref;419}420 421static inline _PyStackRef422PyStackRef_Borrow(_PyStackRef stackref)423{424    return (_PyStackRef){ .bits = stackref.bits | Py_TAG_DEFERRED };425}426 427// Convert a possibly deferred reference to a strong reference.428static inline _PyStackRef429PyStackRef_AsStrongReference(_PyStackRef stackref)430{431    return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref));432}433 434#define PyStackRef_XCLOSE(stackref) \435    do {                            \436        _PyStackRef _tmp = (stackref); \437        if (!PyStackRef_IsNull(_tmp)) { \438            PyStackRef_CLOSE(_tmp); \439        } \440    } while (0);441 442#define PyStackRef_CLEAR(op) \443    do { \444        _PyStackRef *_tmp_op_ptr = &(op); \445        _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \446        if (!PyStackRef_IsNull(_tmp_old_op)) { \447            *_tmp_op_ptr = PyStackRef_NULL; \448            PyStackRef_CLOSE(_tmp_old_op); \449        } \450    } while (0)451 452#define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew453 454#else // Py_GIL_DISABLED455 456// With GIL457 458/* References to immortal objects always have their tag bit set to Py_TAG_REFCNT459 * as they can (must) have their reclamation deferred */460 461#define Py_TAG_BITS 3462#if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT463#  error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT"464#endif465 466#define BITS_TO_PTR(REF) ((PyObject *)((REF).bits))467#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT)))468 469#define PyStackRef_NULL_BITS Py_TAG_REFCNT470static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS };471 472#define PyStackRef_IsNull(ref) ((ref).bits == PyStackRef_NULL_BITS)473#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT })474#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT })475#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT })476 477#define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT))478#define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT))479#define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT))480 481#ifdef Py_DEBUG482 483static inline void PyStackRef_CheckValid(_PyStackRef ref) {484    assert(ref.bits != 0);485    int tag = ref.bits & Py_TAG_BITS;486    PyObject *obj = BITS_TO_PTR_MASKED(ref);487    switch (tag) {488        case 0:489            /* Can be immortal if object was made immortal after reference came into existence */490            assert(!_Py_IsStaticImmortal(obj));491            break;492        case Py_TAG_REFCNT:493            break;494        default:495            assert(0);496    }497}498 499#else500 501#define PyStackRef_CheckValid(REF) ((void)0)502 503#endif504 505#ifdef _WIN32506#define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_REFCNT) == 0)507#define PyStackRef_AsPyObjectBorrow BITS_TO_PTR_MASKED508#define PyStackRef_Borrow(REF) (_PyStackRef){ .bits = ((REF).bits) | Py_TAG_REFCNT};509#else510/* Does this ref not have an embedded refcount and thus not refer to a declared immmortal object? */511static inline int512PyStackRef_RefcountOnObject(_PyStackRef ref)513{514    return (ref.bits & Py_TAG_REFCNT) == 0;515}516 517static inline PyObject *518PyStackRef_AsPyObjectBorrow(_PyStackRef ref)519{520    assert(!PyStackRef_IsTaggedInt(ref));521    return BITS_TO_PTR_MASKED(ref);522}523 524static inline _PyStackRef525PyStackRef_Borrow(_PyStackRef ref)526{527    return (_PyStackRef){ .bits = ref.bits | Py_TAG_REFCNT };528}529#endif530 531static inline PyObject *532PyStackRef_AsPyObjectSteal(_PyStackRef ref)533{534    if (PyStackRef_RefcountOnObject(ref)) {535        return BITS_TO_PTR(ref);536    }537    else {538        return Py_NewRef(BITS_TO_PTR_MASKED(ref));539    }540}541 542static inline _PyStackRef543PyStackRef_FromPyObjectSteal(PyObject *obj)544{545    assert(obj != NULL);546#if SIZEOF_VOID_P > 4547    unsigned int tag = obj->ob_flags & Py_TAG_REFCNT;548#else549    unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0;550#endif551    _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});552    PyStackRef_CheckValid(ref);553    return ref;554}555 556static inline _PyStackRef557PyStackRef_FromPyObjectBorrow(PyObject *obj)558{559    assert(obj != NULL);560    return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT };561}562 563static inline _PyStackRef564PyStackRef_FromPyObjectStealMortal(PyObject *obj)565{566    assert(obj != NULL);567    assert(!_Py_IsImmortal(obj));568    _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) });569    PyStackRef_CheckValid(ref);570    return ref;571}572 573static inline _PyStackRef574_PyStackRef_FromPyObjectNew(PyObject *obj)575{576    assert(obj != NULL);577    if (_Py_IsImmortal(obj)) {578        return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT};579    }580    _Py_INCREF_MORTAL(obj);581    _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };582    PyStackRef_CheckValid(ref);583    return ref;584}585#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))586 587static inline _PyStackRef588_PyStackRef_FromPyObjectNewMortal(PyObject *obj)589{590    assert(obj != NULL);591    _Py_INCREF_MORTAL(obj);592    _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };593    PyStackRef_CheckValid(ref);594    return ref;595}596#define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj))597 598/* Create a new reference from an object with an embedded reference count */599static inline _PyStackRef600PyStackRef_FromPyObjectImmortal(PyObject *obj)601{602    assert(_Py_IsImmortal(obj));603    return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT};604}605 606/* WARNING: This macro evaluates its argument more than once */607#ifdef _WIN32608#define PyStackRef_DUP(REF) \609    (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF))610#else611static inline _PyStackRef612PyStackRef_DUP(_PyStackRef ref)613{614    assert(!PyStackRef_IsNull(ref));615    if (PyStackRef_RefcountOnObject(ref)) {616        _Py_INCREF_MORTAL(BITS_TO_PTR(ref));617    }618    return ref;619}620#endif621 622static inline bool623PyStackRef_IsHeapSafe(_PyStackRef ref)624{625    return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS ||  _Py_IsImmortal(BITS_TO_PTR_MASKED(ref));626}627 628static inline _PyStackRef629PyStackRef_MakeHeapSafe(_PyStackRef ref)630{631    if (PyStackRef_IsHeapSafe(ref)) {632        return ref;633    }634    PyObject *obj = BITS_TO_PTR_MASKED(ref);635    Py_INCREF(obj);636    ref.bits = (uintptr_t)obj;637    PyStackRef_CheckValid(ref);638    return ref;639}640 641#ifdef _WIN32642#define PyStackRef_CLOSE(REF) \643do { \644    _PyStackRef _temp = (REF); \645    if (PyStackRef_RefcountOnObject(_temp)) Py_DECREF_MORTAL(BITS_TO_PTR(_temp)); \646} while (0)647#else648static inline void649PyStackRef_CLOSE(_PyStackRef ref)650{651    assert(!PyStackRef_IsNull(ref));652    if (PyStackRef_RefcountOnObject(ref)) {653        Py_DECREF_MORTAL(BITS_TO_PTR(ref));654    }655}656#endif657 658static inline bool659PyStackRef_IsNullOrInt(_PyStackRef ref)660{661    return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);662}663 664static inline void665PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)666{667    assert(!PyStackRef_IsNull(ref));668    if (PyStackRef_RefcountOnObject(ref)) {669        Py_DECREF_MORTAL_SPECIALIZED(BITS_TO_PTR(ref), destruct);670    }671}672 673#ifdef _WIN32674#define PyStackRef_XCLOSE PyStackRef_CLOSE675#else676static inline void677PyStackRef_XCLOSE(_PyStackRef ref)678{679    assert(ref.bits != 0);680    if (PyStackRef_RefcountOnObject(ref)) {681        assert(!PyStackRef_IsNull(ref));682        Py_DECREF_MORTAL(BITS_TO_PTR(ref));683    }684}685#endif686 687#define PyStackRef_CLEAR(REF) \688    do { \689        _PyStackRef *_tmp_op_ptr = &(REF); \690        _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \691        *_tmp_op_ptr = PyStackRef_NULL; \692        PyStackRef_XCLOSE(_tmp_old_op); \693    } while (0)694 695 696#endif // Py_GIL_DISABLED697 698// Note: this is a macro because MSVC (Windows) has trouble inlining it.699 700#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT)))701 702 703#endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)704 705#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref))706 707// Converts a PyStackRef back to a PyObject *, converting the708// stackref to a new reference.709#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))710 711// StackRef type checks712 713static inline bool714PyStackRef_GenCheck(_PyStackRef stackref)715{716    return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref));717}718 719static inline bool720PyStackRef_BoolCheck(_PyStackRef stackref)721{722    return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref));723}724 725static inline bool726PyStackRef_LongCheck(_PyStackRef stackref)727{728    return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref));729}730 731static inline bool732PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref)733{734    return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref));735}736 737static inline bool738PyStackRef_CodeCheck(_PyStackRef stackref)739{740    return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref));741}742 743static inline bool744PyStackRef_FunctionCheck(_PyStackRef stackref)745{746    return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref));747}748 749static inline void750_PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref)751{752#ifdef Py_GIL_DISABLED753    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;754    ref->next = tstate_impl->c_stack_refs;755    tstate_impl->c_stack_refs = ref;756#endif757    ref->ref = PyStackRef_NULL;758}759 760static inline void761_PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj)762{763    _PyThreadState_PushCStackRef(tstate, ref);764    ref->ref = PyStackRef_FromPyObjectNew(obj);765}766 767static inline void768_PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref)769{770#ifdef Py_GIL_DISABLED771    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;772    assert(tstate_impl->c_stack_refs == ref);773    tstate_impl->c_stack_refs = ref->next;774#endif775    PyStackRef_XCLOSE(ref->ref);776}777 778static inline _PyStackRef779_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref)780{781#ifdef Py_GIL_DISABLED782    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;783    assert(tstate_impl->c_stack_refs == ref);784    tstate_impl->c_stack_refs = ref->next;785#endif786    return ref->ref;787}788 789#ifdef Py_GIL_DISABLED790 791static inline int792_Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out)793{794    if (_PyObject_HasDeferredRefcount(op)) {795        *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_DEFERRED };796        return 1;797    }798    if (_Py_TryIncrefCompare(src, op)) {799        *out = PyStackRef_FromPyObjectSteal(op);800        return 1;801    }802    return 0;803}804 805static inline int806_Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)807{808    PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src));809    if (op == NULL) {810        *out = PyStackRef_NULL;811        return 1;812    }813    return _Py_TryIncrefCompareStackRef(src, op, out);814}815 816#endif817 818#define PyStackRef_XSETREF(dst, src) \819    do { \820        _PyStackRef _tmp_dst_ref = (dst); \821        (dst) = (src); \822        PyStackRef_XCLOSE(_tmp_dst_ref); \823    } while(0)824 825// Like Py_VISIT but for _PyStackRef fields826#define _Py_VISIT_STACKREF(ref)                                         \827    do {                                                                \828        if (!PyStackRef_IsNullOrInt(ref)) {                             \829            int vret = _PyGC_VisitStackRef(&(ref), visit, arg);         \830            if (vret)                                                   \831                return vret;                                            \832        }                                                               \833    } while (0)834 835#ifdef __cplusplus836}837#endif838#endif /* !Py_INTERNAL_STACKREF_H */839