codekingpro/portable-devtools

1#ifndef Py_INTERNAL_CRITICAL_SECTION_H2#define Py_INTERNAL_CRITICAL_SECTION_H3 4#ifndef Py_BUILD_CORE5#  error "this header requires Py_BUILD_CORE define"6#endif7 8#include "pycore_lock.h"        // PyMutex_LockFast()9#include "pycore_pystate.h"     // _PyThreadState_GET()10#include <stdint.h>11 12#ifdef __cplusplus13extern "C" {14#endif15 16// Tagged pointers to critical sections use the two least significant bits to17// mark if the pointed-to critical section is inactive and whether it is a18// PyCriticalSection2 object.19#define _Py_CRITICAL_SECTION_INACTIVE       0x120#define _Py_CRITICAL_SECTION_TWO_MUTEXES    0x221#define _Py_CRITICAL_SECTION_MASK           0x322 23#ifdef Py_GIL_DISABLED24// Specialized version of critical section locking to safely use25// PySequence_Fast APIs without the GIL. For performance, the argument *to*26// PySequence_Fast() is provided to the macro, not the *result* of27// PySequence_Fast(), which would require an extra test to determine if the28// lock must be acquired.29# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original)              \30    {                                                                   \31        PyObject *_orig_seq = _PyObject_CAST(original);                 \32        const bool _should_lock_cs = PyList_CheckExact(_orig_seq);      \33        PyCriticalSection _cs;                                          \34        if (_should_lock_cs) {                                          \35            _PyCriticalSection_Begin(&_cs, _orig_seq);                  \36        }37 38# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST()                        \39        if (_should_lock_cs) {                                          \40            PyCriticalSection_End(&_cs);                                \41        }                                                               \42    }43 44// Asserts that the mutex is locked.  The mutex must be held by the45// top-most critical section otherwise there's the possibility46// that the mutex would be swalled out in some code paths.47#define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \48    _PyCriticalSection_AssertHeld(mutex)49 50// Asserts that the mutex for the given object is locked. The mutex must51// be held by the top-most critical section otherwise there's the52// possibility that the mutex would be swalled out in some code paths.53#ifdef Py_DEBUG54 55# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op)                           \56    if (Py_REFCNT(op) != 1) {                                                    \57        _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \58    }59 60#else   /* Py_DEBUG */61 62# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op)63 64#endif  /* Py_DEBUG */65 66#else  /* !Py_GIL_DISABLED */67// The critical section APIs are no-ops with the GIL.68# define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) {69# define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() }70# define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex)71# define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op)72#endif  /* !Py_GIL_DISABLED */73 74// Resumes the top-most critical section.75PyAPI_FUNC(void)76_PyCriticalSection_Resume(PyThreadState *tstate);77 78// (private) slow path for locking the mutex79PyAPI_FUNC(void)80_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m);81 82PyAPI_FUNC(void)83_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,84                             int is_m1_locked);85 86PyAPI_FUNC(void)87_PyCriticalSection_SuspendAll(PyThreadState *tstate);88 89#ifdef Py_GIL_DISABLED90 91static inline int92_PyCriticalSection_IsActive(uintptr_t tag)93{94    return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0;95}96 97static inline void98_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m)99{100    if (PyMutex_LockFast(m)) {101        PyThreadState *tstate = _PyThreadState_GET();102        c->_cs_mutex = m;103        c->_cs_prev = tstate->critical_section;104        tstate->critical_section = (uintptr_t)c;105    }106    else {107        _PyCriticalSection_BeginSlow(c, m);108    }109}110#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex111 112static inline void113_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op)114{115    _PyCriticalSection_BeginMutex(c, &op->ob_mutex);116}117#define PyCriticalSection_Begin _PyCriticalSection_Begin118 119// Removes the top-most critical section from the thread's stack of critical120// sections. If the new top-most critical section is inactive, then it is121// resumed.122static inline void123_PyCriticalSection_Pop(PyCriticalSection *c)124{125    PyThreadState *tstate = _PyThreadState_GET();126    uintptr_t prev = c->_cs_prev;127    tstate->critical_section = prev;128 129    if ((prev & _Py_CRITICAL_SECTION_INACTIVE) != 0) {130        _PyCriticalSection_Resume(tstate);131    }132}133 134static inline void135_PyCriticalSection_End(PyCriticalSection *c)136{137    // If the mutex is NULL, we used the fast path in138    // _PyCriticalSection_BeginSlow for locks already held in the top-most139    // critical section, and we shouldn't unlock or pop this critical section.140    if (c->_cs_mutex == NULL) {141        return;142    }143    PyMutex_Unlock(c->_cs_mutex);144    _PyCriticalSection_Pop(c);145}146#define PyCriticalSection_End _PyCriticalSection_End147 148static inline void149_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)150{151    if (m1 == m2) {152        // If the two mutex arguments are the same, treat this as a critical153        // section with a single mutex.154        c->_cs_mutex2 = NULL;155        _PyCriticalSection_BeginMutex(&c->_cs_base, m1);156        return;157    }158 159    if ((uintptr_t)m2 < (uintptr_t)m1) {160        // Sort the mutexes so that the lower address is locked first.161        // The exact order does not matter, but we need to acquire the mutexes162        // in a consistent order to avoid lock ordering deadlocks.163        PyMutex *tmp = m1;164        m1 = m2;165        m2 = tmp;166    }167 168    if (PyMutex_LockFast(m1)) {169        if (PyMutex_LockFast(m2)) {170            PyThreadState *tstate = _PyThreadState_GET();171            c->_cs_base._cs_mutex = m1;172            c->_cs_mutex2 = m2;173            c->_cs_base._cs_prev = tstate->critical_section;174 175            uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES;176            tstate->critical_section = p;177        }178        else {179            _PyCriticalSection2_BeginSlow(c, m1, m2, 1);180        }181    }182    else {183        _PyCriticalSection2_BeginSlow(c, m1, m2, 0);184    }185}186#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex187 188static inline void189_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b)190{191    _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex);192}193#define PyCriticalSection2_Begin _PyCriticalSection2_Begin194 195static inline void196_PyCriticalSection2_End(PyCriticalSection2 *c)197{198    // if mutex1 is NULL, we used the fast path in199    // _PyCriticalSection_BeginSlow for mutexes that are already held,200    // which should only happen when mutex1 and mutex2 were the same mutex,201    // and mutex2 should also be NULL.202    if (c->_cs_base._cs_mutex == NULL) {203        assert(c->_cs_mutex2 == NULL);204        return;205    }206    if (c->_cs_mutex2) {207        PyMutex_Unlock(c->_cs_mutex2);208    }209    PyMutex_Unlock(c->_cs_base._cs_mutex);210    _PyCriticalSection_Pop(&c->_cs_base);211}212#define PyCriticalSection2_End _PyCriticalSection2_End213 214static inline void215_PyCriticalSection_AssertHeld(PyMutex *mutex)216{217#ifdef Py_DEBUG218    PyThreadState *tstate = _PyThreadState_GET();219    uintptr_t prev = tstate->critical_section;220    if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) {221        PyCriticalSection2 *cs = (PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK);222        assert(cs != NULL && (cs->_cs_base._cs_mutex == mutex || cs->_cs_mutex2 == mutex));223    }224    else {225        PyCriticalSection *cs = (PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK);226        assert(cs != NULL && cs->_cs_mutex == mutex);227    }228 229#endif230}231 232#endif /* Py_GIL_DISABLED */233 234#ifdef __cplusplus235}236#endif237#endif /* !Py_INTERNAL_CRITICAL_SECTION_H */238