codekingpro/portable-devtools

1// This header provides cross-platform low-level atomic operations2// similar to C11 atomics.3//4// Operations are sequentially consistent unless they have a suffix indicating5// otherwise. If in doubt, prefer the sequentially consistent operations.6//7// The "_relaxed" suffix for load and store operations indicates the "relaxed"8// memory order. They don't provide synchronization, but (roughly speaking)9// guarantee somewhat sane behavior for races instead of undefined behavior.10// In practice, they correspond to "normal" hardware load and store11// instructions, so they are almost as inexpensive as plain loads and stores12// in C.13//14// Note that atomic read-modify-write operations like _Py_atomic_add_* return15// the previous value of the atomic variable, not the new value.16//17// See https://en.cppreference.com/w/c/atomic for more information on C1118// atomics.19// See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf20// "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage21// or relaxed atomics.22//23// Functions with pseudo Python code:24//25//   def _Py_atomic_load(obj):26//        return obj  # sequential consistency27//28//   def _Py_atomic_load_relaxed(obj):29//       return obj  # relaxed consistency30//31//   def _Py_atomic_store(obj, value):32//       obj = value  # sequential consistency33//34//   def _Py_atomic_store_relaxed(obj, value):35//       obj = value  # relaxed consistency36//37//   def _Py_atomic_exchange(obj, value):38//       # sequential consistency39//       old_obj = obj40//       obj = value41//       return old_obj42//43//   def _Py_atomic_compare_exchange(obj, expected, desired):44//       # sequential consistency45//       if obj == expected:46//           obj = desired47//           return True48//       else:49//           expected = obj50//           return False51//52//   def _Py_atomic_add(obj, value):53//       # sequential consistency54//       old_obj = obj55//       obj += value56//       return old_obj57//58//   def _Py_atomic_and(obj, value):59//       # sequential consistency60//       old_obj = obj61//       obj &= value62//       return old_obj63//64//   def _Py_atomic_or(obj, value):65//       # sequential consistency66//       old_obj = obj67//       obj |= value68//       return old_obj69//70// Other functions:71//72//   def _Py_atomic_load_ptr_acquire(obj):73//       return obj  # acquire74//75//   def _Py_atomic_store_ptr_release(obj, value):76//       obj = value  # release77//78//   def _Py_atomic_fence_seq_cst():79//       # sequential consistency80//       ...81//82//   def _Py_atomic_fence_release():83//       # release84//       ...85 86#ifndef Py_CPYTHON_ATOMIC_H87#  error "this header file must not be included directly"88#endif89 90// --- _Py_atomic_add --------------------------------------------------------91// Atomically adds `value` to `obj` and returns the previous value92 93static inline int94_Py_atomic_add_int(int *obj, int value);95 96static inline int8_t97_Py_atomic_add_int8(int8_t *obj, int8_t value);98 99static inline int16_t100_Py_atomic_add_int16(int16_t *obj, int16_t value);101 102static inline int32_t103_Py_atomic_add_int32(int32_t *obj, int32_t value);104 105static inline int64_t106_Py_atomic_add_int64(int64_t *obj, int64_t value);107 108static inline intptr_t109_Py_atomic_add_intptr(intptr_t *obj, intptr_t value);110 111static inline unsigned int112_Py_atomic_add_uint(unsigned int *obj, unsigned int value);113 114static inline uint8_t115_Py_atomic_add_uint8(uint8_t *obj, uint8_t value);116 117static inline uint16_t118_Py_atomic_add_uint16(uint16_t *obj, uint16_t value);119 120static inline uint32_t121_Py_atomic_add_uint32(uint32_t *obj, uint32_t value);122 123static inline uint64_t124_Py_atomic_add_uint64(uint64_t *obj, uint64_t value);125 126static inline uintptr_t127_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value);128 129static inline Py_ssize_t130_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value);131 132 133// --- _Py_atomic_compare_exchange -------------------------------------------134// Performs an atomic compare-and-exchange.135//136// - If `*obj` and `*expected` are equal, store `desired` into `*obj`137//   and return 1 (success).138// - Otherwise, store the `*obj` current value into `*expected`139//   and return 0 (failure).140//141// These correspond to the C11 atomic_compare_exchange_strong() function.142 143static inline int144_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired);145 146static inline int147_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired);148 149static inline int150_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired);151 152static inline int153_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired);154 155static inline int156_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired);157 158static inline int159_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired);160 161static inline int162_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired);163 164static inline int165_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired);166 167static inline int168_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired);169 170static inline int171_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired);172 173static inline int174_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired);175 176static inline int177_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired);178 179static inline int180_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired);181 182// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*`183// so that we can pass types like `PyObject**` without a cast.184static inline int185_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value);186 187 188// --- _Py_atomic_exchange ---------------------------------------------------189// Atomically replaces `*obj` with `value` and returns the previous value of `*obj`.190 191static inline int192_Py_atomic_exchange_int(int *obj, int value);193 194static inline int8_t195_Py_atomic_exchange_int8(int8_t *obj, int8_t value);196 197static inline int16_t198_Py_atomic_exchange_int16(int16_t *obj, int16_t value);199 200static inline int32_t201_Py_atomic_exchange_int32(int32_t *obj, int32_t value);202 203static inline int64_t204_Py_atomic_exchange_int64(int64_t *obj, int64_t value);205 206static inline intptr_t207_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value);208 209static inline unsigned int210_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value);211 212static inline uint8_t213_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value);214 215static inline uint16_t216_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value);217 218static inline uint32_t219_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value);220 221static inline uint64_t222_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value);223 224static inline uintptr_t225_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value);226 227static inline Py_ssize_t228_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value);229 230static inline void *231_Py_atomic_exchange_ptr(void *obj, void *value);232 233 234// --- _Py_atomic_and --------------------------------------------------------235// Performs `*obj &= value` atomically and returns the previous value of `*obj`.236 237static inline uint8_t238_Py_atomic_and_uint8(uint8_t *obj, uint8_t value);239 240static inline uint16_t241_Py_atomic_and_uint16(uint16_t *obj, uint16_t value);242 243static inline uint32_t244_Py_atomic_and_uint32(uint32_t *obj, uint32_t value);245 246static inline uint64_t247_Py_atomic_and_uint64(uint64_t *obj, uint64_t value);248 249static inline uintptr_t250_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value);251 252 253// --- _Py_atomic_or ---------------------------------------------------------254// Performs `*obj |= value` atomically and returns the previous value of `*obj`.255 256static inline uint8_t257_Py_atomic_or_uint8(uint8_t *obj, uint8_t value);258 259static inline uint16_t260_Py_atomic_or_uint16(uint16_t *obj, uint16_t value);261 262static inline uint32_t263_Py_atomic_or_uint32(uint32_t *obj, uint32_t value);264 265static inline uint64_t266_Py_atomic_or_uint64(uint64_t *obj, uint64_t value);267 268static inline uintptr_t269_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value);270 271 272// --- _Py_atomic_load -------------------------------------------------------273// Atomically loads `*obj` (sequential consistency)274 275static inline int276_Py_atomic_load_int(const int *obj);277 278static inline int8_t279_Py_atomic_load_int8(const int8_t *obj);280 281static inline int16_t282_Py_atomic_load_int16(const int16_t *obj);283 284static inline int32_t285_Py_atomic_load_int32(const int32_t *obj);286 287static inline int64_t288_Py_atomic_load_int64(const int64_t *obj);289 290static inline intptr_t291_Py_atomic_load_intptr(const intptr_t *obj);292 293static inline uint8_t294_Py_atomic_load_uint8(const uint8_t *obj);295 296static inline uint16_t297_Py_atomic_load_uint16(const uint16_t *obj);298 299static inline uint32_t300_Py_atomic_load_uint32(const uint32_t *obj);301 302static inline uint64_t303_Py_atomic_load_uint64(const uint64_t *obj);304 305static inline uintptr_t306_Py_atomic_load_uintptr(const uintptr_t *obj);307 308static inline unsigned int309_Py_atomic_load_uint(const unsigned int *obj);310 311static inline Py_ssize_t312_Py_atomic_load_ssize(const Py_ssize_t *obj);313 314static inline void *315_Py_atomic_load_ptr(const void *obj);316 317 318// --- _Py_atomic_load_relaxed -----------------------------------------------319// Loads `*obj` (relaxed consistency, i.e., no ordering)320 321static inline int322_Py_atomic_load_int_relaxed(const int *obj);323 324static inline char325_Py_atomic_load_char_relaxed(const char *obj);326 327static inline unsigned char328_Py_atomic_load_uchar_relaxed(const unsigned char *obj);329 330static inline short331_Py_atomic_load_short_relaxed(const short *obj);332 333static inline unsigned short334_Py_atomic_load_ushort_relaxed(const unsigned short *obj);335 336static inline long337_Py_atomic_load_long_relaxed(const long *obj);338 339static inline double340_Py_atomic_load_double_relaxed(const double *obj);341 342static inline long long343_Py_atomic_load_llong_relaxed(const long long *obj);344 345static inline int8_t346_Py_atomic_load_int8_relaxed(const int8_t *obj);347 348static inline int16_t349_Py_atomic_load_int16_relaxed(const int16_t *obj);350 351static inline int32_t352_Py_atomic_load_int32_relaxed(const int32_t *obj);353 354static inline int64_t355_Py_atomic_load_int64_relaxed(const int64_t *obj);356 357static inline intptr_t358_Py_atomic_load_intptr_relaxed(const intptr_t *obj);359 360static inline uint8_t361_Py_atomic_load_uint8_relaxed(const uint8_t *obj);362 363static inline uint16_t364_Py_atomic_load_uint16_relaxed(const uint16_t *obj);365 366static inline uint32_t367_Py_atomic_load_uint32_relaxed(const uint32_t *obj);368 369static inline uint64_t370_Py_atomic_load_uint64_relaxed(const uint64_t *obj);371 372static inline uintptr_t373_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj);374 375static inline unsigned int376_Py_atomic_load_uint_relaxed(const unsigned int *obj);377 378static inline Py_ssize_t379_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj);380 381static inline void *382_Py_atomic_load_ptr_relaxed(const void *obj);383 384static inline unsigned long long385_Py_atomic_load_ullong_relaxed(const unsigned long long *obj);386 387// --- _Py_atomic_store ------------------------------------------------------388// Atomically performs `*obj = value` (sequential consistency)389 390static inline void391_Py_atomic_store_int(int *obj, int value);392 393static inline void394_Py_atomic_store_int8(int8_t *obj, int8_t value);395 396static inline void397_Py_atomic_store_int16(int16_t *obj, int16_t value);398 399static inline void400_Py_atomic_store_int32(int32_t *obj, int32_t value);401 402static inline void403_Py_atomic_store_int64(int64_t *obj, int64_t value);404 405static inline void406_Py_atomic_store_intptr(intptr_t *obj, intptr_t value);407 408static inline void409_Py_atomic_store_uint8(uint8_t *obj, uint8_t value);410 411static inline void412_Py_atomic_store_uint16(uint16_t *obj, uint16_t value);413 414static inline void415_Py_atomic_store_uint32(uint32_t *obj, uint32_t value);416 417static inline void418_Py_atomic_store_uint64(uint64_t *obj, uint64_t value);419 420static inline void421_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value);422 423static inline void424_Py_atomic_store_uint(unsigned int *obj, unsigned int value);425 426static inline void427_Py_atomic_store_ptr(void *obj, void *value);428 429static inline void430_Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value);431 432 433// --- _Py_atomic_store_relaxed ----------------------------------------------434// Stores `*obj = value` (relaxed consistency, i.e., no ordering)435 436static inline void437_Py_atomic_store_int_relaxed(int *obj, int value);438 439static inline void440_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value);441 442static inline void443_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value);444 445static inline void446_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value);447 448static inline void449_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value);450 451static inline void452_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value);453 454static inline void455_Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value);456 457static inline void458_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value);459 460static inline void461_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value);462 463static inline void464_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value);465 466static inline void467_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value);468 469static inline void470_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value);471 472static inline void473_Py_atomic_store_ptr_relaxed(void *obj, void *value);474 475static inline void476_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);477 478static inline void479_Py_atomic_store_ullong_relaxed(unsigned long long *obj,480                                unsigned long long value);481 482static inline void483_Py_atomic_store_char_relaxed(char *obj, char value);484 485static inline void486_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value);487 488static inline void489_Py_atomic_store_short_relaxed(short *obj, short value);490 491static inline void492_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value);493 494static inline void495_Py_atomic_store_long_relaxed(long *obj, long value);496 497static inline void498_Py_atomic_store_float_relaxed(float *obj, float value);499 500static inline void501_Py_atomic_store_double_relaxed(double *obj, double value);502 503static inline void504_Py_atomic_store_llong_relaxed(long long *obj, long long value);505 506 507// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------508 509// Loads `*obj` (acquire operation)510static inline void *511_Py_atomic_load_ptr_acquire(const void *obj);512 513static inline uintptr_t514_Py_atomic_load_uintptr_acquire(const uintptr_t *obj);515 516// Stores `*obj = value` (release operation)517static inline void518_Py_atomic_store_ptr_release(void *obj, void *value);519 520static inline void521_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value);522 523static inline void524_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value);525 526static inline void527_Py_atomic_store_int_release(int *obj, int value);528 529static inline int530_Py_atomic_load_int_acquire(const int *obj);531 532static inline void533_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value);534 535static inline void536_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value);537 538static inline void539_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value);540 541static inline uint64_t542_Py_atomic_load_uint64_acquire(const uint64_t *obj);543 544static inline uint32_t545_Py_atomic_load_uint32_acquire(const uint32_t *obj);546 547static inline Py_ssize_t548_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj);549 550 551 552 553// --- _Py_atomic_fence ------------------------------------------------------554 555// Sequential consistency fence. C11 fences have complex semantics. When556// possible, use the atomic operations on variables defined above, which557// generally do not require explicit use of a fence.558// See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence559static inline void _Py_atomic_fence_seq_cst(void);560 561// Acquire fence562static inline void _Py_atomic_fence_acquire(void);563 564// Release fence565static inline void _Py_atomic_fence_release(void);566 567 568#ifndef _Py_USE_GCC_BUILTIN_ATOMICS569#  if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))570#    define _Py_USE_GCC_BUILTIN_ATOMICS 1571#  elif defined(__clang__)572#    if __has_builtin(__atomic_load)573#      define _Py_USE_GCC_BUILTIN_ATOMICS 1574#    endif575#  endif576#endif577 578#if _Py_USE_GCC_BUILTIN_ATOMICS579#  define Py_ATOMIC_GCC_H580#  include "pyatomic_gcc.h"581#  undef Py_ATOMIC_GCC_H582#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)583#  define Py_ATOMIC_STD_H584#  include "pyatomic_std.h"585#  undef Py_ATOMIC_STD_H586#elif defined(_MSC_VER)587#  define Py_ATOMIC_MSC_H588#  include "pyatomic_msc.h"589#  undef Py_ATOMIC_MSC_H590#else591#  error "no available pyatomic implementation for this platform/compiler"592#endif593 594 595// --- aliases ---------------------------------------------------------------596 597// Compilers don't really support "consume" semantics, so we fake it. Use598// "acquire" with TSan to support false positives. Use "relaxed" otherwise,599// because CPUs on all platforms we support respect address dependencies without600// extra barriers.601// See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf602#if defined(_Py_THREAD_SANITIZER)603# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire604#else605# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed606#endif607 608#if SIZEOF_LONG == 8609# define _Py_atomic_load_ulong(p) \610    _Py_atomic_load_uint64((uint64_t *)p)611# define _Py_atomic_load_ulong_relaxed(p) \612    _Py_atomic_load_uint64_relaxed((uint64_t *)p)613# define _Py_atomic_store_ulong(p, v) \614    _Py_atomic_store_uint64((uint64_t *)p, v)615# define _Py_atomic_store_ulong_relaxed(p, v) \616    _Py_atomic_store_uint64_relaxed((uint64_t *)p, v)617#elif SIZEOF_LONG == 4618# define _Py_atomic_load_ulong(p) \619    _Py_atomic_load_uint32((uint32_t *)p)620# define _Py_atomic_load_ulong_relaxed(p) \621    _Py_atomic_load_uint32_relaxed((uint32_t *)p)622# define _Py_atomic_store_ulong(p, v) \623    _Py_atomic_store_uint32((uint32_t *)p, v)624# define _Py_atomic_store_ulong_relaxed(p, v) \625    _Py_atomic_store_uint32_relaxed((uint32_t *)p, v)626#else627# error "long must be 4 or 8 bytes in size"628#endif  // SIZEOF_LONG629