codekingpro/portable-devtools
115k
1// This is the implementation of Python atomic operations for MSVC if the2// compiler does not support C11 or C++11 atomics.3//4// MSVC intrinsics are defined on char, short, long, __int64, and pointer5// types. Note that long and int are both 32-bits even on 64-bit Windows,6// so operations on int are cast to long.7//8// The volatile keyword has additional memory ordering semantics on MSVC. On9// x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64,10// volatile accesses behave like C11's memory_order_relaxed.11 12#ifndef Py_ATOMIC_MSC_H13# error "this header file must not be included directly"14#endif15 16#include <intrin.h>17 18#define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \19 Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE))20 21 22// --- _Py_atomic_add --------------------------------------------------------23 24static inline int8_t25_Py_atomic_add_int8(int8_t *obj, int8_t value)26{27 _Py_atomic_ASSERT_ARG_TYPE(char);28 return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value);29}30 31static inline int16_t32_Py_atomic_add_int16(int16_t *obj, int16_t value)33{34 _Py_atomic_ASSERT_ARG_TYPE(short);35 return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value);36}37 38static inline int32_t39_Py_atomic_add_int32(int32_t *obj, int32_t value)40{41 _Py_atomic_ASSERT_ARG_TYPE(long);42 return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value);43}44 45static inline int64_t46_Py_atomic_add_int64(int64_t *obj, int64_t value)47{48#if defined(_M_X64) || defined(_M_ARM64)49 _Py_atomic_ASSERT_ARG_TYPE(__int64);50 return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value);51#else52 int64_t old_value = _Py_atomic_load_int64_relaxed(obj);53 for (;;) {54 int64_t new_value = old_value + value;55 if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) {56 return old_value;57 }58 }59#endif60}61 62 63static inline uint8_t64_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)65{66 return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value);67}68 69static inline uint16_t70_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)71{72 return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value);73}74 75static inline uint32_t76_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)77{78 return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);79}80 81static inline int82_Py_atomic_add_int(int *obj, int value)83{84 _Py_atomic_ASSERT_ARG_TYPE(int32_t);85 return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);86}87 88static inline unsigned int89_Py_atomic_add_uint(unsigned int *obj, unsigned int value)90{91 _Py_atomic_ASSERT_ARG_TYPE(int32_t);92 return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);93}94 95static inline uint64_t96_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)97{98 return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);99}100 101static inline intptr_t102_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)103{104#if SIZEOF_VOID_P == 8105 _Py_atomic_ASSERT_ARG_TYPE(int64_t);106 return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);107#else108 _Py_atomic_ASSERT_ARG_TYPE(int32_t);109 return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);110#endif111}112 113static inline uintptr_t114_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)115{116 _Py_atomic_ASSERT_ARG_TYPE(intptr_t);117 return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);118}119 120static inline Py_ssize_t121_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)122{123 _Py_atomic_ASSERT_ARG_TYPE(intptr_t);124 return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);125}126 127 128// --- _Py_atomic_compare_exchange -------------------------------------------129 130static inline int131_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value)132{133 _Py_atomic_ASSERT_ARG_TYPE(char);134 int8_t initial = (int8_t)_InterlockedCompareExchange8(135 (volatile char *)obj,136 (char)value,137 (char)*expected);138 if (initial == *expected) {139 return 1;140 }141 *expected = initial;142 return 0;143}144 145static inline int146_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value)147{148 _Py_atomic_ASSERT_ARG_TYPE(short);149 int16_t initial = (int16_t)_InterlockedCompareExchange16(150 (volatile short *)obj,151 (short)value,152 (short)*expected);153 if (initial == *expected) {154 return 1;155 }156 *expected = initial;157 return 0;158}159 160static inline int161_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value)162{163 _Py_atomic_ASSERT_ARG_TYPE(long);164 int32_t initial = (int32_t)_InterlockedCompareExchange(165 (volatile long *)obj,166 (long)value,167 (long)*expected);168 if (initial == *expected) {169 return 1;170 }171 *expected = initial;172 return 0;173}174 175static inline int176_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value)177{178 _Py_atomic_ASSERT_ARG_TYPE(__int64);179 int64_t initial = (int64_t)_InterlockedCompareExchange64(180 (volatile __int64 *)obj,181 (__int64)value,182 (__int64)*expected);183 if (initial == *expected) {184 return 1;185 }186 *expected = initial;187 return 0;188}189 190static inline int191_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value)192{193 void *initial = _InterlockedCompareExchangePointer(194 (void**)obj,195 value,196 *(void**)expected);197 if (initial == *(void**)expected) {198 return 1;199 }200 *(void**)expected = initial;201 return 0;202}203 204 205static inline int206_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value)207{208 return _Py_atomic_compare_exchange_int8((int8_t *)obj,209 (int8_t *)expected,210 (int8_t)value);211}212 213static inline int214_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value)215{216 return _Py_atomic_compare_exchange_int16((int16_t *)obj,217 (int16_t *)expected,218 (int16_t)value);219}220 221static inline int222_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value)223{224 return _Py_atomic_compare_exchange_int32((int32_t *)obj,225 (int32_t *)expected,226 (int32_t)value);227}228 229static inline int230_Py_atomic_compare_exchange_int(int *obj, int *expected, int value)231{232 _Py_atomic_ASSERT_ARG_TYPE(int32_t);233 return _Py_atomic_compare_exchange_int32((int32_t *)obj,234 (int32_t *)expected,235 (int32_t)value);236}237 238static inline int239_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value)240{241 _Py_atomic_ASSERT_ARG_TYPE(int32_t);242 return _Py_atomic_compare_exchange_int32((int32_t *)obj,243 (int32_t *)expected,244 (int32_t)value);245}246 247static inline int248_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value)249{250 return _Py_atomic_compare_exchange_int64((int64_t *)obj,251 (int64_t *)expected,252 (int64_t)value);253}254 255static inline int256_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value)257{258 _Py_atomic_ASSERT_ARG_TYPE(void*);259 return _Py_atomic_compare_exchange_ptr((void**)obj,260 (void**)expected,261 (void*)value);262}263 264static inline int265_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value)266{267 _Py_atomic_ASSERT_ARG_TYPE(void*);268 return _Py_atomic_compare_exchange_ptr((void**)obj,269 (void**)expected,270 (void*)value);271}272 273static inline int274_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value)275{276 _Py_atomic_ASSERT_ARG_TYPE(void*);277 return _Py_atomic_compare_exchange_ptr((void**)obj,278 (void**)expected,279 (void*)value);280}281 282 283// --- _Py_atomic_exchange ---------------------------------------------------284 285static inline int8_t286_Py_atomic_exchange_int8(int8_t *obj, int8_t value)287{288 _Py_atomic_ASSERT_ARG_TYPE(char);289 return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value);290}291 292static inline int16_t293_Py_atomic_exchange_int16(int16_t *obj, int16_t value)294{295 _Py_atomic_ASSERT_ARG_TYPE(short);296 return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value);297}298 299static inline int32_t300_Py_atomic_exchange_int32(int32_t *obj, int32_t value)301{302 _Py_atomic_ASSERT_ARG_TYPE(long);303 return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value);304}305 306static inline int64_t307_Py_atomic_exchange_int64(int64_t *obj, int64_t value)308{309#if defined(_M_X64) || defined(_M_ARM64)310 _Py_atomic_ASSERT_ARG_TYPE(__int64);311 return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value);312#else313 int64_t old_value = _Py_atomic_load_int64_relaxed(obj);314 for (;;) {315 if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) {316 return old_value;317 }318 }319#endif320}321 322static inline void*323_Py_atomic_exchange_ptr(void *obj, void *value)324{325 return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value);326}327 328 329static inline uint8_t330_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)331{332 return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj,333 (int8_t)value);334}335 336static inline uint16_t337_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)338{339 return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj,340 (int16_t)value);341}342 343static inline uint32_t344_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)345{346 return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj,347 (int32_t)value);348}349 350static inline int351_Py_atomic_exchange_int(int *obj, int value)352{353 _Py_atomic_ASSERT_ARG_TYPE(int32_t);354 return (int)_Py_atomic_exchange_int32((int32_t *)obj,355 (int32_t)value);356}357 358static inline unsigned int359_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)360{361 _Py_atomic_ASSERT_ARG_TYPE(int32_t);362 return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj,363 (int32_t)value);364}365 366static inline uint64_t367_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)368{369 return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj,370 (int64_t)value);371}372 373static inline intptr_t374_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)375{376 _Py_atomic_ASSERT_ARG_TYPE(void*);377 return (intptr_t)_Py_atomic_exchange_ptr((void**)obj,378 (void*)value);379}380 381static inline uintptr_t382_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)383{384 _Py_atomic_ASSERT_ARG_TYPE(void*);385 return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj,386 (void*)value);387}388 389static inline Py_ssize_t390_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)391{392 _Py_atomic_ASSERT_ARG_TYPE(void*);393 return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj,394 (void*)value);395}396 397 398// --- _Py_atomic_and --------------------------------------------------------399 400static inline uint8_t401_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)402{403 _Py_atomic_ASSERT_ARG_TYPE(char);404 return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value);405}406 407static inline uint16_t408_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)409{410 _Py_atomic_ASSERT_ARG_TYPE(short);411 return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value);412}413 414static inline uint32_t415_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)416{417 _Py_atomic_ASSERT_ARG_TYPE(long);418 return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value);419}420 421static inline uint64_t422_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)423{424#if defined(_M_X64) || defined(_M_ARM64)425 _Py_atomic_ASSERT_ARG_TYPE(__int64);426 return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value);427#else428 uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);429 for (;;) {430 uint64_t new_value = old_value & value;431 if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {432 return old_value;433 }434 }435#endif436}437 438static inline uintptr_t439_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)440{441#if SIZEOF_VOID_P == 8442 _Py_atomic_ASSERT_ARG_TYPE(uint64_t);443 return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj,444 (uint64_t)value);445#else446 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);447 return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj,448 (uint32_t)value);449#endif450}451 452 453// --- _Py_atomic_or ---------------------------------------------------------454 455static inline uint8_t456_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)457{458 _Py_atomic_ASSERT_ARG_TYPE(char);459 return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value);460}461 462static inline uint16_t463_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)464{465 _Py_atomic_ASSERT_ARG_TYPE(short);466 return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value);467}468 469static inline uint32_t470_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)471{472 _Py_atomic_ASSERT_ARG_TYPE(long);473 return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value);474}475 476static inline uint64_t477_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)478{479#if defined(_M_X64) || defined(_M_ARM64)480 _Py_atomic_ASSERT_ARG_TYPE(__int64);481 return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value);482#else483 uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);484 for (;;) {485 uint64_t new_value = old_value | value;486 if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {487 return old_value;488 }489 }490#endif491}492 493 494static inline uintptr_t495_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)496{497#if SIZEOF_VOID_P == 8498 _Py_atomic_ASSERT_ARG_TYPE(uint64_t);499 return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj,500 (uint64_t)value);501#else502 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);503 return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj,504 (uint32_t)value);505#endif506}507 508 509// --- _Py_atomic_load -------------------------------------------------------510 511static inline uint8_t512_Py_atomic_load_uint8(const uint8_t *obj)513{514#if defined(_M_X64) || defined(_M_IX86)515 return *(volatile uint8_t *)obj;516#elif defined(_M_ARM64)517 return (uint8_t)__ldar8((unsigned __int8 volatile *)obj);518#else519# error "no implementation of _Py_atomic_load_uint8"520#endif521}522 523static inline uint16_t524_Py_atomic_load_uint16(const uint16_t *obj)525{526#if defined(_M_X64) || defined(_M_IX86)527 return *(volatile uint16_t *)obj;528#elif defined(_M_ARM64)529 return (uint16_t)__ldar16((unsigned __int16 volatile *)obj);530#else531# error "no implementation of _Py_atomic_load_uint16"532#endif533}534 535static inline uint32_t536_Py_atomic_load_uint32(const uint32_t *obj)537{538#if defined(_M_X64) || defined(_M_IX86)539 return *(volatile uint32_t *)obj;540#elif defined(_M_ARM64)541 return (uint32_t)__ldar32((unsigned __int32 volatile *)obj);542#else543# error "no implementation of _Py_atomic_load_uint32"544#endif545}546 547static inline uint64_t548_Py_atomic_load_uint64(const uint64_t *obj)549{550#if defined(_M_X64) || defined(_M_IX86)551 return *(volatile uint64_t *)obj;552#elif defined(_M_ARM64)553 return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);554#else555# error "no implementation of _Py_atomic_load_uint64"556#endif557}558 559static inline int8_t560_Py_atomic_load_int8(const int8_t *obj)561{562 return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj);563}564 565static inline int16_t566_Py_atomic_load_int16(const int16_t *obj)567{568 return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj);569}570 571static inline int32_t572_Py_atomic_load_int32(const int32_t *obj)573{574 return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj);575}576 577static inline int578_Py_atomic_load_int(const int *obj)579{580 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);581 return (int)_Py_atomic_load_uint32((uint32_t *)obj);582}583 584static inline unsigned int585_Py_atomic_load_uint(const unsigned int *obj)586{587 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);588 return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj);589}590 591static inline int64_t592_Py_atomic_load_int64(const int64_t *obj)593{594 return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj);595}596 597static inline void*598_Py_atomic_load_ptr(const void *obj)599{600#if SIZEOF_VOID_P == 8601 return (void*)_Py_atomic_load_uint64((const uint64_t *)obj);602#else603 return (void*)_Py_atomic_load_uint32((const uint32_t *)obj);604#endif605}606 607static inline intptr_t608_Py_atomic_load_intptr(const intptr_t *obj)609{610 _Py_atomic_ASSERT_ARG_TYPE(void*);611 return (intptr_t)_Py_atomic_load_ptr((void*)obj);612}613 614static inline uintptr_t615_Py_atomic_load_uintptr(const uintptr_t *obj)616{617 _Py_atomic_ASSERT_ARG_TYPE(void*);618 return (uintptr_t)_Py_atomic_load_ptr((void*)obj);619}620 621static inline Py_ssize_t622_Py_atomic_load_ssize(const Py_ssize_t *obj)623{624 _Py_atomic_ASSERT_ARG_TYPE(void*);625 return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj);626}627 628 629// --- _Py_atomic_load_relaxed -----------------------------------------------630 631static inline int632_Py_atomic_load_int_relaxed(const int *obj)633{634 return *(volatile int *)obj;635}636 637static inline char638_Py_atomic_load_char_relaxed(const char *obj)639{640 return *(volatile char *)obj;641}642 643static inline unsigned char644_Py_atomic_load_uchar_relaxed(const unsigned char *obj)645{646 return *(volatile unsigned char *)obj;647}648 649static inline short650_Py_atomic_load_short_relaxed(const short *obj)651{652 return *(volatile short *)obj;653}654 655static inline unsigned short656_Py_atomic_load_ushort_relaxed(const unsigned short *obj)657{658 return *(volatile unsigned short *)obj;659}660 661static inline long662_Py_atomic_load_long_relaxed(const long *obj)663{664 return *(volatile long *)obj;665}666 667static inline float668_Py_atomic_load_float_relaxed(const float *obj)669{670 return *(volatile float *)obj;671}672 673static inline double674_Py_atomic_load_double_relaxed(const double *obj)675{676 return *(volatile double *)obj;677}678 679static inline int8_t680_Py_atomic_load_int8_relaxed(const int8_t *obj)681{682 return *(volatile int8_t *)obj;683}684 685static inline int16_t686_Py_atomic_load_int16_relaxed(const int16_t *obj)687{688 return *(volatile int16_t *)obj;689}690 691static inline int32_t692_Py_atomic_load_int32_relaxed(const int32_t *obj)693{694 return *(volatile int32_t *)obj;695}696 697static inline int64_t698_Py_atomic_load_int64_relaxed(const int64_t *obj)699{700 return *(volatile int64_t *)obj;701}702 703static inline intptr_t704_Py_atomic_load_intptr_relaxed(const intptr_t *obj)705{706 return *(volatile intptr_t *)obj;707}708 709static inline uint8_t710_Py_atomic_load_uint8_relaxed(const uint8_t *obj)711{712 return *(volatile uint8_t *)obj;713}714 715static inline uint16_t716_Py_atomic_load_uint16_relaxed(const uint16_t *obj)717{718 return *(volatile uint16_t *)obj;719}720 721static inline uint32_t722_Py_atomic_load_uint32_relaxed(const uint32_t *obj)723{724 return *(volatile uint32_t *)obj;725}726 727static inline uint64_t728_Py_atomic_load_uint64_relaxed(const uint64_t *obj)729{730 return *(volatile uint64_t *)obj;731}732 733static inline uintptr_t734_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)735{736 return *(volatile uintptr_t *)obj;737}738 739static inline unsigned int740_Py_atomic_load_uint_relaxed(const unsigned int *obj)741{742 return *(volatile unsigned int *)obj;743}744 745static inline Py_ssize_t746_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)747{748 return *(volatile Py_ssize_t *)obj;749}750 751static inline void*752_Py_atomic_load_ptr_relaxed(const void *obj)753{754 return *(void * volatile *)obj;755}756 757static inline unsigned long long758_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)759{760 return *(volatile unsigned long long *)obj;761}762 763static inline long long764_Py_atomic_load_llong_relaxed(const long long *obj)765{766 return *(volatile long long *)obj;767}768 769 770// --- _Py_atomic_store ------------------------------------------------------771 772static inline void773_Py_atomic_store_int(int *obj, int value)774{775 (void)_Py_atomic_exchange_int(obj, value);776}777 778static inline void779_Py_atomic_store_int8(int8_t *obj, int8_t value)780{781 (void)_Py_atomic_exchange_int8(obj, value);782}783 784static inline void785_Py_atomic_store_int16(int16_t *obj, int16_t value)786{787 (void)_Py_atomic_exchange_int16(obj, value);788}789 790static inline void791_Py_atomic_store_int32(int32_t *obj, int32_t value)792{793 (void)_Py_atomic_exchange_int32(obj, value);794}795 796static inline void797_Py_atomic_store_int64(int64_t *obj, int64_t value)798{799 (void)_Py_atomic_exchange_int64(obj, value);800}801 802static inline void803_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)804{805 (void)_Py_atomic_exchange_intptr(obj, value);806}807 808static inline void809_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)810{811 (void)_Py_atomic_exchange_uint8(obj, value);812}813 814static inline void815_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)816{817 (void)_Py_atomic_exchange_uint16(obj, value);818}819 820static inline void821_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)822{823 (void)_Py_atomic_exchange_uint32(obj, value);824}825 826static inline void827_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)828{829 (void)_Py_atomic_exchange_uint64(obj, value);830}831 832static inline void833_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)834{835 (void)_Py_atomic_exchange_uintptr(obj, value);836}837 838static inline void839_Py_atomic_store_uint(unsigned int *obj, unsigned int value)840{841 (void)_Py_atomic_exchange_uint(obj, value);842}843 844static inline void845_Py_atomic_store_ptr(void *obj, void *value)846{847 (void)_Py_atomic_exchange_ptr(obj, value);848}849 850static inline void851_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)852{853 (void)_Py_atomic_exchange_ssize(obj, value);854}855 856 857// --- _Py_atomic_store_relaxed ----------------------------------------------858 859static inline void860_Py_atomic_store_int_relaxed(int *obj, int value)861{862 *(volatile int *)obj = value;863}864 865static inline void866_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)867{868 *(volatile int8_t *)obj = value;869}870 871static inline void872_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)873{874 *(volatile int16_t *)obj = value;875}876 877static inline void878_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)879{880 *(volatile int32_t *)obj = value;881}882 883static inline void884_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)885{886 *(volatile int64_t *)obj = value;887}888 889static inline void890_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)891{892 *(volatile intptr_t *)obj = value;893}894 895static inline void896_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)897{898 *(volatile uint8_t *)obj = value;899}900 901static inline void902_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)903{904 *(volatile uint16_t *)obj = value;905}906 907static inline void908_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)909{910 *(volatile uint32_t *)obj = value;911}912 913static inline void914_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)915{916 *(volatile uint64_t *)obj = value;917}918 919static inline void920_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)921{922 *(volatile uintptr_t *)obj = value;923}924 925static inline void926_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)927{928 *(volatile unsigned int *)obj = value;929}930 931static inline void932_Py_atomic_store_ptr_relaxed(void *obj, void* value)933{934 *(void * volatile *)obj = value;935}936 937static inline void938_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)939{940 *(volatile Py_ssize_t *)obj = value;941}942 943static inline void944_Py_atomic_store_ullong_relaxed(unsigned long long *obj,945 unsigned long long value)946{947 *(volatile unsigned long long *)obj = value;948}949 950static inline void951_Py_atomic_store_char_relaxed(char *obj, char value)952{953 *(volatile char *)obj = value;954}955 956static inline void957_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value)958{959 *(volatile unsigned char *)obj = value;960}961 962static inline void963_Py_atomic_store_short_relaxed(short *obj, short value)964{965 *(volatile short *)obj = value;966}967 968static inline void969_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value)970{971 *(volatile unsigned short *)obj = value;972}973 974static inline void975_Py_atomic_store_long_relaxed(long *obj, long value)976{977 *(volatile long *)obj = value;978}979 980static inline void981_Py_atomic_store_float_relaxed(float *obj, float value)982{983 *(volatile float *)obj = value;984}985 986static inline void987_Py_atomic_store_double_relaxed(double *obj, double value)988{989 *(volatile double *)obj = value;990}991 992static inline void993_Py_atomic_store_llong_relaxed(long long *obj, long long value)994{995 *(volatile long long *)obj = value;996}997 998 999// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------1000 1001static inline void *1002_Py_atomic_load_ptr_acquire(const void *obj)1003{1004#if defined(_M_X64) || defined(_M_IX86)1005 return *(void * volatile *)obj;1006#elif defined(_M_ARM64)1007 return (void *)__ldar64((unsigned __int64 volatile *)obj);1008#else1009# error "no implementation of _Py_atomic_load_ptr_acquire"1010#endif1011}1012 1013static inline uintptr_t1014_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)1015{1016#if defined(_M_X64) || defined(_M_IX86)1017 return *(uintptr_t volatile *)obj;1018#elif defined(_M_ARM64)1019 return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj);1020#else1021# error "no implementation of _Py_atomic_load_uintptr_acquire"1022#endif1023}1024 1025static inline void1026_Py_atomic_store_ptr_release(void *obj, void *value)1027{1028#if defined(_M_X64) || defined(_M_IX86)1029 *(void * volatile *)obj = value;1030#elif defined(_M_ARM64)1031 __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value);1032#else1033# error "no implementation of _Py_atomic_store_ptr_release"1034#endif1035}1036 1037static inline void1038_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)1039{1040#if defined(_M_X64) || defined(_M_IX86)1041 *(uintptr_t volatile *)obj = value;1042#elif defined(_M_ARM64)1043 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);1044 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);1045#else1046# error "no implementation of _Py_atomic_store_uintptr_release"1047#endif1048}1049 1050static inline void1051_Py_atomic_store_int_release(int *obj, int value)1052{1053#if defined(_M_X64) || defined(_M_IX86)1054 *(int volatile *)obj = value;1055#elif defined(_M_ARM64)1056 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);1057 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);1058#else1059# error "no implementation of _Py_atomic_store_int_release"1060#endif1061}1062 1063static inline void1064_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value)1065{1066#if defined(_M_X64) || defined(_M_IX86)1067 *(volatile unsigned int *)obj = value;1068#elif defined(_M_ARM64)1069 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);1070 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);1071#else1072# error "no implementation of _Py_atomic_store_uint_release"1073#endif1074}1075 1076static inline void1077_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)1078{1079#if defined(_M_X64) || defined(_M_IX86)1080 *(Py_ssize_t volatile *)obj = value;1081#elif defined(_M_ARM64)1082 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);1083#else1084# error "no implementation of _Py_atomic_store_ssize_release"1085#endif1086}1087 1088static inline int1089_Py_atomic_load_int_acquire(const int *obj)1090{1091#if defined(_M_X64) || defined(_M_IX86)1092 return *(int volatile *)obj;1093#elif defined(_M_ARM64)1094 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);1095 return (int)__ldar32((unsigned __int32 volatile *)obj);1096#else1097# error "no implementation of _Py_atomic_load_int_acquire"1098#endif1099}1100 1101static inline void1102_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)1103{1104#if defined(_M_X64) || defined(_M_IX86)1105 *(uint32_t volatile *)obj = value;1106#elif defined(_M_ARM64)1107 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);1108 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);1109#else1110# error "no implementation of _Py_atomic_store_uint32_release"1111#endif1112}1113 1114static inline void1115_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)1116{1117#if defined(_M_X64) || defined(_M_IX86)1118 *(uint64_t volatile *)obj = value;1119#elif defined(_M_ARM64)1120 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);1121 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);1122#else1123# error "no implementation of _Py_atomic_store_uint64_release"1124#endif1125}1126 1127static inline uint64_t1128_Py_atomic_load_uint64_acquire(const uint64_t *obj)1129{1130#if defined(_M_X64) || defined(_M_IX86)1131 return *(uint64_t volatile *)obj;1132#elif defined(_M_ARM64)1133 _Py_atomic_ASSERT_ARG_TYPE(__int64);1134 return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);1135#else1136# error "no implementation of _Py_atomic_load_uint64_acquire"1137#endif1138}1139 1140static inline uint32_t1141_Py_atomic_load_uint32_acquire(const uint32_t *obj)1142{1143#if defined(_M_X64) || defined(_M_IX86)1144 return *(uint32_t volatile *)obj;1145#elif defined(_M_ARM64)1146 return (uint32_t)__ldar32((uint32_t volatile *)obj);1147#else1148# error "no implementation of _Py_atomic_load_uint32_acquire"1149#endif1150}1151 1152static inline Py_ssize_t1153_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)1154{1155#if defined(_M_X64) || defined(_M_IX86)1156 return *(Py_ssize_t volatile *)obj;1157#elif defined(_M_ARM64)1158 return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj);1159#else1160# error "no implementation of _Py_atomic_load_ssize_acquire"1161#endif1162}1163 1164// --- _Py_atomic_fence ------------------------------------------------------1165 1166 static inline void1167_Py_atomic_fence_seq_cst(void)1168{1169#if defined(_M_ARM64)1170 __dmb(_ARM64_BARRIER_ISH);1171#elif defined(_M_X64)1172 __faststorefence();1173#elif defined(_M_IX86)1174 _mm_mfence();1175#else1176# error "no implementation of _Py_atomic_fence_seq_cst"1177#endif1178}1179 1180 static inline void1181_Py_atomic_fence_acquire(void)1182{1183#if defined(_M_ARM64)1184 __dmb(_ARM64_BARRIER_ISHLD);1185#elif defined(_M_X64) || defined(_M_IX86)1186 _ReadBarrier();1187#else1188# error "no implementation of _Py_atomic_fence_acquire"1189#endif1190}1191 1192 static inline void1193_Py_atomic_fence_release(void)1194{1195#if defined(_M_ARM64)1196 __dmb(_ARM64_BARRIER_ISH);1197#elif defined(_M_X64) || defined(_M_IX86)1198 _ReadWriteBarrier();1199#else1200# error "no implementation of _Py_atomic_fence_release"