codekingpro/portable-devtools
114k
1// -*- C++ -*-2//===--------------------------- atomic -----------------------------------===//3//4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5// See https://llvm.org/LICENSE.txt for license information.6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7//8//===----------------------------------------------------------------------===//9 10#ifndef _LIBCUDACXX_ATOMIC11#define _LIBCUDACXX_ATOMIC12 13/*14 atomic synopsis15 16namespace std17{18 19// feature test macro20 21#define __cpp_lib_atomic_is_always_lock_free // as specified by SG1022 23 // order and consistency24 25 enum memory_order: unspecified // enum class in C++2026 {27 relaxed,28 consume, // load-consume29 acquire, // load-acquire30 release, // store-release31 acq_rel, // store-release load-acquire32 seq_cst // store-release load-acquire33 };34 35 inline constexpr auto memory_order_relaxed = memory_order::relaxed;36 inline constexpr auto memory_order_consume = memory_order::consume;37 inline constexpr auto memory_order_acquire = memory_order::acquire;38 inline constexpr auto memory_order_release = memory_order::release;39 inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;40 inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;41 42template <class T> T kill_dependency(T y) noexcept;43 44// lock-free property45 46#define ATOMIC_BOOL_LOCK_FREE unspecified47#define ATOMIC_CHAR_LOCK_FREE unspecified48#define ATOMIC_CHAR16_T_LOCK_FREE unspecified49#define ATOMIC_CHAR32_T_LOCK_FREE unspecified50#define ATOMIC_WCHAR_T_LOCK_FREE unspecified51#define ATOMIC_SHORT_LOCK_FREE unspecified52#define ATOMIC_INT_LOCK_FREE unspecified53#define ATOMIC_LONG_LOCK_FREE unspecified54#define ATOMIC_LLONG_LOCK_FREE unspecified55#define ATOMIC_POINTER_LOCK_FREE unspecified56 57// flag type and operations58 59typedef struct atomic_flag60{61 bool test_and_set(memory_order m = memory_order_seq_cst) volatile noexcept;62 bool test_and_set(memory_order m = memory_order_seq_cst) noexcept;63 void clear(memory_order m = memory_order_seq_cst) volatile noexcept;64 void clear(memory_order m = memory_order_seq_cst) noexcept;65 atomic_flag() noexcept = default;66 atomic_flag(const atomic_flag&) = delete;67 atomic_flag& operator=(const atomic_flag&) = delete;68 atomic_flag& operator=(const atomic_flag&) volatile = delete;69} atomic_flag;70 71bool72 atomic_flag_test_and_set(volatile atomic_flag* obj) noexcept;73 74bool75 atomic_flag_test_and_set(atomic_flag* obj) noexcept;76 77bool78 atomic_flag_test_and_set_explicit(volatile atomic_flag* obj,79 memory_order m) noexcept;80 81bool82 atomic_flag_test_and_set_explicit(atomic_flag* obj, memory_order m) noexcept;83 84void85 atomic_flag_clear(volatile atomic_flag* obj) noexcept;86 87void88 atomic_flag_clear(atomic_flag* obj) noexcept;89 90void91 atomic_flag_clear_explicit(volatile atomic_flag* obj, memory_order m) noexcept;92 93void94 atomic_flag_clear_explicit(atomic_flag* obj, memory_order m) noexcept;95 96#define ATOMIC_FLAG_INIT see below97#define ATOMIC_VAR_INIT(value) see below98 99template <class T>100struct atomic101{102 static constexpr bool is_always_lock_free;103 bool is_lock_free() const volatile noexcept;104 bool is_lock_free() const noexcept;105 void store(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;106 void store(T desr, memory_order m = memory_order_seq_cst) noexcept;107 T load(memory_order m = memory_order_seq_cst) const volatile noexcept;108 T load(memory_order m = memory_order_seq_cst) const noexcept;109 operator T() const volatile noexcept;110 operator T() const noexcept;111 T exchange(T desr, memory_order m = memory_order_seq_cst) volatile noexcept;112 T exchange(T desr, memory_order m = memory_order_seq_cst) noexcept;113 bool compare_exchange_weak(T& expc, T desr,114 memory_order s, memory_order f) volatile noexcept;115 bool compare_exchange_weak(T& expc, T desr, memory_order s, memory_order f) noexcept;116 bool compare_exchange_strong(T& expc, T desr,117 memory_order s, memory_order f) volatile noexcept;118 bool compare_exchange_strong(T& expc, T desr,119 memory_order s, memory_order f) noexcept;120 bool compare_exchange_weak(T& expc, T desr,121 memory_order m = memory_order_seq_cst) volatile noexcept;122 bool compare_exchange_weak(T& expc, T desr,123 memory_order m = memory_order_seq_cst) noexcept;124 bool compare_exchange_strong(T& expc, T desr,125 memory_order m = memory_order_seq_cst) volatile noexcept;126 bool compare_exchange_strong(T& expc, T desr,127 memory_order m = memory_order_seq_cst) noexcept;128 129 atomic() noexcept = default;130 constexpr atomic(T desr) noexcept;131 atomic(const atomic&) = delete;132 atomic& operator=(const atomic&) = delete;133 atomic& operator=(const atomic&) volatile = delete;134 T operator=(T) volatile noexcept;135 T operator=(T) noexcept;136};137 138template <>139struct atomic<integral>140{141 static constexpr bool is_always_lock_free;142 bool is_lock_free() const volatile noexcept;143 bool is_lock_free() const noexcept;144 void store(integral desr, memory_order m = memory_order_seq_cst) volatile noexcept;145 void store(integral desr, memory_order m = memory_order_seq_cst) noexcept;146 integral load(memory_order m = memory_order_seq_cst) const volatile noexcept;147 integral load(memory_order m = memory_order_seq_cst) const noexcept;148 operator integral() const volatile noexcept;149 operator integral() const noexcept;150 integral exchange(integral desr,151 memory_order m = memory_order_seq_cst) volatile noexcept;152 integral exchange(integral desr, memory_order m = memory_order_seq_cst) noexcept;153 bool compare_exchange_weak(integral& expc, integral desr,154 memory_order s, memory_order f) volatile noexcept;155 bool compare_exchange_weak(integral& expc, integral desr,156 memory_order s, memory_order f) noexcept;157 bool compare_exchange_strong(integral& expc, integral desr,158 memory_order s, memory_order f) volatile noexcept;159 bool compare_exchange_strong(integral& expc, integral desr,160 memory_order s, memory_order f) noexcept;161 bool compare_exchange_weak(integral& expc, integral desr,162 memory_order m = memory_order_seq_cst) volatile noexcept;163 bool compare_exchange_weak(integral& expc, integral desr,164 memory_order m = memory_order_seq_cst) noexcept;165 bool compare_exchange_strong(integral& expc, integral desr,166 memory_order m = memory_order_seq_cst) volatile noexcept;167 bool compare_exchange_strong(integral& expc, integral desr,168 memory_order m = memory_order_seq_cst) noexcept;169 170 integral171 fetch_add(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;172 integral fetch_add(integral op, memory_order m = memory_order_seq_cst) noexcept;173 integral174 fetch_sub(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;175 integral fetch_sub(integral op, memory_order m = memory_order_seq_cst) noexcept;176 integral177 fetch_and(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;178 integral fetch_and(integral op, memory_order m = memory_order_seq_cst) noexcept;179 integral180 fetch_or(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;181 integral fetch_or(integral op, memory_order m = memory_order_seq_cst) noexcept;182 integral183 fetch_xor(integral op, memory_order m = memory_order_seq_cst) volatile noexcept;184 integral fetch_xor(integral op, memory_order m = memory_order_seq_cst) noexcept;185 186 atomic() noexcept = default;187 constexpr atomic(integral desr) noexcept;188 atomic(const atomic&) = delete;189 atomic& operator=(const atomic&) = delete;190 atomic& operator=(const atomic&) volatile = delete;191 integral operator=(integral desr) volatile noexcept;192 integral operator=(integral desr) noexcept;193 194 integral operator++(int) volatile noexcept;195 integral operator++(int) noexcept;196 integral operator--(int) volatile noexcept;197 integral operator--(int) noexcept;198 integral operator++() volatile noexcept;199 integral operator++() noexcept;200 integral operator--() volatile noexcept;201 integral operator--() noexcept;202 integral operator+=(integral op) volatile noexcept;203 integral operator+=(integral op) noexcept;204 integral operator-=(integral op) volatile noexcept;205 integral operator-=(integral op) noexcept;206 integral operator&=(integral op) volatile noexcept;207 integral operator&=(integral op) noexcept;208 integral operator|=(integral op) volatile noexcept;209 integral operator|=(integral op) noexcept;210 integral operator^=(integral op) volatile noexcept;211 integral operator^=(integral op) noexcept;212};213 214template <class T>215struct atomic<T*>216{217 static constexpr bool is_always_lock_free;218 bool is_lock_free() const volatile noexcept;219 bool is_lock_free() const noexcept;220 void store(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;221 void store(T* desr, memory_order m = memory_order_seq_cst) noexcept;222 T* load(memory_order m = memory_order_seq_cst) const volatile noexcept;223 T* load(memory_order m = memory_order_seq_cst) const noexcept;224 operator T*() const volatile noexcept;225 operator T*() const noexcept;226 T* exchange(T* desr, memory_order m = memory_order_seq_cst) volatile noexcept;227 T* exchange(T* desr, memory_order m = memory_order_seq_cst) noexcept;228 bool compare_exchange_weak(T*& expc, T* desr,229 memory_order s, memory_order f) volatile noexcept;230 bool compare_exchange_weak(T*& expc, T* desr,231 memory_order s, memory_order f) noexcept;232 bool compare_exchange_strong(T*& expc, T* desr,233 memory_order s, memory_order f) volatile noexcept;234 bool compare_exchange_strong(T*& expc, T* desr,235 memory_order s, memory_order f) noexcept;236 bool compare_exchange_weak(T*& expc, T* desr,237 memory_order m = memory_order_seq_cst) volatile noexcept;238 bool compare_exchange_weak(T*& expc, T* desr,239 memory_order m = memory_order_seq_cst) noexcept;240 bool compare_exchange_strong(T*& expc, T* desr,241 memory_order m = memory_order_seq_cst) volatile noexcept;242 bool compare_exchange_strong(T*& expc, T* desr,243 memory_order m = memory_order_seq_cst) noexcept;244 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;245 T* fetch_add(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;246 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) volatile noexcept;247 T* fetch_sub(ptrdiff_t op, memory_order m = memory_order_seq_cst) noexcept;248 249 atomic() noexcept = default;250 constexpr atomic(T* desr) noexcept;251 atomic(const atomic&) = delete;252 atomic& operator=(const atomic&) = delete;253 atomic& operator=(const atomic&) volatile = delete;254 255 T* operator=(T*) volatile noexcept;256 T* operator=(T*) noexcept;257 T* operator++(int) volatile noexcept;258 T* operator++(int) noexcept;259 T* operator--(int) volatile noexcept;260 T* operator--(int) noexcept;261 T* operator++() volatile noexcept;262 T* operator++() noexcept;263 T* operator--() volatile noexcept;264 T* operator--() noexcept;265 T* operator+=(ptrdiff_t op) volatile noexcept;266 T* operator+=(ptrdiff_t op) noexcept;267 T* operator-=(ptrdiff_t op) volatile noexcept;268 T* operator-=(ptrdiff_t op) noexcept;269};270 271 272template <class T>273 bool274 atomic_is_lock_free(const volatile atomic<T>* obj) noexcept;275 276template <class T>277 bool278 atomic_is_lock_free(const atomic<T>* obj) noexcept;279 280template <class T>281 void282 atomic_init(volatile atomic<T>* obj, T desr) noexcept;283 284template <class T>285 void286 atomic_init(atomic<T>* obj, T desr) noexcept;287 288template <class T>289 void290 atomic_store(volatile atomic<T>* obj, T desr) noexcept;291 292template <class T>293 void294 atomic_store(atomic<T>* obj, T desr) noexcept;295 296template <class T>297 void298 atomic_store_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept;299 300template <class T>301 void302 atomic_store_explicit(atomic<T>* obj, T desr, memory_order m) noexcept;303 304template <class T>305 T306 atomic_load(const volatile atomic<T>* obj) noexcept;307 308template <class T>309 T310 atomic_load(const atomic<T>* obj) noexcept;311 312template <class T>313 T314 atomic_load_explicit(const volatile atomic<T>* obj, memory_order m) noexcept;315 316template <class T>317 T318 atomic_load_explicit(const atomic<T>* obj, memory_order m) noexcept;319 320template <class T>321 T322 atomic_exchange(volatile atomic<T>* obj, T desr) noexcept;323 324template <class T>325 T326 atomic_exchange(atomic<T>* obj, T desr) noexcept;327 328template <class T>329 T330 atomic_exchange_explicit(volatile atomic<T>* obj, T desr, memory_order m) noexcept;331 332template <class T>333 T334 atomic_exchange_explicit(atomic<T>* obj, T desr, memory_order m) noexcept;335 336template <class T>337 bool338 atomic_compare_exchange_weak(volatile atomic<T>* obj, T* expc, T desr) noexcept;339 340template <class T>341 bool342 atomic_compare_exchange_weak(atomic<T>* obj, T* expc, T desr) noexcept;343 344template <class T>345 bool346 atomic_compare_exchange_strong(volatile atomic<T>* obj, T* expc, T desr) noexcept;347 348template <class T>349 bool350 atomic_compare_exchange_strong(atomic<T>* obj, T* expc, T desr) noexcept;351 352template <class T>353 bool354 atomic_compare_exchange_weak_explicit(volatile atomic<T>* obj, T* expc,355 T desr,356 memory_order s, memory_order f) noexcept;357 358template <class T>359 bool360 atomic_compare_exchange_weak_explicit(atomic<T>* obj, T* expc, T desr,361 memory_order s, memory_order f) noexcept;362 363template <class T>364 bool365 atomic_compare_exchange_strong_explicit(volatile atomic<T>* obj,366 T* expc, T desr,367 memory_order s, memory_order f) noexcept;368 369template <class T>370 bool371 atomic_compare_exchange_strong_explicit(atomic<T>* obj, T* expc,372 T desr,373 memory_order s, memory_order f) noexcept;374 375template <class Integral>376 Integral377 atomic_fetch_add(volatile atomic<Integral>* obj, Integral op) noexcept;378 379template <class Integral>380 Integral381 atomic_fetch_add(atomic<Integral>* obj, Integral op) noexcept;382 383template <class Integral>384 Integral385 atomic_fetch_add_explicit(volatile atomic<Integral>* obj, Integral op,386 memory_order m) noexcept;387template <class Integral>388 Integral389 atomic_fetch_add_explicit(atomic<Integral>* obj, Integral op,390 memory_order m) noexcept;391template <class Integral>392 Integral393 atomic_fetch_sub(volatile atomic<Integral>* obj, Integral op) noexcept;394 395template <class Integral>396 Integral397 atomic_fetch_sub(atomic<Integral>* obj, Integral op) noexcept;398 399template <class Integral>400 Integral401 atomic_fetch_sub_explicit(volatile atomic<Integral>* obj, Integral op,402 memory_order m) noexcept;403template <class Integral>404 Integral405 atomic_fetch_sub_explicit(atomic<Integral>* obj, Integral op,406 memory_order m) noexcept;407template <class Integral>408 Integral409 atomic_fetch_and(volatile atomic<Integral>* obj, Integral op) noexcept;410 411template <class Integral>412 Integral413 atomic_fetch_and(atomic<Integral>* obj, Integral op) noexcept;414 415template <class Integral>416 Integral417 atomic_fetch_and_explicit(volatile atomic<Integral>* obj, Integral op,418 memory_order m) noexcept;419template <class Integral>420 Integral421 atomic_fetch_and_explicit(atomic<Integral>* obj, Integral op,422 memory_order m) noexcept;423template <class Integral>424 Integral425 atomic_fetch_or(volatile atomic<Integral>* obj, Integral op) noexcept;426 427template <class Integral>428 Integral429 atomic_fetch_or(atomic<Integral>* obj, Integral op) noexcept;430 431template <class Integral>432 Integral433 atomic_fetch_or_explicit(volatile atomic<Integral>* obj, Integral op,434 memory_order m) noexcept;435template <class Integral>436 Integral437 atomic_fetch_or_explicit(atomic<Integral>* obj, Integral op,438 memory_order m) noexcept;439template <class Integral>440 Integral441 atomic_fetch_xor(volatile atomic<Integral>* obj, Integral op) noexcept;442 443template <class Integral>444 Integral445 atomic_fetch_xor(atomic<Integral>* obj, Integral op) noexcept;446 447template <class Integral>448 Integral449 atomic_fetch_xor_explicit(volatile atomic<Integral>* obj, Integral op,450 memory_order m) noexcept;451template <class Integral>452 Integral453 atomic_fetch_xor_explicit(atomic<Integral>* obj, Integral op,454 memory_order m) noexcept;455 456template <class T>457 T*458 atomic_fetch_add(volatile atomic<T*>* obj, ptrdiff_t op) noexcept;459 460template <class T>461 T*462 atomic_fetch_add(atomic<T*>* obj, ptrdiff_t op) noexcept;463 464template <class T>465 T*466 atomic_fetch_add_explicit(volatile atomic<T*>* obj, ptrdiff_t op,467 memory_order m) noexcept;468template <class T>469 T*470 atomic_fetch_add_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept;471 472template <class T>473 T*474 atomic_fetch_sub(volatile atomic<T*>* obj, ptrdiff_t op) noexcept;475 476template <class T>477 T*478 atomic_fetch_sub(atomic<T*>* obj, ptrdiff_t op) noexcept;479 480template <class T>481 T*482 atomic_fetch_sub_explicit(volatile atomic<T*>* obj, ptrdiff_t op,483 memory_order m) noexcept;484template <class T>485 T*486 atomic_fetch_sub_explicit(atomic<T*>* obj, ptrdiff_t op, memory_order m) noexcept;487 488// Atomics for standard typedef types489 490typedef atomic<bool> atomic_bool;491typedef atomic<char> atomic_char;492typedef atomic<signed char> atomic_schar;493typedef atomic<unsigned char> atomic_uchar;494typedef atomic<short> atomic_short;495typedef atomic<unsigned short> atomic_ushort;496typedef atomic<int> atomic_int;497typedef atomic<unsigned int> atomic_uint;498typedef atomic<long> atomic_long;499typedef atomic<unsigned long> atomic_ulong;500typedef atomic<long long> atomic_llong;501typedef atomic<unsigned long long> atomic_ullong;502typedef atomic<char16_t> atomic_char16_t;503typedef atomic<char32_t> atomic_char32_t;504typedef atomic<wchar_t> atomic_wchar_t;505 506typedef atomic<int_least8_t> atomic_int_least8_t;507typedef atomic<uint_least8_t> atomic_uint_least8_t;508typedef atomic<int_least16_t> atomic_int_least16_t;509typedef atomic<uint_least16_t> atomic_uint_least16_t;510typedef atomic<int_least32_t> atomic_int_least32_t;511typedef atomic<uint_least32_t> atomic_uint_least32_t;512typedef atomic<int_least64_t> atomic_int_least64_t;513typedef atomic<uint_least64_t> atomic_uint_least64_t;514 515typedef atomic<int_fast8_t> atomic_int_fast8_t;516typedef atomic<uint_fast8_t> atomic_uint_fast8_t;517typedef atomic<int_fast16_t> atomic_int_fast16_t;518typedef atomic<uint_fast16_t> atomic_uint_fast16_t;519typedef atomic<int_fast32_t> atomic_int_fast32_t;520typedef atomic<uint_fast32_t> atomic_uint_fast32_t;521typedef atomic<int_fast64_t> atomic_int_fast64_t;522typedef atomic<uint_fast64_t> atomic_uint_fast64_t;523 524typedef atomic<int8_t> atomic_int8_t;525typedef atomic<uint8_t> atomic_uint8_t;526typedef atomic<int16_t> atomic_int16_t;527typedef atomic<uint16_t> atomic_uint16_t;528typedef atomic<int32_t> atomic_int32_t;529typedef atomic<uint32_t> atomic_uint32_t;530typedef atomic<int64_t> atomic_int64_t;531typedef atomic<uint64_t> atomic_uint64_t;532 533typedef atomic<intptr_t> atomic_intptr_t;534typedef atomic<uintptr_t> atomic_uintptr_t;535typedef atomic<size_t> atomic_size_t;536typedef atomic<ptrdiff_t> atomic_ptrdiff_t;537typedef atomic<intmax_t> atomic_intmax_t;538typedef atomic<uintmax_t> atomic_uintmax_t;539 540// fences541 542void atomic_thread_fence(memory_order m) noexcept;543void atomic_signal_fence(memory_order m) noexcept;544 545} // std546 547*/548 549#ifndef __cuda_std__550#include <__config>551#include <cstring>552#endif // __cuda_std__553 554#include "__assert" // all public C++ headers provide the assertion handler555#include "__debug"556#include "__threading_support"557#include "__type_traits/conditional.h"558#include "__type_traits/enable_if.h"559#include "__type_traits/is_assignable.h"560#include "__type_traits/is_floating_point.h"561#include "__type_traits/is_integral.h"562#include "__type_traits/is_same.h"563#include "__type_traits/is_trivially_copyable.h"564#include "__type_traits/underlying_type.h"565#include "__utility/forward.h"566#include "cstddef"567#include "cstdint"568#include "type_traits"569#include "version"570 571#ifndef __cuda_std__572#include <__pragma_push>573#endif // __cuda_std__574 575#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)576# pragma GCC system_header577#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)578# pragma clang system_header579#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)580# pragma system_header581#endif // no system header582 583#ifdef _LIBCUDACXX_HAS_NO_THREADS584# error <atomic> is not supported on this single threaded system585#endif586#ifdef _LIBCUDACXX_HAS_NO_ATOMIC_HEADER587# error <atomic> is not implemented588#endif589#ifdef _LIBCUDACXX_UNSUPPORTED_THREAD_API590# error "<atomic> is not supported on this system"591#endif592#ifdef kill_dependency593# error C++ standard library is incompatible with <stdatomic.h>594#endif595 596#define _LIBCUDACXX_CHECK_STORE_MEMORY_ORDER(__m) \597 _LIBCUDACXX_DIAGNOSE_WARNING(__m == memory_order_consume || \598 __m == memory_order_acquire || \599 __m == memory_order_acq_rel, \600 "memory order argument to atomic operation is invalid")601 602#define _LIBCUDACXX_CHECK_LOAD_MEMORY_ORDER(__m) \603 _LIBCUDACXX_DIAGNOSE_WARNING(__m == memory_order_release || \604 __m == memory_order_acq_rel, \605 "memory order argument to atomic operation is invalid")606 607#define _LIBCUDACXX_CHECK_EXCHANGE_MEMORY_ORDER(__m, __f) \608 _LIBCUDACXX_DIAGNOSE_WARNING(__f == memory_order_release || \609 __f == memory_order_acq_rel, \610 "memory order argument to atomic operation is invalid")611 612#if defined(_LIBCUDACXX_HAS_MSVC_ATOMIC_IMPL)613# include <intrin.h>614#endif615 616#if !defined(_LIBCUDACXX_COMPILER_NVRTC)617# include <string.h>618#endif619 620#if !defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) && !defined(__GCC_ATOMIC_BOOL_LOCK_FREE)621#define ATOMIC_BOOL_LOCK_FREE 2622#define ATOMIC_CHAR_LOCK_FREE 2623#define ATOMIC_CHAR16_T_LOCK_FREE 2624#define ATOMIC_CHAR32_T_LOCK_FREE 2625#define ATOMIC_WCHAR_T_LOCK_FREE 2626#define ATOMIC_SHORT_LOCK_FREE 2627#define ATOMIC_INT_LOCK_FREE 2628#define ATOMIC_LONG_LOCK_FREE 2629#define ATOMIC_LLONG_LOCK_FREE 2630#define ATOMIC_POINTER_LOCK_FREE 2631#endif //!defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) && !defined(__GCC_ATOMIC_BOOL_LOCK_FREE)632 633#ifndef __ATOMIC_RELAXED634#define __ATOMIC_RELAXED 0635#define __ATOMIC_CONSUME 1636#define __ATOMIC_ACQUIRE 2637#define __ATOMIC_RELEASE 3638#define __ATOMIC_ACQ_REL 4639#define __ATOMIC_SEQ_CST 5640#endif //__ATOMIC_RELAXED641 642_LIBCUDACXX_BEGIN_NAMESPACE_STD643 644// Figure out what the underlying type for `memory_order` would be if it were645// declared as an unscoped enum (accounting for -fshort-enums). Use this result646// to pin the underlying type in C++20.647enum __legacy_memory_order {648 __mo_relaxed,649 __mo_consume,650 __mo_acquire,651 __mo_release,652 __mo_acq_rel,653 __mo_seq_cst654};655 656typedef underlying_type<__legacy_memory_order>::type __memory_order_underlying_t;657 658#if _LIBCUDACXX_STD_VER > 17659 660enum class memory_order : __memory_order_underlying_t {661 relaxed = __mo_relaxed,662 consume = __mo_consume,663 acquire = __mo_acquire,664 release = __mo_release,665 acq_rel = __mo_acq_rel,666 seq_cst = __mo_seq_cst667};668 669inline constexpr auto memory_order_relaxed = memory_order::relaxed;670inline constexpr auto memory_order_consume = memory_order::consume;671inline constexpr auto memory_order_acquire = memory_order::acquire;672inline constexpr auto memory_order_release = memory_order::release;673inline constexpr auto memory_order_acq_rel = memory_order::acq_rel;674inline constexpr auto memory_order_seq_cst = memory_order::seq_cst;675 676#else677 678typedef enum memory_order {679 memory_order_relaxed = __mo_relaxed,680 memory_order_consume = __mo_consume,681 memory_order_acquire = __mo_acquire,682 memory_order_release = __mo_release,683 memory_order_acq_rel = __mo_acq_rel,684 memory_order_seq_cst = __mo_seq_cst,685} memory_order;686 687#endif // _LIBCUDACXX_STD_VER > 17688 689template <typename _Tp> _LIBCUDACXX_INLINE_VISIBILITY690bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {691#if defined(_LIBCUDACXX_COMPILER_NVCC) \692 || defined(_LIBCUDACXX_COMPILER_NVRTC) \693 || defined(_LIBCUDACXX_COMPILER_CLANG_CUDA)694 return __lhs == __rhs;695#else696 return memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0;697#endif698}699 700static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value),701 "unexpected underlying type for std::memory_order");702 703#if defined(_LIBCUDACXX_HAS_GCC_ATOMIC_IMP) || \704 defined(_LIBCUDACXX_ATOMIC_ONLY_USE_BUILTINS)705 706// [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because707// the default operator= in an object is not volatile, a byte-by-byte copy708// is required.709template <typename _Tp, typename _Tv> _LIBCUDACXX_INLINE_VISIBILITY710__enable_if_t<is_assignable<_Tp&, _Tv>::value>711__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val) {712 __a_value = __val;713}714template <typename _Tp, typename _Tv> _LIBCUDACXX_INLINE_VISIBILITY715__enable_if_t<is_assignable<_Tp&, _Tv>::value>716__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) {717 volatile char* __to = reinterpret_cast<volatile char*>(&__a_value);718 volatile char* __end = __to + sizeof(_Tp);719 volatile const char* __from = reinterpret_cast<volatile const char*>(&__val);720 while (__to != __end)721 *__to++ = *__from++;722}723 724#endif725 726// Headers are wrapped like so: (cuda::std::|std::)detail727namespace __detail {728#if defined(_LIBCUDACXX_HAS_CUDA_ATOMIC_EXT)729# include "support/atomic/atomic_scopes.h"730#endif731 732#if defined(_LIBCUDACXX_HAS_CUDA_ATOMIC_IMPL)733# include "support/atomic/atomic_cuda.h"734#elif defined(_LIBCUDACXX_HAS_MSVC_ATOMIC_IMPL)735# include "support/atomic/atomic_msvc.h"736#elif defined(_LIBCUDACXX_HAS_GCC_ATOMIC_IMP)737# include "support/atomic/atomic_gcc.h"738#elif defined(_LIBCUDACXX_HAS_C_ATOMIC_IMP)739// TODO: Maybe support C11 atomics?740// #include "support/atomic/atomic_c11.h"741#endif // _LIBCUDACXX_HAS_GCC_ATOMIC_IMP, _LIBCUDACXX_HAS_C_ATOMIC_IMP742}743 744using __detail::__cxx_atomic_base_impl;745using __detail::__cxx_atomic_ref_base_impl;746using __detail::__cxx_atomic_thread_fence;747using __detail::__cxx_atomic_signal_fence;748using __detail::__cxx_atomic_load;749using __detail::__cxx_atomic_store;750using __detail::__cxx_atomic_exchange;751using __detail::__cxx_atomic_compare_exchange_weak;752using __detail::__cxx_atomic_compare_exchange_strong;753using __detail::__cxx_atomic_fetch_add;754using __detail::__cxx_atomic_fetch_sub;755using __detail::__cxx_atomic_fetch_or;756using __detail::__cxx_atomic_fetch_and;757using __detail::__cxx_atomic_fetch_xor;758 759template <class _Tp>760_LIBCUDACXX_INLINE_VISIBILITY761_Tp kill_dependency(_Tp __y) noexcept762{763 return __y;764}765 766#if defined(__CLANG_ATOMIC_BOOL_LOCK_FREE)767# define ATOMIC_BOOL_LOCK_FREE __CLANG_ATOMIC_BOOL_LOCK_FREE768# define ATOMIC_CHAR_LOCK_FREE __CLANG_ATOMIC_CHAR_LOCK_FREE769# define ATOMIC_CHAR16_T_LOCK_FREE __CLANG_ATOMIC_CHAR16_T_LOCK_FREE770# define ATOMIC_CHAR32_T_LOCK_FREE __CLANG_ATOMIC_CHAR32_T_LOCK_FREE771# define ATOMIC_WCHAR_T_LOCK_FREE __CLANG_ATOMIC_WCHAR_T_LOCK_FREE772# define ATOMIC_SHORT_LOCK_FREE __CLANG_ATOMIC_SHORT_LOCK_FREE773# define ATOMIC_INT_LOCK_FREE __CLANG_ATOMIC_INT_LOCK_FREE774# define ATOMIC_LONG_LOCK_FREE __CLANG_ATOMIC_LONG_LOCK_FREE775# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE776# define ATOMIC_POINTER_LOCK_FREE __CLANG_ATOMIC_POINTER_LOCK_FREE777#elif defined(__GCC_ATOMIC_BOOL_LOCK_FREE)778# define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE779# define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE780# define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE781# define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE782# define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE783# define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE784# define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE785# define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE786# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE787# define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE788#endif789 790#ifdef _LIBCUDACXX_ATOMIC_ONLY_USE_BUILTINS791 792template<typename _Tp, int _Sco>793struct __cxx_atomic_lock_impl {794 795 _LIBCUDACXX_INLINE_VISIBILITY796 __cxx_atomic_lock_impl() noexcept797 : __a_value(), __a_lock(0) {}798 _LIBCUDACXX_INLINE_VISIBILITY constexpr explicit799 __cxx_atomic_lock_impl(_Tp value) noexcept800 : __a_value(value), __a_lock(0) {}801 802 _Tp __a_value;803 mutable __cxx_atomic_base_impl<_LIBCUDACXX_ATOMIC_FLAG_TYPE, _Sco> __a_lock;804 805 _LIBCUDACXX_INLINE_VISIBILITY void __lock() const volatile {806 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCUDACXX_ATOMIC_FLAG_TYPE(true), memory_order_acquire))807 /*spin*/;808 }809 _LIBCUDACXX_INLINE_VISIBILITY void __lock() const {810 while(1 == __cxx_atomic_exchange(&__a_lock, _LIBCUDACXX_ATOMIC_FLAG_TYPE(true), memory_order_acquire))811 /*spin*/;812 }813 _LIBCUDACXX_INLINE_VISIBILITY void __unlock() const volatile {814 __cxx_atomic_store(&__a_lock, _LIBCUDACXX_ATOMIC_FLAG_TYPE(false), memory_order_release);815 }816 _LIBCUDACXX_INLINE_VISIBILITY void __unlock() const {817 __cxx_atomic_store(&__a_lock, _LIBCUDACXX_ATOMIC_FLAG_TYPE(false), memory_order_release);818 }819 _LIBCUDACXX_INLINE_VISIBILITY _Tp __read() const volatile {820 __lock();821 _Tp __old;822 __cxx_atomic_assign_volatile(__old, __a_value);823 __unlock();824 return __old;825 }826 _LIBCUDACXX_INLINE_VISIBILITY _Tp __read() const {827 __lock();828 _Tp __old = __a_value;829 __unlock();830 return __old;831 }832};833 834template <typename _Tp, int _Sco>835_LIBCUDACXX_INLINE_VISIBILITY836void __cxx_atomic_init(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a, _Tp __val) {837 __cxx_atomic_assign_volatile(__a->__a_value, __val);838}839template <typename _Tp, int _Sco>840_LIBCUDACXX_INLINE_VISIBILITY841void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp, _Sco>* __a, _Tp __val) {842 __a->__a_value = __val;843}844 845template <typename _Tp, int _Sco>846_LIBCUDACXX_INLINE_VISIBILITY847void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a, _Tp __val, memory_order) {848 __a->__lock();849 __cxx_atomic_assign_volatile(__a->__a_value, __val);850 __a->__unlock();851}852template <typename _Tp, int _Sco>853_LIBCUDACXX_INLINE_VISIBILITY854void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp, _Sco>* __a, _Tp __val, memory_order) {855 __a->__lock();856 __a->__a_value = __val;857 __a->__unlock();858}859 860template <typename _Tp, int _Sco>861_LIBCUDACXX_INLINE_VISIBILITY862_Tp __cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a, memory_order) {863 return __a->__read();864}865template <typename _Tp, int _Sco>866_LIBCUDACXX_INLINE_VISIBILITY867_Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp, _Sco>* __a, memory_order) {868 return __a->__read();869}870 871template <typename _Tp, int _Sco>872_LIBCUDACXX_INLINE_VISIBILITY873_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a, _Tp __value, memory_order) {874 __a->__lock();875 _Tp __old;876 __cxx_atomic_assign_volatile(__old, __a->__a_value);877 __cxx_atomic_assign_volatile(__a->__a_value, __value);878 __a->__unlock();879 return __old;880}881template <typename _Tp, int _Sco>882_LIBCUDACXX_INLINE_VISIBILITY883_Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp, _Sco>* __a, _Tp __value, memory_order) {884 __a->__lock();885 _Tp __old = __a->__a_value;886 __a->__a_value = __value;887 __a->__unlock();888 return __old;889}890 891template <typename _Tp, int _Sco>892_LIBCUDACXX_INLINE_VISIBILITY893bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a,894 _Tp* __expected, _Tp __value, memory_order, memory_order) {895 __a->__lock();896 _Tp __temp;897 __cxx_atomic_assign_volatile(__temp, __a->__a_value);898 bool __ret = __temp == *__expected;899 if(__ret)900 __cxx_atomic_assign_volatile(__a->__a_value, __value);901 else902 __cxx_atomic_assign_volatile(*__expected, __a->__a_value);903 __a->__unlock();904 return __ret;905}906template <typename _Tp, int _Sco>907_LIBCUDACXX_INLINE_VISIBILITY908bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp, _Sco>* __a,909 _Tp* __expected, _Tp __value, memory_order, memory_order) {910 __a->__lock();911 bool __ret = __a->__a_value == *__expected;912 if(__ret)913 __a->__a_value = __value;914 else915 *__expected = __a->__a_value;916 __a->__unlock();917 return __ret;918}919 920template <typename _Tp, int _Sco>921_LIBCUDACXX_INLINE_VISIBILITY922bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a,923 _Tp* __expected, _Tp __value, memory_order, memory_order) {924 __a->__lock();925 _Tp __temp;926 __cxx_atomic_assign_volatile(__temp, __a->__a_value);927 bool __ret = __temp == *__expected;928 if(__ret)929 __cxx_atomic_assign_volatile(__a->__a_value, __value);930 else931 __cxx_atomic_assign_volatile(*__expected, __a->__a_value);932 __a->__unlock();933 return __ret;934}935template <typename _Tp, int _Sco>936_LIBCUDACXX_INLINE_VISIBILITY937bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp, _Sco>* __a,938 _Tp* __expected, _Tp __value, memory_order, memory_order) {939 __a->__lock();940 bool __ret = __a->__a_value == *__expected;941 if(__ret)942 __a->__a_value = __value;943 else944 *__expected = __a->__a_value;945 __a->__unlock();946 return __ret;947}948 949template <typename _Tp, typename _Td, int _Sco>950_LIBCUDACXX_INLINE_VISIBILITY951_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a,952 _Td __delta, memory_order) {953 __a->__lock();954 _Tp __old;955 __cxx_atomic_assign_volatile(__old, __a->__a_value);956 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old + __delta));957 __a->__unlock();958 return __old;959}960template <typename _Tp, typename _Td, int _Sco>961_LIBCUDACXX_INLINE_VISIBILITY962_Tp __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp, _Sco>* __a,963 _Td __delta, memory_order) {964 __a->__lock();965 _Tp __old = __a->__a_value;966 __a->__a_value += __delta;967 __a->__unlock();968 return __old;969}970 971template <typename _Tp, typename _Td, int _Sco>972_LIBCUDACXX_INLINE_VISIBILITY973_Tp* __cxx_atomic_fetch_add(volatile __cxx_atomic_lock_impl<_Tp*, _Sco>* __a,974 ptrdiff_t __delta, memory_order) {975 __a->__lock();976 _Tp* __old;977 __cxx_atomic_assign_volatile(__old, __a->__a_value);978 __cxx_atomic_assign_volatile(__a->__a_value, __old + __delta);979 __a->__unlock();980 return __old;981}982template <typename _Tp, typename _Td, int _Sco>983_LIBCUDACXX_INLINE_VISIBILITY984_Tp* __cxx_atomic_fetch_add(__cxx_atomic_lock_impl<_Tp*, _Sco>* __a,985 ptrdiff_t __delta, memory_order) {986 __a->__lock();987 _Tp* __old = __a->__a_value;988 __a->__a_value += __delta;989 __a->__unlock();990 return __old;991}992 993template <typename _Tp, typename _Td, int _Sco>994_LIBCUDACXX_INLINE_VISIBILITY995_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a,996 _Td __delta, memory_order) {997 __a->__lock();998 _Tp __old;999 __cxx_atomic_assign_volatile(__old, __a->__a_value);1000 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old - __delta));1001 __a->__unlock();1002 return __old;1003}1004template <typename _Tp, typename _Td, int _Sco>1005_LIBCUDACXX_INLINE_VISIBILITY1006_Tp __cxx_atomic_fetch_sub(__cxx_atomic_lock_impl<_Tp, _Sco>* __a,1007 _Td __delta, memory_order) {1008 __a->__lock();1009 _Tp __old = __a->__a_value;1010 __a->__a_value -= __delta;1011 __a->__unlock();1012 return __old;1013}1014 1015template <typename _Tp, int _Sco>1016_LIBCUDACXX_INLINE_VISIBILITY1017_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a,1018 _Tp __pattern, memory_order) {1019 __a->__lock();1020 _Tp __old;1021 __cxx_atomic_assign_volatile(__old, __a->__a_value);1022 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old & __pattern));1023 __a->__unlock();1024 return __old;1025}1026template <typename _Tp, int _Sco>1027_LIBCUDACXX_INLINE_VISIBILITY1028_Tp __cxx_atomic_fetch_and(__cxx_atomic_lock_impl<_Tp, _Sco>* __a,1029 _Tp __pattern, memory_order) {1030 __a->__lock();1031 _Tp __old = __a->__a_value;1032 __a->__a_value &= __pattern;1033 __a->__unlock();1034 return __old;1035}1036 1037template <typename _Tp, int _Sco>1038_LIBCUDACXX_INLINE_VISIBILITY1039_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a,1040 _Tp __pattern, memory_order) {1041 __a->__lock();1042 _Tp __old;1043 __cxx_atomic_assign_volatile(__old, __a->__a_value);1044 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old | __pattern));1045 __a->__unlock();1046 return __old;1047}1048template <typename _Tp, int _Sco>1049_LIBCUDACXX_INLINE_VISIBILITY1050_Tp __cxx_atomic_fetch_or(__cxx_atomic_lock_impl<_Tp, _Sco>* __a,1051 _Tp __pattern, memory_order) {1052 __a->__lock();1053 _Tp __old = __a->__a_value;1054 __a->__a_value |= __pattern;1055 __a->__unlock();1056 return __old;1057}1058 1059template <typename _Tp, int _Sco>1060_LIBCUDACXX_INLINE_VISIBILITY1061_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_lock_impl<_Tp, _Sco>* __a,1062 _Tp __pattern, memory_order) {1063 __a->__lock();1064 _Tp __old;1065 __cxx_atomic_assign_volatile(__old, __a->__a_value);1066 __cxx_atomic_assign_volatile(__a->__a_value, _Tp(__old ^ __pattern));1067 __a->__unlock();1068 return __old;1069}1070template <typename _Tp, int _Sco>1071_LIBCUDACXX_INLINE_VISIBILITY1072_Tp __cxx_atomic_fetch_xor(__cxx_atomic_lock_impl<_Tp, _Sco>* __a,1073 _Tp __pattern, memory_order) {1074 __a->__lock();1075 _Tp __old = __a->__a_value;1076 __a->__a_value ^= __pattern;1077 __a->__unlock();1078 return __old;1079}1080 1081#if defined(_LIBCUDACXX_ATOMIC_ALWAYS_LOCK_FREE)1082 1083template<typename _Tp> struct __cxx_is_always_lock_free {1084 enum { __value = _LIBCUDACXX_ATOMIC_ALWAYS_LOCK_FREE(sizeof(_Tp), 0) }; };1085 1086#else1087 1088template<typename _Tp> struct __cxx_is_always_lock_free {1089 enum { __value = sizeof(_Tp) <= 8 }; };1090 1091#endif // defined(_LIBCUDACXX_ATOMIC_ALWAYS_LOCK_FREE)1092 1093template <typename _Tp, int _Sco>1094struct __cxx_atomic_impl_conditional {1095 using type = __conditional_t<__cxx_is_always_lock_free<_Tp>::__value,1096 __cxx_atomic_base_impl<_Tp, _Sco>,1097 __cxx_atomic_lock_impl<_Tp, _Sco> >;1098};1099 1100template <typename _Tp, int _Sco,1101 typename _Base = typename __cxx_atomic_impl_conditional<_Tp, _Sco>::type >1102#else1103template <typename _Tp, int _Sco,1104 typename _Base = __cxx_atomic_base_impl<_Tp, _Sco> >1105#endif //_LIBCUDACXX_ATOMIC_ONLY_USE_BUILTINS1106struct __cxx_atomic_impl : public _Base {1107 __cxx_atomic_impl() noexcept = default;1108 _LIBCUDACXX_INLINE_VISIBILITY constexpr explicit __cxx_atomic_impl(_Tp value) noexcept1109 : _Base(value) {}1110};1111 1112 1113template<int _Sco, typename _Tp = int>1114_LIBCUDACXX_INLINE_VISIBILITY1115__cxx_atomic_impl<_Tp, _Sco>* __cxx_atomic_rebind(_Tp* __inst) {1116 static_assert(sizeof(__cxx_atomic_impl<_Tp, _Sco>) == sizeof(_Tp),"");1117 static_assert(alignof(__cxx_atomic_impl<_Tp, _Sco>) == alignof(_Tp),"");1118 return (__cxx_atomic_impl<_Tp, _Sco>*)__inst;1119}1120 1121template <typename _Tp, int _Sco>1122using __cxx_atomic_ref_impl = __cxx_atomic_ref_base_impl<_Tp, _Sco>;1123 1124#ifdef _LIBCUDACXX_HAS_NO_THREAD_CONTENTION_TABLE1125 1126template <class _Ty, class _Tp = __detail::__cxx_atomic_underlying_t<_Ty>, int _Sco = _Ty::__sco>1127struct __cxx_atomic_poll_tester {1128 _Ty const volatile* __a;1129 _Tp __val;1130 memory_order __order;1131 1132 _LIBCUDACXX_INLINE_VISIBILITY __cxx_atomic_poll_tester(_Ty const volatile* __a_, _Tp __val_, memory_order __order_)1133 : __a(__a_)1134 , __val(__val_)1135 , __order(__order_)1136 {}1137 1138 _LIBCUDACXX_INLINE_VISIBILITY bool operator()() const {1139 return !(__cxx_atomic_load(__a, __order) == __val);1140 }1141};1142 1143template <class _Ty, class _Tp = __detail::__cxx_atomic_underlying_t<_Ty>, int _Sco = _Ty::__sco>1144_LIBCUDACXX_INLINE_VISIBILITY void __cxx_atomic_try_wait_slow_fallback(_Ty const volatile* __a, _Tp __val, memory_order __order) {1145 __libcpp_thread_poll_with_backoff(__cxx_atomic_poll_tester<_Ty>(__a, __val, __order));1146}1147 1148#endif1149 1150#ifdef _LIBCUDACXX_HAS_PLATFORM_WAIT1151 1152template <class _Tp, int _Sco, __enable_if_t<!__libcpp_platform_wait_uses_type<_Tp>::__value, int> = 1>1153_LIBCUDACXX_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp, _Sco> const volatile* __a) {1154#ifndef _LIBCUDACXX_HAS_NO_THREAD_CONTENTION_TABLE1155 auto * const __c = __libcpp_contention_state(__a);1156 __cxx_atomic_fetch_add(__cxx_atomic_rebind<_Sco>(&__c->__version), (__libcpp_platform_wait_t)1, memory_order_relaxed);1157 __cxx_atomic_thread_fence(memory_order_seq_cst);1158 if (0 != __cxx_atomic_exchange(__cxx_atomic_rebind<_Sco>(&__c->__waiters), (ptrdiff_t)0, memory_order_relaxed))1159 __libcpp_platform_wake(&__c->__version, true);1160#endif1161}1162template <class _Tp, int _Sco, __enable_if_t<!__libcpp_platform_wait_uses_type<_Tp>::__value, int> = 1>1163_LIBCUDACXX_INLINE_VISIBILITY void __cxx_atomic_notify_one(__cxx_atomic_impl<_Tp, _Sco> const volatile* __a) {1164 __cxx_atomic_notify_all(__a);1165}1166template <class _Ty, class _Tp = __detail::__cxx_atomic_underlying_t<_Ty>, int _Sco = _Ty::__sco, __enable_if_t<!__libcpp_platform_wait_uses_type<_Tp>::__value, int> = 1>1167_LIBCUDACXX_INLINE_VISIBILITY void __cxx_atomic_try_wait_slow(_Ty const volatile* __a, _Tp const __val, memory_order __order) {1168#ifndef _LIBCUDACXX_HAS_NO_THREAD_CONTENTION_TABLE1169 auto * const __c = __libcpp_contention_state(__a);1170 __cxx_atomic_store(__cxx_atomic_rebind<_Sco>(&__c->__waiters), (ptrdiff_t)1, memory_order_relaxed);1171 __cxx_atomic_thread_fence(memory_order_seq_cst);1172 auto const __version = __cxx_atomic_load(__cxx_atomic_rebind<_Sco>(&__c->__version), memory_order_relaxed);1173 if (!__cxx_nonatomic_compare_equal(__cxx_atomic_load(__a, __order), __val))1174 return;1175 if(sizeof(__libcpp_platform_wait_t) < 8) {1176 constexpr timespec __timeout = { 2, 0 }; // Hedge on rare 'int version' aliasing.1177 __libcpp_platform_wait(&__c->__version, __version, &__timeout);1178 }1179 else1180 __libcpp_platform_wait(&__c->__version, __version, nullptr);1181#else1182 __cxx_atomic_try_wait_slow_fallback(__a, __val, __order);1183#endif // _LIBCUDACXX_HAS_NO_THREAD_CONTENTION_TABLE1184}1185 1186template <class _Tp, int _Sco, __enable_if_t<__libcpp_platform_wait_uses_type<_Tp>::__value, int> = 1>1187_LIBCUDACXX_INLINE_VISIBILITY void __cxx_atomic_try_wait_slow(__cxx_atomic_impl<_Tp, _Sco> const volatile* __a, _Tp __val, memory_order) {1188#ifndef _LIBCUDACXX_HAS_NO_THREAD_CONTENTION_TABLE1189 auto * const __c = __libcpp_contention_state(__a);1190 __cxx_atomic_fetch_add(__cxx_atomic_rebind<_Sco>(&__c->__waiters), (ptrdiff_t)1, memory_order_relaxed);1191 __cxx_atomic_thread_fence(memory_order_seq_cst);1192#endif1193 __libcpp_platform_wait((_Tp*)__a, __val, nullptr);1194#ifndef _LIBCUDACXX_HAS_NO_THREAD_CONTENTION_TABLE1195 __cxx_atomic_fetch_sub(__cxx_atomic_rebind<_Sco>(&__c->__waiters), (ptrdiff_t)1, memory_order_relaxed);1196#endif1197}1198template <class _Tp, int _Sco, __enable_if_t<__libcpp_platform_wait_uses_type<_Tp>::__value, int> = 1>1199_LIBCUDACXX_INLINE_VISIBILITY void __cxx_atomic_notify_all(__cxx_atomic_impl<_Tp, _Sco> const volatile* __a) {1200#ifndef _LIBCUDACXX_HAS_NO_THREAD_CONTENTION_TABLE