codekingpro/portable-devtools

1// -*- C++ -*-2//===----------------------------------------------------------------------===//3//4// Part of libcu++, the C++ Standard Library for your entire system,5// under the Apache License v2.0 with LLVM Exceptions.6// See https://llvm.org/LICENSE.txt for license information.7// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception8// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.9//10//===----------------------------------------------------------------------===//11 12#ifndef _LIBCUDACXX_ATOMIC_BASE_H13#define _LIBCUDACXX_ATOMIC_BASE_H14 15#include "cxx_atomic.h"16 17// Guard ifdef for lock free query in case it is assigned elsewhere (MSVC/CUDA)18#ifndef _LIBCUDACXX_ATOMIC_IS_LOCK_FREE19#define _LIBCUDACXX_ATOMIC_IS_LOCK_FREE(__x) __atomic_is_lock_free(__x, 0)20#endif21 22_LIBCUDACXX_INLINE_VISIBILITY inline constexpr int __cxx_atomic_order_to_int(memory_order __order) {23  // Avoid switch statement to make this a constexpr.24  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:25         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:26          (__order == memory_order_release ? __ATOMIC_RELEASE:27           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:28            (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL:29              __ATOMIC_CONSUME))));30}31 32_LIBCUDACXX_INLINE_VISIBILITY inline constexpr int __cxx_atomic_failure_order_to_int(memory_order __order) {33  // Avoid switch statement to make this a constexpr.34  return __order == memory_order_relaxed ? __ATOMIC_RELAXED:35         (__order == memory_order_acquire ? __ATOMIC_ACQUIRE:36          (__order == memory_order_release ? __ATOMIC_RELAXED:37           (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST:38            (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE:39              __ATOMIC_CONSUME))));40}41 42template <typename _Tp, typename _Up>43inline void __cxx_atomic_init(volatile _Tp* __a,  _Up __val) {44  auto __a_tmp = __cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a));45  __cxx_atomic_assign_volatile(*__a_tmp, __val);46}47 48template <typename _Tp, typename _Up>49inline void __cxx_atomic_init(_Tp* __a,  _Up __val) {50  auto __a_tmp = __cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a));51  *__a_tmp = __val;52}53 54inline55void __cxx_atomic_thread_fence(memory_order __order) {56  __atomic_thread_fence(__cxx_atomic_order_to_int(__order));57}58 59inline60void __cxx_atomic_signal_fence(memory_order __order) {61  __atomic_signal_fence(__cxx_atomic_order_to_int(__order));62}63 64template <typename _Tp, typename _Up>65inline void __cxx_atomic_store(_Tp* __a,  _Up __val,66                        memory_order __order) {67  auto __v_temp = __cxx_atomic_wrap_to_base(__a, __val);68  __atomic_store(__cxx_atomic_unwrap(__a), &__v_temp, __cxx_atomic_order_to_int(__order));69}70 71template <typename _Tp>72inline auto __cxx_atomic_load(const _Tp* __a,73                       memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {74  auto __ret = __cxx_atomic_base_temporary(__a);75  __atomic_load(__cxx_atomic_unwrap(__a), &__ret, __cxx_atomic_order_to_int(__order));76  return *__cxx_get_underlying_atomic(&__ret);77}78 79template <typename _Tp, typename _Up>80inline auto __cxx_atomic_exchange(_Tp* __a, _Up __val,81                          memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {82  auto __v_temp = __cxx_atomic_wrap_to_base(__a, __val);83  auto __ret = __cxx_atomic_base_temporary(__a);84  __atomic_exchange(__cxx_atomic_unwrap(__a), &__v_temp, &__ret, __cxx_atomic_order_to_int(__order));85  return *__cxx_get_underlying_atomic(&__ret);86}87 88template <typename _Tp, typename _Up>89inline bool __cxx_atomic_compare_exchange_strong(90    _Tp* __a, _Up* __expected, _Up __value, memory_order __success,91    memory_order __failure) {92  (void)__expected;93  return __atomic_compare_exchange(__cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a)),94                                   __expected, &__value, false,95                                   __cxx_atomic_order_to_int(__success),96                                   __cxx_atomic_failure_order_to_int(__failure));97}98 99template <typename _Tp, typename _Up>100inline bool __cxx_atomic_compare_exchange_weak(101    _Tp* __a, _Up* __expected, _Up __value, memory_order __success,102    memory_order __failure) {103  (void)__expected;104  return __atomic_compare_exchange(__cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a)),105                                   __expected, &__value, true,106                                   __cxx_atomic_order_to_int(__success),107                                   __cxx_atomic_failure_order_to_int(__failure));108}109 110template <typename _Tp>111struct __atomic_ptr_inc { enum {value = 1}; };112 113template <typename _Tp>114struct __atomic_ptr_inc<_Tp*> { enum {value = sizeof(_Tp)}; };115 116// FIXME: Haven't figured out what the spec says about using arrays with117// atomic_fetch_add. Force a failure rather than creating bad behavior.118template <typename _Tp>119struct __atomic_ptr_inc<_Tp[]> { };120template <typename _Tp, int n>121struct __atomic_ptr_inc<_Tp[n]> { };122 123template <typename _Tp, typename _Td, __enable_if_t<!is_floating_point<__cxx_atomic_underlying_t<_Tp>>::value, int> = 0>124inline auto __cxx_atomic_fetch_add(_Tp* __a, _Td __delta,125                           memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {126  constexpr auto __skip_v = __atomic_ptr_inc<__cxx_atomic_underlying_t<_Tp>>::value;127  auto __a_tmp = __cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a));128  return __atomic_fetch_add(__a_tmp, __delta * __skip_v,129                            __cxx_atomic_order_to_int(__order));130}131 132template <typename _Tp, typename _Td, __enable_if_t<is_floating_point<__cxx_atomic_underlying_t<_Tp>>::value, int> = 0>133inline auto __cxx_atomic_fetch_add(_Tp* __a, _Td __delta,134                           memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {135  auto __expected = __cxx_atomic_load(__a, memory_order_relaxed);136  auto __desired = __expected + __delta;137 138  while(!__cxx_atomic_compare_exchange_strong(__a, &__expected, __desired, __order, __order)) {139      __desired = __expected + __delta;140  }141 142  return __expected;143}144 145template <typename _Tp, typename _Td, __enable_if_t<!is_floating_point<__cxx_atomic_underlying_t<_Tp>>::value, int> = 0>146inline auto __cxx_atomic_fetch_sub(_Tp* __a, _Td __delta,147                           memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {148  constexpr auto __skip_v = __atomic_ptr_inc<__cxx_atomic_underlying_t<_Tp>>::value;149  auto __a_tmp = __cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a));150  return __atomic_fetch_sub(__a_tmp, __delta * __skip_v,151                            __cxx_atomic_order_to_int(__order));152}153 154template <typename _Tp, typename _Td, __enable_if_t<is_floating_point<__cxx_atomic_underlying_t<_Tp>>::value, int> = 0>155inline auto __cxx_atomic_fetch_sub(_Tp* __a, _Td __delta,156                           memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {157  auto __expected = __cxx_atomic_load(__a, memory_order_relaxed);158  auto __desired = __expected - __delta;159 160  while(!__cxx_atomic_compare_exchange_strong(__a, &__expected, __desired, __order, __order)) {161      __desired = __expected - __delta;162  }163 164  return __expected;165}166 167template <typename _Tp, typename _Td>168inline auto __cxx_atomic_fetch_and(_Tp* __a, _Td __pattern,169                            memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {170  auto __a_tmp = __cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a));171  return __atomic_fetch_and(__a_tmp, __pattern,172                            __cxx_atomic_order_to_int(__order));173}174 175template <typename _Tp, typename _Td>176inline auto __cxx_atomic_fetch_or(_Tp* __a, _Td __pattern,177                          memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {178  auto __a_tmp = __cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a));179  return __atomic_fetch_or(__a_tmp, __pattern,180                           __cxx_atomic_order_to_int(__order));181}182 183template <typename _Tp, typename _Td>184inline auto __cxx_atomic_fetch_xor(_Tp* __a, _Td __pattern,185                           memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {186  auto __a_tmp = __cxx_get_underlying_atomic(__cxx_atomic_unwrap(__a));187  return __atomic_fetch_xor(__a_tmp, __pattern,188                            __cxx_atomic_order_to_int(__order));189}190 191template <typename _Tp, typename _Td>192inline auto __cxx_atomic_fetch_max(_Tp* __a, _Td __val,193                           memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {194  auto __expected = __cxx_atomic_load(__a, memory_order_relaxed);195  auto __desired = __expected > __val ? __expected : __val;196 197  while(__desired == __val &&198          !__cxx_atomic_compare_exchange_strong(__a, &__expected, __desired, __order, __order)) {199      __desired = __expected > __val ? __expected : __val;200  }201 202  return __expected;203}204 205template <typename _Tp, typename _Td>206inline auto __cxx_atomic_fetch_min(_Tp* __a, _Td __val,207                           memory_order __order) -> __cxx_atomic_underlying_t<_Tp> {208  auto __expected = __cxx_atomic_load(__a, memory_order_relaxed);209  auto __desired = __expected < __val ? __expected : __val;210 211  while(__desired == __val &&212          !__cxx_atomic_compare_exchange_strong(__a, &__expected, __desired, __order, __order)) {213      __desired = __expected < __val ? __expected : __val;214  }215 216  return __expected;217}218 219#endif // _LIBCUDACXX_ATOMIC_BASE_H220