codekingpro/portable-devtools

1// -*- C++ -*-2//===--------------------------- semaphore --------------------------------===//3//4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5// See https://llvm.org/LICENSE.txt for license information.6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7//8//===----------------------------------------------------------------------===//9 10#ifndef _LIBCUDACXX_SEMAPHORE11#define _LIBCUDACXX_SEMAPHORE12 13/*14    semaphore synopsis15 16namespace std {17 18template<ptrdiff_t least_max_value = implementation-defined>19class counting_semaphore20{21public:22static constexpr ptrdiff_t max() noexcept;23 24constexpr explicit counting_semaphore(ptrdiff_t desired);25~counting_semaphore();26 27counting_semaphore(const counting_semaphore&) = delete;28counting_semaphore& operator=(const counting_semaphore&) = delete;29 30void release(ptrdiff_t __update = 1);31void acquire();32bool try_acquire() noexcept;33template<class Rep, class Period>34    bool try_acquire_for(const chrono::duration<Rep, Period>& __rel_time);35template<class Clock, class Duration>36    bool try_acquire_until(const chrono::time_point<Clock, Duration>& __abs_time);37 38private:39ptrdiff_t counter; // exposition only40};41 42using binary_semaphore = counting_semaphore<1>;43 44}45 46*/47 48#ifndef __cuda_std__49#include <__config>50#endif // __cuda_std__51 52#include "__assert" // all public C++ headers provide the assertion handler53#include "__type_traits/conditional.h"54#include "atomic"55#include "type_traits"56 57#ifndef __cuda_std__58#include <__pragma_push>59#endif // __cuda_std__60 61#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)62#  pragma GCC system_header63#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)64#  pragma clang system_header65#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)66#  pragma system_header67#endif // no system header68 69#ifdef _LIBCUDACXX_HAS_NO_THREADS70# error <semaphore> is not supported on this single threaded system71#endif72 73_LIBCUDACXX_BEGIN_NAMESPACE_STD74 75template<int _Sco, ptrdiff_t __least_max_value>76class __atomic_semaphore_base77{78    _LIBCUDACXX_INLINE_VISIBILITY79    bool __fetch_sub_if_slow(ptrdiff_t __old)80    {81        while (__old != 0) {82            if (__count.compare_exchange_weak(__old, __old - 1, memory_order_acquire, memory_order_relaxed))83                return true;84        }85        return false;86    }87 88    _LIBCUDACXX_INLINE_VISIBILITY89    bool __fetch_sub_if()90    {91        ptrdiff_t __old = __count.load(memory_order_acquire);92        if (__old == 0)93            return false;94        if(__count.compare_exchange_weak(__old, __old - 1, memory_order_acquire, memory_order_relaxed))95            return true;96        return __fetch_sub_if_slow(__old); // fail only if not __available97    }98 99    _LIBCUDACXX_INLINE_VISIBILITY100    void __wait_slow()101    {102        while (1) {103            ptrdiff_t const __old = __count.load(memory_order_acquire);104            if(__old != 0)105                break;106            __count.wait(__old, memory_order_relaxed);107        }108    }109 110    _LIBCUDACXX_INLINE_VISIBILITY111    bool __acquire_slow_timed(chrono::nanoseconds const& __rel_time)112    {113        return __libcpp_thread_poll_with_backoff([this]() {114            ptrdiff_t const __old = __count.load(memory_order_acquire);115            return __old != 0 && __fetch_sub_if_slow(__old);116        }, __rel_time);117    }118    __atomic_base<ptrdiff_t, _Sco> __count;119 120public:121    _LIBCUDACXX_INLINE_VISIBILITY122    static constexpr ptrdiff_t max() noexcept123    {124        return numeric_limits<ptrdiff_t>::max();125    }126 127    _LIBCUDACXX_INLINE_VISIBILITY constexpr128    __atomic_semaphore_base(ptrdiff_t __count) noexcept : __count(__count) { }129 130    ~__atomic_semaphore_base() = default;131 132    __atomic_semaphore_base(__atomic_semaphore_base const&) = delete;133    __atomic_semaphore_base& operator=(__atomic_semaphore_base const&) = delete;134 135    _LIBCUDACXX_INLINE_VISIBILITY136    void release(ptrdiff_t __update = 1)137    {138        __count.fetch_add(__update, memory_order_release);139        if(__update > 1)140            __count.notify_all();141        else142            __count.notify_one();143    }144 145    _LIBCUDACXX_INLINE_VISIBILITY146    void acquire()147    {148        while (!try_acquire())149            __wait_slow();150    }151 152    _LIBCUDACXX_INLINE_VISIBILITY153    bool try_acquire() noexcept154    {155        return __fetch_sub_if();156    }157 158    template <class Clock, class Duration>159    _LIBCUDACXX_INLINE_VISIBILITY160    bool try_acquire_until(chrono::time_point<Clock, Duration> const& __abs_time)161    {162        if (try_acquire())163            return true;164        else165            return __acquire_slow_timed(__abs_time - Clock::now());166    }167 168    template <class Rep, class Period>169    _LIBCUDACXX_INLINE_VISIBILITY170    bool try_acquire_for(chrono::duration<Rep, Period> const& __rel_time)171    {172 173        if (try_acquire())174            return true;175        else176            return __acquire_slow_timed(__rel_time);177    }178};179 180#ifndef _LIBCUDACXX_USE_NATIVE_SEMAPHORES181 182template<int _Sco>183class __atomic_semaphore_base<_Sco, 1> {184 185    _LIBCUDACXX_INLINE_VISIBILITY186    bool __acquire_slow_timed(chrono::nanoseconds const& __rel_time)187    {188        return __libcpp_thread_poll_with_backoff([this]() {189            return try_acquire();190        }, __rel_time);191    }192    __atomic_base<int, _Sco> __available;193 194public:195    _LIBCUDACXX_INLINE_VISIBILITY196    static constexpr ptrdiff_t max() noexcept { return 1; }197 198    _LIBCUDACXX_INLINE_VISIBILITY constexpr199    __atomic_semaphore_base(ptrdiff_t __available) : __available(__available) { }200 201    ~__atomic_semaphore_base() = default;202 203    __atomic_semaphore_base(__atomic_semaphore_base const&) = delete;204    __atomic_semaphore_base& operator=(__atomic_semaphore_base const&) = delete;205 206    _LIBCUDACXX_INLINE_VISIBILITY207    void release(ptrdiff_t __update = 1)208    {209        _LIBCUDACXX_ASSERT(__update == 1, "");210        __available.store(1, memory_order_release);211        __available.notify_one();212        (void)__update;213    }214 215    _LIBCUDACXX_INLINE_VISIBILITY216    void acquire()217    {218        while (!try_acquire())219            __available.wait(0, memory_order_relaxed);220    }221 222    _LIBCUDACXX_INLINE_VISIBILITY223    bool try_acquire() noexcept224    {225        return 1 == __available.exchange(0, memory_order_acquire);226    }227 228    template <class Clock, class Duration>229    _LIBCUDACXX_INLINE_VISIBILITY230    bool try_acquire_until(chrono::time_point<Clock, Duration> const& __abs_time)231    {232        if (try_acquire())233            return true;234        else235            return __acquire_slow_timed(__abs_time - Clock::now());236    }237 238    template <class Rep, class Period>239    _LIBCUDACXX_INLINE_VISIBILITY240    bool try_acquire_for(chrono::duration<Rep, Period> const& __rel_time)241    {242        if (try_acquire())243            return true;244        else245            return __acquire_slow_timed(__rel_time);246    }247};248 249#else250 251template<int _Sco>252class __sem_semaphore_base {253 254    _LIBCUDACXX_INLINE_VISIBILITY255    bool __backfill(bool __success)256    {257#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_BACK_BUFFER258        if(__success) {259            auto const __back_amount = __backbuffer.fetch_sub(2, memory_order_acquire);260            bool const __post_one = __back_amount > 0;261            bool const __post_two = __back_amount > 1;262            auto const __success = (!__post_one || __libcpp_semaphore_post(&__semaphore)) &&263                                 (!__post_two || __libcpp_semaphore_post(&__semaphore));264            _LIBCUDACXX_ASSERT(__success, "");265            if(!__post_one || !__post_two)266                __backbuffer.fetch_add(!__post_one ? 2 : 1, memory_order_relaxed);267        }268#endif269        return __success;270    }271 272    _LIBCUDACXX_INLINE_VISIBILITY273    bool __try_acquire_fast()274    {275#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_FRONT_BUFFER276 277        ptrdiff_t __old;278        __libcpp_thread_poll_with_backoff([&]() {279            __old = __frontbuffer.load(memory_order_relaxed);280            return 0 != (__old >> 32);281        }, chrono::microseconds(5));282 283        // always steal if you can284        while(__old >> 32)285            if(__frontbuffer.compare_exchange_weak(__old, __old - (1ll << 32), memory_order_acquire))286                return true;287        // record we're waiting288        __old = __frontbuffer.fetch_add(1ll, memory_order_release);289        // ALWAYS steal if you can!290        while(__old >> 32)291            if(__frontbuffer.compare_exchange_weak(__old, __old - (1ll << 32), memory_order_acquire))292                break;293        // not going to wait after all294        if(__old >> 32)295            return __try_done(true);296#endif297        // the wait has begun...298        return false;299    }300 301    _LIBCUDACXX_INLINE_VISIBILITY302    bool __try_done(bool __success)303    {304#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_FRONT_BUFFER305        // record we're NOT waiting306        __frontbuffer.fetch_sub(1ll, memory_order_release);307#endif308        return __backfill(__success);309    }310 311    _LIBCUDACXX_INLINE_VISIBILITY312    void __release_slow(ptrdiff_t __post_amount)313    {314    #ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_BACK_BUFFER315        bool const __post_one = __post_amount > 0;316        bool const __post_two = __post_amount > 1;317        if(__post_amount > 2)318            __backbuffer.fetch_add(__post_amount - 2, memory_order_acq_rel);319        auto const __success = (!__post_one || __libcpp_semaphore_post(&__semaphore)) &&320                             (!__post_two || __libcpp_semaphore_post(&__semaphore));321        _LIBCUDACXX_ASSERT(__success, "");322    #else323        for(; __post_amount; --__post_amount) {324            auto const __success = __libcpp_semaphore_post(&__semaphore);325            _LIBCUDACXX_ASSERT(__success, "");326        }327    #endif328    }329 330    __libcpp_semaphore_t __semaphore;331#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_FRONT_BUFFER332    __atomic_base<ptrdiff_t, _Sco> __frontbuffer;333#endif334#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_BACK_BUFFER335    __atomic_base<ptrdiff_t, _Sco> __backbuffer;336#endif337 338public:339    static constexpr ptrdiff_t max() noexcept {340        return _LIBCUDACXX_SEMAPHORE_MAX;341    }342 343    _LIBCUDACXX_INLINE_VISIBILITY344    __sem_semaphore_base(ptrdiff_t __count = 0) : __semaphore()345#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_FRONT_BUFFER346    , __frontbuffer(__count << 32)347#endif348#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_BACK_BUFFER349    , __backbuffer(0)350#endif351    {352        _LIBCUDACXX_ASSERT(__count <= max(), "");353        auto const __success =354#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_FRONT_BUFFER355            __libcpp_semaphore_init(&__semaphore, 0);356#else357            __libcpp_semaphore_init(&__semaphore, __count);358#endif359        _LIBCUDACXX_ASSERT(__success, "");360    }361 362    _LIBCUDACXX_INLINE_VISIBILITY363    ~__sem_semaphore_base() {364#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_FRONT_BUFFER365        _LIBCUDACXX_ASSERT(0 == (__frontbuffer.load(memory_order_relaxed) & ~0u), "");366#endif367        auto const __success = __libcpp_semaphore_destroy(&__semaphore);368        _LIBCUDACXX_ASSERT(__success, "");369    }370 371    __sem_semaphore_base(const __sem_semaphore_base&) = delete;372    __sem_semaphore_base& operator=(const __sem_semaphore_base&) = delete;373 374    _LIBCUDACXX_INLINE_VISIBILITY375    void release(ptrdiff_t __update = 1)376    {377#ifndef _LIBCUDACXX_HAS_NO_SEMAPHORE_FRONT_BUFFER378        // boldly assume the semaphore is taken but uncontended379        ptrdiff_t __old = 0;380        // try to fast-release as long as it's uncontended381        while(0 == (__old & ~0ul))382            if(__frontbuffer.compare_exchange_weak(__old, __old + (__update << 32), memory_order_acq_rel))383                return;384#endif385        // slow-release it is386        __release_slow(__update);387    }388 389    _LIBCUDACXX_INLINE_VISIBILITY390    void acquire()391    {392        if(!__try_acquire_fast())393            __try_done(__libcpp_semaphore_wait(&__semaphore));394    }395 396    _LIBCUDACXX_INLINE_VISIBILITY397    bool try_acquire() noexcept398    {399        return try_acquire_for(chrono::nanoseconds(0));400    }401 402    template <class Clock, class Duration>403    _LIBCUDACXX_INLINE_VISIBILITY404    bool try_acquire_until(chrono::time_point<Clock, Duration> const& __abs_time)405    {406        auto const current = max(Clock::now(), __abs_time);407        return try_acquire_for(chrono::duration_cast<chrono::nanoseconds>(__abs_time - current));408    }409 410    template <class Rep, class Period>411    _LIBCUDACXX_INLINE_VISIBILITY412    bool try_acquire_for(chrono::duration<Rep, Period> const& __rel_time)413    {414        return __try_acquire_fast() ||415               __try_done(__libcpp_semaphore_wait_timed(&__semaphore, __rel_time));416    }417};418 419#endif //_LIBCUDACXX_HAS_NO_SEMAPHORES420 421template<ptrdiff_t __least_max_value, int _Sco>422using __semaphore_base =423#ifdef _LIBCUDACXX_USE_NATIVE_SEMAPHORES424    __conditional_t<__least_max_value <= __sem_semaphore_base<_Sco>::max(),425                    __sem_semaphore_base<_Sco>,426                    __atomic_semaphore_base<_Sco, __least_max_value>>427#else428    __atomic_semaphore_base<_Sco, __least_max_value>429#endif430    ;431 432template<ptrdiff_t __least_max_value = INT_MAX>433class counting_semaphore : public __semaphore_base<__least_max_value, 0>434{435    static_assert(__least_max_value <= __semaphore_base<__least_max_value, 0>::max(), "");436public:437    _LIBCUDACXX_INLINE_VISIBILITY constexpr438    counting_semaphore(ptrdiff_t __count = 0) : __semaphore_base<__least_max_value, 0>(__count) { }439    ~counting_semaphore() = default;440 441    counting_semaphore(const counting_semaphore&) = delete;442    counting_semaphore& operator=(const counting_semaphore&) = delete;443};444 445using binary_semaphore = counting_semaphore<1>;446 447_LIBCUDACXX_END_NAMESPACE_STD448 449#ifndef __cuda_std__450#include <__pragma_pop>451#else452#include "__cuda/semaphore.h"453#endif //__cuda_std__454 455#endif //_LIBCUDACXX_SEMAPHORE456