codekingpro/portable-devtools

1/*-------------------------------------------------------------------------2 *3 * generic.h4 *	  Implement higher level operations based on some lower level atomic5 *	  operations.6 *7 * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group8 * Portions Copyright (c) 1994, Regents of the University of California9 *10 * src/include/port/atomics/generic.h11 *12 *-------------------------------------------------------------------------13 */14 15/* intentionally no include guards, should only be included by atomics.h */16#ifndef INSIDE_ATOMICS_H17#	error "should be included via atomics.h"18#endif19 20/*21 * If read or write barriers are undefined, we upgrade them to full memory22 * barriers.23 */24#if !defined(pg_read_barrier_impl)25#	define pg_read_barrier_impl pg_memory_barrier_impl26#endif27#if !defined(pg_write_barrier_impl)28#	define pg_write_barrier_impl pg_memory_barrier_impl29#endif30 31#ifndef PG_HAVE_SPIN_DELAY32#define PG_HAVE_SPIN_DELAY33#define pg_spin_delay_impl()	((void)0)34#endif35 36 37/* provide fallback */38#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT)39#define PG_HAVE_ATOMIC_FLAG_SUPPORT40typedef pg_atomic_uint32 pg_atomic_flag;41#endif42 43#ifndef PG_HAVE_ATOMIC_READ_U3244#define PG_HAVE_ATOMIC_READ_U3245static inline uint3246pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr)47{48	return ptr->value;49}50#endif51 52#ifndef PG_HAVE_ATOMIC_WRITE_U3253#define PG_HAVE_ATOMIC_WRITE_U3254static inline void55pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)56{57	ptr->value = val;58}59#endif60 61#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U3262#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U3263static inline void64pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val)65{66	ptr->value = val;67}68#endif69 70/*71 * provide fallback for test_and_set using atomic_exchange if available72 */73#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32)74 75#define PG_HAVE_ATOMIC_INIT_FLAG76static inline void77pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)78{79	pg_atomic_write_u32_impl(ptr, 0);80}81 82#define PG_HAVE_ATOMIC_TEST_SET_FLAG83static inline bool84pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)85{86	return pg_atomic_exchange_u32_impl(ptr, 1) == 0;87}88 89#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG90static inline bool91pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)92{93	return pg_atomic_read_u32_impl(ptr) == 0;94}95 96 97#define PG_HAVE_ATOMIC_CLEAR_FLAG98static inline void99pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)100{101	/* XXX: release semantics suffice? */102	pg_memory_barrier_impl();103	pg_atomic_write_u32_impl(ptr, 0);104}105 106/*107 * provide fallback for test_and_set using atomic_compare_exchange if108 * available.109 */110#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)111 112#define PG_HAVE_ATOMIC_INIT_FLAG113static inline void114pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)115{116	pg_atomic_write_u32_impl(ptr, 0);117}118 119#define PG_HAVE_ATOMIC_TEST_SET_FLAG120static inline bool121pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)122{123	uint32 value = 0;124	return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1);125}126 127#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG128static inline bool129pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)130{131	return pg_atomic_read_u32_impl(ptr) == 0;132}133 134#define PG_HAVE_ATOMIC_CLEAR_FLAG135static inline void136pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)137{138	/*139	 * Use a memory barrier + plain write if we have a native memory140	 * barrier. But don't do so if memory barriers use spinlocks - that'd lead141	 * to circularity if flags are used to implement spinlocks.142	 */143#ifndef PG_HAVE_MEMORY_BARRIER_EMULATION144	/* XXX: release semantics suffice? */145	pg_memory_barrier_impl();146	pg_atomic_write_u32_impl(ptr, 0);147#else148	uint32 value = 1;149	pg_atomic_compare_exchange_u32_impl(ptr, &value, 0);150#endif151}152 153#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG)154#	error "No pg_atomic_test_and_set provided"155#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */156 157 158#ifndef PG_HAVE_ATOMIC_INIT_U32159#define PG_HAVE_ATOMIC_INIT_U32160static inline void161pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_)162{163	ptr->value = val_;164}165#endif166 167#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)168#define PG_HAVE_ATOMIC_EXCHANGE_U32169static inline uint32170pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_)171{172	uint32 old;173	old = ptr->value;			/* ok if read is not atomic */174	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_))175		/* skip */;176	return old;177}178#endif179 180#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)181#define PG_HAVE_ATOMIC_FETCH_ADD_U32182static inline uint32183pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)184{185	uint32 old;186	old = ptr->value;			/* ok if read is not atomic */187	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_))188		/* skip */;189	return old;190}191#endif192 193#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)194#define PG_HAVE_ATOMIC_FETCH_SUB_U32195static inline uint32196pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)197{198	return pg_atomic_fetch_add_u32_impl(ptr, -sub_);199}200#endif201 202#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)203#define PG_HAVE_ATOMIC_FETCH_AND_U32204static inline uint32205pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_)206{207	uint32 old;208	old = ptr->value;			/* ok if read is not atomic */209	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_))210		/* skip */;211	return old;212}213#endif214 215#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32)216#define PG_HAVE_ATOMIC_FETCH_OR_U32217static inline uint32218pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_)219{220	uint32 old;221	old = ptr->value;			/* ok if read is not atomic */222	while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_))223		/* skip */;224	return old;225}226#endif227 228#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32)229#define PG_HAVE_ATOMIC_ADD_FETCH_U32230static inline uint32231pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)232{233	return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_;234}235#endif236 237#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32)238#define PG_HAVE_ATOMIC_SUB_FETCH_U32239static inline uint32240pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)241{242	return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_;243}244#endif245 246#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)247#define PG_HAVE_ATOMIC_EXCHANGE_U64248static inline uint64249pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_)250{251	uint64 old;252	old = ptr->value;			/* ok if read is not atomic */253	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_))254		/* skip */;255	return old;256}257#endif258 259#ifndef PG_HAVE_ATOMIC_WRITE_U64260#define PG_HAVE_ATOMIC_WRITE_U64261 262#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \263	!defined(PG_HAVE_ATOMIC_U64_SIMULATION)264 265static inline void266pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)267{268	/*269	 * On this platform aligned 64bit writes are guaranteed to be atomic,270	 * except if using the fallback implementation, where can't guarantee the271	 * required alignment.272	 */273	AssertPointerAlignment(ptr, 8);274	ptr->value = val;275}276 277#else278 279static inline void280pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)281{282	/*283	 * 64 bit writes aren't safe on all platforms. In the generic284	 * implementation implement them as an atomic exchange.285	 */286	pg_atomic_exchange_u64_impl(ptr, val);287}288 289#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */290#endif /* PG_HAVE_ATOMIC_WRITE_U64 */291 292#ifndef PG_HAVE_ATOMIC_READ_U64293#define PG_HAVE_ATOMIC_READ_U64294 295#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \296	!defined(PG_HAVE_ATOMIC_U64_SIMULATION)297 298static inline uint64299pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)300{301	/*302	 * On this platform aligned 64-bit reads are guaranteed to be atomic.303	 */304	AssertPointerAlignment(ptr, 8);305	return ptr->value;306}307 308#else309 310static inline uint64311pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)312{313	uint64 old = 0;314 315	/*316	 * 64-bit reads aren't atomic on all platforms. In the generic317	 * implementation implement them as a compare/exchange with 0. That'll318	 * fail or succeed, but always return the old value. Possibly might store319	 * a 0, but only if the previous value also was a 0 - i.e. harmless.320	 */321	pg_atomic_compare_exchange_u64_impl(ptr, &old, 0);322 323	return old;324}325#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */326#endif /* PG_HAVE_ATOMIC_READ_U64 */327 328#ifndef PG_HAVE_ATOMIC_INIT_U64329#define PG_HAVE_ATOMIC_INIT_U64330static inline void331pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_)332{333	ptr->value = val_;334}335#endif336 337#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)338#define PG_HAVE_ATOMIC_FETCH_ADD_U64339static inline uint64340pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)341{342	uint64 old;343	old = ptr->value;			/* ok if read is not atomic */344	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_))345		/* skip */;346	return old;347}348#endif349 350#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)351#define PG_HAVE_ATOMIC_FETCH_SUB_U64352static inline uint64353pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)354{355	return pg_atomic_fetch_add_u64_impl(ptr, -sub_);356}357#endif358 359#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)360#define PG_HAVE_ATOMIC_FETCH_AND_U64361static inline uint64362pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_)363{364	uint64 old;365	old = ptr->value;			/* ok if read is not atomic */366	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_))367		/* skip */;368	return old;369}370#endif371 372#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)373#define PG_HAVE_ATOMIC_FETCH_OR_U64374static inline uint64375pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_)376{377	uint64 old;378	old = ptr->value;			/* ok if read is not atomic */379	while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_))380		/* skip */;381	return old;382}383#endif384 385#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64)386#define PG_HAVE_ATOMIC_ADD_FETCH_U64387static inline uint64388pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)389{390	return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_;391}392#endif393 394#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64)395#define PG_HAVE_ATOMIC_SUB_FETCH_U64396static inline uint64397pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)398{399	return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_;400}401#endif402