[urcu.git] / include / urcu / uatomic / generic.h

// SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation.  All rights reserved.
// SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics.  All rights reserved.
// SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
// SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
// SPDX-FileCopyrightText: 2010 Paolo Bonzini
//
// SPDX-License-Identifier: LicenseRef-Boehm-GC

#ifndef _URCU_UATOMIC_GENERIC_H
#define _URCU_UATOMIC_GENERIC_H

/*
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <stdint.h>
#include <urcu/compiler.h>
#include <urcu/system.h>

#ifdef __cplusplus
extern "C" {
#endif

#ifndef uatomic_set
#define uatomic_set(addr, v)	((void) CMM_STORE_SHARED(*(addr), (v)))
#endif

#ifndef uatomic_read
#define uatomic_read(addr)	CMM_LOAD_SHARED(*(addr))
#endif

#if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR
#ifdef ILLEGAL_INSTR
static inline __attribute__((always_inline))
void _uatomic_link_error(void)
{
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__(ILLEGAL_INSTR);
}
#else
static inline __attribute__((always_inline, __noreturn__))
void _uatomic_link_error(void)
{
	__builtin_trap();
}
#endif

#else /* #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
extern void _uatomic_link_error(void);
#endif /* #else #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */

/* cmpxchg */

#ifndef uatomic_cmpxchg
static inline __attribute__((always_inline))
unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
				_new);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
				_new);
#endif
	case 4:
		return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
				_new);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
				_new);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) _uatomic_cmpxchg((addr),			      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))


/* uatomic_and */

#ifndef uatomic_and
static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val,
		  int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_and_and_fetch_1((uint8_t *) addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_and_and_fetch_2((uint16_t *) addr, val);
		return;
#endif
	case 4:
		__sync_and_and_fetch_4((uint32_t *) addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_and_and_fetch_8((uint64_t *) addr, val);
		return;
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#endif

/* uatomic_or */

#ifndef uatomic_or
static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val,
		 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		__sync_or_and_fetch_1((uint8_t *) addr, val);
		return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		__sync_or_and_fetch_2((uint16_t *) addr, val);
		return;
#endif
	case 4:
		__sync_or_and_fetch_4((uint32_t *) addr, val);
		return;
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		__sync_or_and_fetch_8((uint64_t *) addr, val);
		return;
#endif
	}
	_uatomic_link_error();
	return;
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#endif


/* uatomic_add_return */

#ifndef uatomic_add_return
static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
		return __sync_add_and_fetch_1((uint8_t *) addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
		return __sync_add_and_fetch_2((uint16_t *) addr, val);
#endif
	case 4:
		return __sync_add_and_fetch_4((uint32_t *) addr, val);
#if (CAA_BITS_PER_LONG == 64)
	case 8:
		return __sync_add_and_fetch_8((uint64_t *) addr, val);
#endif
	}
	_uatomic_link_error();
	return 0;
}


#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old;

		do {
			old = uatomic_read((uint8_t *) addr);
		} while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
				old, val));

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old;

		do {
			old = uatomic_read((uint16_t *) addr);
		} while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
				old, val));

		return old;
	}
#endif
	case 4:
	{
		uint32_t old;

		do {
			old = uatomic_read((uint32_t *) addr);
		} while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
				old, val));

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old;

		do {
			old = uatomic_read((uint64_t *) addr);
		} while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
				old, val));

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#else /* #ifndef uatomic_cmpxchg */

#ifndef uatomic_and
/* uatomic_and */

static inline __attribute__((always_inline))
void _uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
		} while (oldt != old);
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_and(addr, v)			\
	(_uatomic_and((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#endif /* #ifndef uatomic_and */

#ifndef uatomic_or
/* uatomic_or */

static inline __attribute__((always_inline))
void _uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
		} while (oldt != old);

		return;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
		} while (oldt != old);

		return;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
		} while (oldt != old);

		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
		} while (oldt != old);

		return;
	}
#endif
	}
	_uatomic_link_error();
}

#define uatomic_or(addr, v)			\
	(_uatomic_or((addr),			\
		caa_cast_long_keep_sign(v),	\
		sizeof(*(addr))))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#endif /* #ifndef uatomic_or */

#ifndef uatomic_add_return
/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint8_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint16_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint32_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint64_t *) addr,
                                               old, old + val);
		} while (oldt != old);

		return old + val;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) _uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_add_return */

#ifndef uatomic_xchg
/* xchg */

static inline __attribute__((always_inline))
unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
{
	switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
	case 1:
	{
		uint8_t old, oldt;

		oldt = uatomic_read((uint8_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint8_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
	case 2:
	{
		uint16_t old, oldt;

		oldt = uatomic_read((uint16_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint16_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	case 4:
	{
		uint32_t old, oldt;

		oldt = uatomic_read((uint32_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint32_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		uint64_t old, oldt;

		oldt = uatomic_read((uint64_t *) addr);
		do {
			old = oldt;
			oldt = uatomic_cmpxchg((uint64_t *) addr,
                                               old, val);
		} while (oldt != old);

		return old;
	}
#endif
	}
	_uatomic_link_error();
	return 0;
}

#define uatomic_xchg(addr, v)						    \
	((__typeof__(*(addr))) _uatomic_exchange((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))
#endif /* #ifndef uatomic_xchg */

#endif /* #else #ifndef uatomic_cmpxchg */

/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */

#ifndef uatomic_add
#define uatomic_add(addr, v)		(void)uatomic_add_return((addr), (v))
#define cmm_smp_mb__before_uatomic_add()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_add()		cmm_barrier()
#endif

#define uatomic_sub_return(addr, v)	\
	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
#define uatomic_sub(addr, v)		\
	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
#define cmm_smp_mb__before_uatomic_sub()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_sub()		cmm_smp_mb__after_uatomic_add()

#ifndef uatomic_inc
#define uatomic_inc(addr)		uatomic_add((addr), 1)
#define cmm_smp_mb__before_uatomic_inc()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_inc()		cmm_smp_mb__after_uatomic_add()
#endif

#ifndef uatomic_dec
#define uatomic_dec(addr)		uatomic_add((addr), -1)
#define cmm_smp_mb__before_uatomic_dec()	cmm_smp_mb__before_uatomic_add()
#define cmm_smp_mb__after_uatomic_dec()		cmm_smp_mb__after_uatomic_add()
#endif

#ifdef __cplusplus
}
#endif

#endif /* _URCU_UATOMIC_GENERIC_H */
Commit	Line	Data
	1	// SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation. All rights reserved.
	2	// SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics. All rights reserved.
	3	// SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
	4	// SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	// SPDX-FileCopyrightText: 2010 Paolo Bonzini
	6	//
	7	// SPDX-License-Identifier: LicenseRef-Boehm-GC
	8
	9	#ifndef _URCU_UATOMIC_GENERIC_H
	10	#define _URCU_UATOMIC_GENERIC_H
	11
	12	/*
	13	* Code inspired from libuatomic_ops-1.2, inherited in part from the
	14	* Boehm-Demers-Weiser conservative garbage collector.
	15	*/
	16
	17	#include <stdint.h>
	18	#include <urcu/compiler.h>
	19	#include <urcu/system.h>
	20
	21	#ifdef __cplusplus
	22	extern "C" {
	23	#endif
	24
	25	#ifndef uatomic_set
	26	#define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
	27	#endif
	28
	29	#ifndef uatomic_read
	30	#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
	31	#endif
	32
	33	#if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR
	34	#ifdef ILLEGAL_INSTR
	35	static inline __attribute__((always_inline))
	36	void _uatomic_link_error(void)
	37	{
	38	/*
	39	* generate an illegal instruction. Cannot catch this with
	40	* linker tricks when optimizations are disabled.
	41	*/
	42	__asm__ __volatile__(ILLEGAL_INSTR);
	43	}
	44	#else
	45	static inline __attribute__((always_inline, __noreturn__))
	46	void _uatomic_link_error(void)
	47	{
	48	__builtin_trap();
	49	}
	50	#endif
	51
	52	#else /* #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	53	extern void _uatomic_link_error(void);
	54	#endif /* #else #if !defined __OPTIMIZE__ \|\| defined UATOMIC_NO_LINK_ERROR */
	55
	56	/* cmpxchg */
	57
	58	#ifndef uatomic_cmpxchg
	59	static inline __attribute__((always_inline))
	60	unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
	61	unsigned long _new, int len)
	62	{
	63	switch (len) {
	64	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	65	case 1:
	66	return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
	67	_new);
	68	#endif
	69	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	70	case 2:
	71	return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
	72	_new);
	73	#endif
	74	case 4:
	75	return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
	76	_new);
	77	#if (CAA_BITS_PER_LONG == 64)
	78	case 8:
	79	return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
	80	_new);
	81	#endif
	82	}
	83	_uatomic_link_error();
	84	return 0;
	85	}
	86
	87
	88	#define uatomic_cmpxchg(addr, old, _new) \
	89	((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
	90	caa_cast_long_keep_sign(old), \
	91	caa_cast_long_keep_sign(_new),\
	92	sizeof(*(addr))))
	93
	94
	95	/* uatomic_and */
	96
	97	#ifndef uatomic_and
	98	static inline __attribute__((always_inline))
	99	void _uatomic_and(void *addr, unsigned long val,
	100	int len)
	101	{
	102	switch (len) {
	103	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	104	case 1:
	105	__sync_and_and_fetch_1((uint8_t *) addr, val);
	106	return;
	107	#endif
	108	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	109	case 2:
	110	__sync_and_and_fetch_2((uint16_t *) addr, val);
	111	return;
	112	#endif
	113	case 4:
	114	__sync_and_and_fetch_4((uint32_t *) addr, val);
	115	return;
	116	#if (CAA_BITS_PER_LONG == 64)
	117	case 8:
	118	__sync_and_and_fetch_8((uint64_t *) addr, val);
	119	return;
	120	#endif
	121	}
	122	_uatomic_link_error();
	123	}
	124
	125	#define uatomic_and(addr, v) \
	126	(_uatomic_and((addr), \
	127	caa_cast_long_keep_sign(v), \
	128	sizeof(*(addr))))
	129	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
	130	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
	131
	132	#endif
	133
	134	/* uatomic_or */
	135
	136	#ifndef uatomic_or
	137	static inline __attribute__((always_inline))
	138	void _uatomic_or(void *addr, unsigned long val,
	139	int len)
	140	{
	141	switch (len) {
	142	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	143	case 1:
	144	__sync_or_and_fetch_1((uint8_t *) addr, val);
	145	return;
	146	#endif
	147	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	148	case 2:
	149	__sync_or_and_fetch_2((uint16_t *) addr, val);
	150	return;
	151	#endif
	152	case 4:
	153	__sync_or_and_fetch_4((uint32_t *) addr, val);
	154	return;
	155	#if (CAA_BITS_PER_LONG == 64)
	156	case 8:
	157	__sync_or_and_fetch_8((uint64_t *) addr, val);
	158	return;
	159	#endif
	160	}
	161	_uatomic_link_error();
	162	return;
	163	}
	164
	165	#define uatomic_or(addr, v) \
	166	(_uatomic_or((addr), \
	167	caa_cast_long_keep_sign(v), \
	168	sizeof(*(addr))))
	169	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
	170	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
	171
	172	#endif
	173
	174
	175	/* uatomic_add_return */
	176
	177	#ifndef uatomic_add_return
	178	static inline __attribute__((always_inline))
	179	unsigned long _uatomic_add_return(void *addr, unsigned long val,
	180	int len)
	181	{
	182	switch (len) {
	183	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	184	case 1:
	185	return __sync_add_and_fetch_1((uint8_t *) addr, val);
	186	#endif
	187	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	188	case 2:
	189	return __sync_add_and_fetch_2((uint16_t *) addr, val);
	190	#endif
	191	case 4:
	192	return __sync_add_and_fetch_4((uint32_t *) addr, val);
	193	#if (CAA_BITS_PER_LONG == 64)
	194	case 8:
	195	return __sync_add_and_fetch_8((uint64_t *) addr, val);
	196	#endif
	197	}
	198	_uatomic_link_error();
	199	return 0;
	200	}
	201
	202
	203	#define uatomic_add_return(addr, v) \
	204	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	205	caa_cast_long_keep_sign(v), \
	206	sizeof(*(addr))))
	207	#endif /* #ifndef uatomic_add_return */
	208
	209	#ifndef uatomic_xchg
	210	/* xchg */
	211
	212	static inline __attribute__((always_inline))
	213	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	214	{
	215	switch (len) {
	216	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	217	case 1:
	218	{
	219	uint8_t old;
	220
	221	do {
	222	old = uatomic_read((uint8_t *) addr);
	223	} while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
	224	old, val));
	225
	226	return old;
	227	}
	228	#endif
	229	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	230	case 2:
	231	{
	232	uint16_t old;
	233
	234	do {
	235	old = uatomic_read((uint16_t *) addr);
	236	} while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
	237	old, val));
	238
	239	return old;
	240	}
	241	#endif
	242	case 4:
	243	{
	244	uint32_t old;
	245
	246	do {
	247	old = uatomic_read((uint32_t *) addr);
	248	} while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
	249	old, val));
	250
	251	return old;
	252	}
	253	#if (CAA_BITS_PER_LONG == 64)
	254	case 8:
	255	{
	256	uint64_t old;
	257
	258	do {
	259	old = uatomic_read((uint64_t *) addr);
	260	} while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
	261	old, val));
	262
	263	return old;
	264	}
	265	#endif
	266	}
	267	_uatomic_link_error();
	268	return 0;
	269	}
	270
	271	#define uatomic_xchg(addr, v) \
	272	((__typeof__(*(addr))) _uatomic_exchange((addr), \
	273	caa_cast_long_keep_sign(v), \
	274	sizeof(*(addr))))
	275	#endif /* #ifndef uatomic_xchg */
	276
	277	#else /* #ifndef uatomic_cmpxchg */
	278
	279	#ifndef uatomic_and
	280	/* uatomic_and */
	281
	282	static inline __attribute__((always_inline))
	283	void _uatomic_and(void *addr, unsigned long val, int len)
	284	{
	285	switch (len) {
	286	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	287	case 1:
	288	{
	289	uint8_t old, oldt;
	290
	291	oldt = uatomic_read((uint8_t *) addr);
	292	do {
	293	old = oldt;
	294	oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
	295	} while (oldt != old);
	296
	297	return;
	298	}
	299	#endif
	300	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	301	case 2:
	302	{
	303	uint16_t old, oldt;
	304
	305	oldt = uatomic_read((uint16_t *) addr);
	306	do {
	307	old = oldt;
	308	oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
	309	} while (oldt != old);
	310	}
	311	#endif
	312	case 4:
	313	{
	314	uint32_t old, oldt;
	315
	316	oldt = uatomic_read((uint32_t *) addr);
	317	do {
	318	old = oldt;
	319	oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
	320	} while (oldt != old);
	321
	322	return;
	323	}
	324	#if (CAA_BITS_PER_LONG == 64)
	325	case 8:
	326	{
	327	uint64_t old, oldt;
	328
	329	oldt = uatomic_read((uint64_t *) addr);
	330	do {
	331	old = oldt;
	332	oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
	333	} while (oldt != old);
	334
	335	return;
	336	}
	337	#endif
	338	}
	339	_uatomic_link_error();
	340	}
	341
	342	#define uatomic_and(addr, v) \
	343	(_uatomic_and((addr), \
	344	caa_cast_long_keep_sign(v), \
	345	sizeof(*(addr))))
	346	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
	347	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
	348
	349	#endif /* #ifndef uatomic_and */
	350
	351	#ifndef uatomic_or
	352	/* uatomic_or */
	353
	354	static inline __attribute__((always_inline))
	355	void _uatomic_or(void *addr, unsigned long val, int len)
	356	{
	357	switch (len) {
	358	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	359	case 1:
	360	{
	361	uint8_t old, oldt;
	362
	363	oldt = uatomic_read((uint8_t *) addr);
	364	do {
	365	old = oldt;
	366	oldt = _uatomic_cmpxchg(addr, old, old \| val, 1);
	367	} while (oldt != old);
	368
	369	return;
	370	}
	371	#endif
	372	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	373	case 2:
	374	{
	375	uint16_t old, oldt;
	376
	377	oldt = uatomic_read((uint16_t *) addr);
	378	do {
	379	old = oldt;
	380	oldt = _uatomic_cmpxchg(addr, old, old \| val, 2);
	381	} while (oldt != old);
	382
	383	return;
	384	}
	385	#endif
	386	case 4:
	387	{
	388	uint32_t old, oldt;
	389
	390	oldt = uatomic_read((uint32_t *) addr);
	391	do {
	392	old = oldt;
	393	oldt = _uatomic_cmpxchg(addr, old, old \| val, 4);
	394	} while (oldt != old);
	395
	396	return;
	397	}
	398	#if (CAA_BITS_PER_LONG == 64)
	399	case 8:
	400	{
	401	uint64_t old, oldt;
	402
	403	oldt = uatomic_read((uint64_t *) addr);
	404	do {
	405	old = oldt;
	406	oldt = _uatomic_cmpxchg(addr, old, old \| val, 8);
	407	} while (oldt != old);
	408
	409	return;
	410	}
	411	#endif
	412	}
	413	_uatomic_link_error();
	414	}
	415
	416	#define uatomic_or(addr, v) \
	417	(_uatomic_or((addr), \
	418	caa_cast_long_keep_sign(v), \
	419	sizeof(*(addr))))
	420	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
	421	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
	422
	423	#endif /* #ifndef uatomic_or */
	424
	425	#ifndef uatomic_add_return
	426	/* uatomic_add_return */
	427
	428	static inline __attribute__((always_inline))
	429	unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
	430	{
	431	switch (len) {
	432	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	433	case 1:
	434	{
	435	uint8_t old, oldt;
	436
	437	oldt = uatomic_read((uint8_t *) addr);
	438	do {
	439	old = oldt;
	440	oldt = uatomic_cmpxchg((uint8_t *) addr,
	441	old, old + val);
	442	} while (oldt != old);
	443
	444	return old + val;
	445	}
	446	#endif
	447	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	448	case 2:
	449	{
	450	uint16_t old, oldt;
	451
	452	oldt = uatomic_read((uint16_t *) addr);
	453	do {
	454	old = oldt;
	455	oldt = uatomic_cmpxchg((uint16_t *) addr,
	456	old, old + val);
	457	} while (oldt != old);
	458
	459	return old + val;
	460	}
	461	#endif
	462	case 4:
	463	{
	464	uint32_t old, oldt;
	465
	466	oldt = uatomic_read((uint32_t *) addr);
	467	do {
	468	old = oldt;
	469	oldt = uatomic_cmpxchg((uint32_t *) addr,
	470	old, old + val);
	471	} while (oldt != old);
	472
	473	return old + val;
	474	}
	475	#if (CAA_BITS_PER_LONG == 64)
	476	case 8:
	477	{
	478	uint64_t old, oldt;
	479
	480	oldt = uatomic_read((uint64_t *) addr);
	481	do {
	482	old = oldt;
	483	oldt = uatomic_cmpxchg((uint64_t *) addr,
	484	old, old + val);
	485	} while (oldt != old);
	486
	487	return old + val;
	488	}
	489	#endif
	490	}
	491	_uatomic_link_error();
	492	return 0;
	493	}
	494
	495	#define uatomic_add_return(addr, v) \
	496	((__typeof__(*(addr))) _uatomic_add_return((addr), \
	497	caa_cast_long_keep_sign(v), \
	498	sizeof(*(addr))))
	499	#endif /* #ifndef uatomic_add_return */
	500
	501	#ifndef uatomic_xchg
	502	/* xchg */
	503
	504	static inline __attribute__((always_inline))
	505	unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
	506	{
	507	switch (len) {
	508	#ifdef UATOMIC_HAS_ATOMIC_BYTE
	509	case 1:
	510	{
	511	uint8_t old, oldt;
	512
	513	oldt = uatomic_read((uint8_t *) addr);
	514	do {
	515	old = oldt;
	516	oldt = uatomic_cmpxchg((uint8_t *) addr,
	517	old, val);
	518	} while (oldt != old);
	519
	520	return old;
	521	}
	522	#endif
	523	#ifdef UATOMIC_HAS_ATOMIC_SHORT
	524	case 2:
	525	{
	526	uint16_t old, oldt;
	527
	528	oldt = uatomic_read((uint16_t *) addr);
	529	do {
	530	old = oldt;
	531	oldt = uatomic_cmpxchg((uint16_t *) addr,
	532	old, val);
	533	} while (oldt != old);
	534
	535	return old;
	536	}
	537	#endif
	538	case 4:
	539	{
	540	uint32_t old, oldt;
	541
	542	oldt = uatomic_read((uint32_t *) addr);
	543	do {
	544	old = oldt;
	545	oldt = uatomic_cmpxchg((uint32_t *) addr,
	546	old, val);
	547	} while (oldt != old);
	548
	549	return old;
	550	}
	551	#if (CAA_BITS_PER_LONG == 64)
	552	case 8:
	553	{
	554	uint64_t old, oldt;
	555
	556	oldt = uatomic_read((uint64_t *) addr);
	557	do {
	558	old = oldt;
	559	oldt = uatomic_cmpxchg((uint64_t *) addr,
	560	old, val);
	561	} while (oldt != old);
	562
	563	return old;
	564	}
	565	#endif
	566	}
	567	_uatomic_link_error();
	568	return 0;
	569	}
	570
	571	#define uatomic_xchg(addr, v) \
	572	((__typeof__(*(addr))) _uatomic_exchange((addr), \
	573	caa_cast_long_keep_sign(v), \
	574	sizeof(*(addr))))
	575	#endif /* #ifndef uatomic_xchg */
	576
	577	#endif /* #else #ifndef uatomic_cmpxchg */
	578
	579	/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
	580
	581	#ifndef uatomic_add
	582	#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
	583	#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
	584	#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
	585	#endif
	586
	587	#define uatomic_sub_return(addr, v) \
	588	uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
	589	#define uatomic_sub(addr, v) \
	590	uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
	591	#define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
	592	#define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
	593
	594	#ifndef uatomic_inc
	595	#define uatomic_inc(addr) uatomic_add((addr), 1)
	596	#define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
	597	#define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
	598	#endif
	599
	600	#ifndef uatomic_dec
	601	#define uatomic_dec(addr) uatomic_add((addr), -1)
	602	#define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
	603	#define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
	604	#endif
	605
	606	#ifdef __cplusplus
	607	}
	608	#endif
	609
	610	#endif /* _URCU_UATOMIC_GENERIC_H */