42 #define __xchg_op(ptr, arg, op, lock)					\  argument
44 	        __typeof__ (*(ptr)) __ret = (arg);			\
45 		switch (sizeof(*(ptr))) {				\
48 				      : "+q" (__ret), "+m" (*(ptr))	\
53 				      : "+r" (__ret), "+m" (*(ptr))	\
58 				      : "+r" (__ret), "+m" (*(ptr))	\
63 				      : "+r" (__ret), "+m" (*(ptr))	\
78 #define xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")  argument
85 #define __raw_cmpxchg(ptr, old, new, size, lock)			\  argument
87 	__typeof__(*(ptr)) __ret;					\
88 	__typeof__(*(ptr)) __old = (old);				\
89 	__typeof__(*(ptr)) __new = (new);				\
93 		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
102 		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
111 		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
120 		volatile u64 *__ptr = (volatile u64 *)(ptr);		\
133 #define __cmpxchg(ptr, old, new, size)					\  argument
134 	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
136 #define __sync_cmpxchg(ptr, old, new, size)				\  argument
137 	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
139 #define __cmpxchg_local(ptr, old, new, size)				\  argument
140 	__raw_cmpxchg((ptr), (old), (new), (size), "")
148 #define cmpxchg(ptr, old, new)						\  argument
149 	__cmpxchg(ptr, old, new, sizeof(*(ptr)))
151 #define sync_cmpxchg(ptr, old, new)					\  argument
152 	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))
154 #define cmpxchg_local(ptr, old, new)					\  argument
155 	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))
165 #define __xadd(ptr, inc, lock)	__xchg_op((ptr), (inc), xadd, lock)  argument
166 #define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)  argument
167 #define xadd_sync(ptr, inc)	__xadd((ptr), (inc), "lock; ")  argument
168 #define xadd_local(ptr, inc)	__xadd((ptr), (inc), "")  argument
170 #define __add(ptr, inc, lock)						\  argument
172 	        __typeof__ (*(ptr)) __ret = (inc);			\
173 		switch (sizeof(*(ptr))) {				\
176 				      : "+m" (*(ptr)) : "qi" (inc)	\
181 				      : "+m" (*(ptr)) : "ri" (inc)	\
186 				      : "+m" (*(ptr)) : "ri" (inc)	\
191 				      : "+m" (*(ptr)) : "ri" (inc)	\
207 #define add_smp(ptr, inc)	__add((ptr), (inc), LOCK_PREFIX)  argument
208 #define add_sync(ptr, inc)	__add((ptr), (inc), "lock; ")  argument