40 #define __xchg_op(ptr, arg, op, lock)					\  argument
42 	        __typeof__ (*(ptr)) __ret = (arg);			\
43 		switch (sizeof(*(ptr))) {				\
46 				      : "+q" (__ret), "+m" (*(ptr))	\
51 				      : "+r" (__ret), "+m" (*(ptr))	\
56 				      : "+r" (__ret), "+m" (*(ptr))	\
61 				      : "+r" (__ret), "+m" (*(ptr))	\
76 #define xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")  argument
83 #define __raw_cmpxchg(ptr, old, new, size, lock)			\  argument
85 	__typeof__(*(ptr)) __ret;					\
86 	__typeof__(*(ptr)) __old = (old);				\
87 	__typeof__(*(ptr)) __new = (new);				\
91 		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
100 		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
109 		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
118 		volatile u64 *__ptr = (volatile u64 *)(ptr);		\
131 #define __cmpxchg(ptr, old, new, size)					\  argument
132 	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
134 #define __sync_cmpxchg(ptr, old, new, size)				\  argument
135 	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
137 #define __cmpxchg_local(ptr, old, new, size)				\  argument
138 	__raw_cmpxchg((ptr), (old), (new), (size), "")
146 #define cmpxchg(ptr, old, new)						\  argument
147 	__cmpxchg(ptr, old, new, sizeof(*(ptr)))
149 #define sync_cmpxchg(ptr, old, new)					\  argument
150 	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))
152 #define cmpxchg_local(ptr, old, new)					\  argument
153 	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))
163 #define __xadd(ptr, inc, lock)	__xchg_op((ptr), (inc), xadd, lock)  argument
164 #define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)  argument
165 #define xadd_sync(ptr, inc)	__xadd((ptr), (inc), "lock; ")  argument
166 #define xadd_local(ptr, inc)	__xadd((ptr), (inc), "")  argument
168 #define __add(ptr, inc, lock)						\  argument
170 	        __typeof__ (*(ptr)) __ret = (inc);			\
171 		switch (sizeof(*(ptr))) {				\
174 				      : "+m" (*(ptr)) : "qi" (inc)	\
179 				      : "+m" (*(ptr)) : "ri" (inc)	\
184 				      : "+m" (*(ptr)) : "ri" (inc)	\
189 				      : "+m" (*(ptr)) : "ri" (inc)	\
205 #define add_smp(ptr, inc)	__add((ptr), (inc), LOCK_PREFIX)  argument
206 #define add_sync(ptr, inc)	__add((ptr), (inc), "lock; ")  argument