1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2014 Regents of the University of California
4 */
5
6 #ifndef _ASM_RISCV_CMPXCHG_H
7 #define _ASM_RISCV_CMPXCHG_H
8
9 #include <linux/bug.h>
10
11 #include <asm/barrier.h>
12 #include <asm/fence.h>
13
14 #define __xchg_relaxed(ptr, new, size) \
15 ({ \
16 __typeof__(ptr) __ptr = (ptr); \
17 __typeof__(new) __new = (new); \
18 __typeof__(*(ptr)) __ret; \
19 switch (size) { \
20 case 4: \
21 __asm__ __volatile__ ( \
22 " amoswap.w %0, %2, %1\n" \
23 : "=r" (__ret), "+A" (*__ptr) \
24 : "r" (__new) \
25 : "memory"); \
26 break; \
27 case 8: \
28 __asm__ __volatile__ ( \
29 " amoswap.d %0, %2, %1\n" \
30 : "=r" (__ret), "+A" (*__ptr) \
31 : "r" (__new) \
32 : "memory"); \
33 break; \
34 default: \
35 BUILD_BUG(); \
36 } \
37 __ret; \
38 })
39
40 #define xchg_relaxed(ptr, x) \
41 ({ \
42 __typeof__(*(ptr)) _x_ = (x); \
43 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
44 _x_, sizeof(*(ptr))); \
45 })
46
47 #define __xchg_acquire(ptr, new, size) \
48 ({ \
49 __typeof__(ptr) __ptr = (ptr); \
50 __typeof__(new) __new = (new); \
51 __typeof__(*(ptr)) __ret; \
52 switch (size) { \
53 case 4: \
54 __asm__ __volatile__ ( \
55 " amoswap.w %0, %2, %1\n" \
56 RISCV_ACQUIRE_BARRIER \
57 : "=r" (__ret), "+A" (*__ptr) \
58 : "r" (__new) \
59 : "memory"); \
60 break; \
61 case 8: \
62 __asm__ __volatile__ ( \
63 " amoswap.d %0, %2, %1\n" \
64 RISCV_ACQUIRE_BARRIER \
65 : "=r" (__ret), "+A" (*__ptr) \
66 : "r" (__new) \
67 : "memory"); \
68 break; \
69 default: \
70 BUILD_BUG(); \
71 } \
72 __ret; \
73 })
74
75 #define xchg_acquire(ptr, x) \
76 ({ \
77 __typeof__(*(ptr)) _x_ = (x); \
78 (__typeof__(*(ptr))) __xchg_acquire((ptr), \
79 _x_, sizeof(*(ptr))); \
80 })
81
82 #define __xchg_release(ptr, new, size) \
83 ({ \
84 __typeof__(ptr) __ptr = (ptr); \
85 __typeof__(new) __new = (new); \
86 __typeof__(*(ptr)) __ret; \
87 switch (size) { \
88 case 4: \
89 __asm__ __volatile__ ( \
90 RISCV_RELEASE_BARRIER \
91 " amoswap.w %0, %2, %1\n" \
92 : "=r" (__ret), "+A" (*__ptr) \
93 : "r" (__new) \
94 : "memory"); \
95 break; \
96 case 8: \
97 __asm__ __volatile__ ( \
98 RISCV_RELEASE_BARRIER \
99 " amoswap.d %0, %2, %1\n" \
100 : "=r" (__ret), "+A" (*__ptr) \
101 : "r" (__new) \
102 : "memory"); \
103 break; \
104 default: \
105 BUILD_BUG(); \
106 } \
107 __ret; \
108 })
109
110 #define xchg_release(ptr, x) \
111 ({ \
112 __typeof__(*(ptr)) _x_ = (x); \
113 (__typeof__(*(ptr))) __xchg_release((ptr), \
114 _x_, sizeof(*(ptr))); \
115 })
116
117 #define __xchg(ptr, new, size) \
118 ({ \
119 __typeof__(ptr) __ptr = (ptr); \
120 __typeof__(new) __new = (new); \
121 __typeof__(*(ptr)) __ret; \
122 switch (size) { \
123 case 4: \
124 __asm__ __volatile__ ( \
125 " amoswap.w.aqrl %0, %2, %1\n" \
126 : "=r" (__ret), "+A" (*__ptr) \
127 : "r" (__new) \
128 : "memory"); \
129 break; \
130 case 8: \
131 __asm__ __volatile__ ( \
132 " amoswap.d.aqrl %0, %2, %1\n" \
133 : "=r" (__ret), "+A" (*__ptr) \
134 : "r" (__new) \
135 : "memory"); \
136 break; \
137 default: \
138 BUILD_BUG(); \
139 } \
140 __ret; \
141 })
142
143 #define xchg(ptr, x) \
144 ({ \
145 __typeof__(*(ptr)) _x_ = (x); \
146 (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
147 })
148
149 #define xchg32(ptr, x) \
150 ({ \
151 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
152 xchg((ptr), (x)); \
153 })
154
155 #define xchg64(ptr, x) \
156 ({ \
157 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
158 xchg((ptr), (x)); \
159 })
160
161 /*
162 * Atomic compare and exchange. Compare OLD with MEM, if identical,
163 * store NEW in MEM. Return the initial value in MEM. Success is
164 * indicated by comparing RETURN with OLD.
165 */
166 #define __cmpxchg_relaxed(ptr, old, new, size) \
167 ({ \
168 __typeof__(ptr) __ptr = (ptr); \
169 __typeof__(*(ptr)) __old = (old); \
170 __typeof__(*(ptr)) __new = (new); \
171 __typeof__(*(ptr)) __ret; \
172 register unsigned int __rc; \
173 switch (size) { \
174 case 4: \
175 __asm__ __volatile__ ( \
176 "0: lr.w %0, %2\n" \
177 " bne %0, %z3, 1f\n" \
178 " sc.w %1, %z4, %2\n" \
179 " bnez %1, 0b\n" \
180 "1:\n" \
181 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
182 : "rJ" (__old), "rJ" (__new) \
183 : "memory"); \
184 break; \
185 case 8: \
186 __asm__ __volatile__ ( \
187 "0: lr.d %0, %2\n" \
188 " bne %0, %z3, 1f\n" \
189 " sc.d %1, %z4, %2\n" \
190 " bnez %1, 0b\n" \
191 "1:\n" \
192 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
193 : "rJ" (__old), "rJ" (__new) \
194 : "memory"); \
195 break; \
196 default: \
197 BUILD_BUG(); \
198 } \
199 __ret; \
200 })
201
202 #define cmpxchg_relaxed(ptr, o, n) \
203 ({ \
204 __typeof__(*(ptr)) _o_ = (o); \
205 __typeof__(*(ptr)) _n_ = (n); \
206 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
207 _o_, _n_, sizeof(*(ptr))); \
208 })
209
210 #define __cmpxchg_acquire(ptr, old, new, size) \
211 ({ \
212 __typeof__(ptr) __ptr = (ptr); \
213 __typeof__(*(ptr)) __old = (old); \
214 __typeof__(*(ptr)) __new = (new); \
215 __typeof__(*(ptr)) __ret; \
216 register unsigned int __rc; \
217 switch (size) { \
218 case 4: \
219 __asm__ __volatile__ ( \
220 "0: lr.w %0, %2\n" \
221 " bne %0, %z3, 1f\n" \
222 " sc.w %1, %z4, %2\n" \
223 " bnez %1, 0b\n" \
224 RISCV_ACQUIRE_BARRIER \
225 "1:\n" \
226 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
227 : "rJ" (__old), "rJ" (__new) \
228 : "memory"); \
229 break; \
230 case 8: \
231 __asm__ __volatile__ ( \
232 "0: lr.d %0, %2\n" \
233 " bne %0, %z3, 1f\n" \
234 " sc.d %1, %z4, %2\n" \
235 " bnez %1, 0b\n" \
236 RISCV_ACQUIRE_BARRIER \
237 "1:\n" \
238 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
239 : "rJ" (__old), "rJ" (__new) \
240 : "memory"); \
241 break; \
242 default: \
243 BUILD_BUG(); \
244 } \
245 __ret; \
246 })
247
248 #define cmpxchg_acquire(ptr, o, n) \
249 ({ \
250 __typeof__(*(ptr)) _o_ = (o); \
251 __typeof__(*(ptr)) _n_ = (n); \
252 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
253 _o_, _n_, sizeof(*(ptr))); \
254 })
255
256 #define __cmpxchg_release(ptr, old, new, size) \
257 ({ \
258 __typeof__(ptr) __ptr = (ptr); \
259 __typeof__(*(ptr)) __old = (old); \
260 __typeof__(*(ptr)) __new = (new); \
261 __typeof__(*(ptr)) __ret; \
262 register unsigned int __rc; \
263 switch (size) { \
264 case 4: \
265 __asm__ __volatile__ ( \
266 RISCV_RELEASE_BARRIER \
267 "0: lr.w %0, %2\n" \
268 " bne %0, %z3, 1f\n" \
269 " sc.w %1, %z4, %2\n" \
270 " bnez %1, 0b\n" \
271 "1:\n" \
272 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
273 : "rJ" (__old), "rJ" (__new) \
274 : "memory"); \
275 break; \
276 case 8: \
277 __asm__ __volatile__ ( \
278 RISCV_RELEASE_BARRIER \
279 "0: lr.d %0, %2\n" \
280 " bne %0, %z3, 1f\n" \
281 " sc.d %1, %z4, %2\n" \
282 " bnez %1, 0b\n" \
283 "1:\n" \
284 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
285 : "rJ" (__old), "rJ" (__new) \
286 : "memory"); \
287 break; \
288 default: \
289 BUILD_BUG(); \
290 } \
291 __ret; \
292 })
293
294 #define cmpxchg_release(ptr, o, n) \
295 ({ \
296 __typeof__(*(ptr)) _o_ = (o); \
297 __typeof__(*(ptr)) _n_ = (n); \
298 (__typeof__(*(ptr))) __cmpxchg_release((ptr), \
299 _o_, _n_, sizeof(*(ptr))); \
300 })
301
302 #define __cmpxchg(ptr, old, new, size) \
303 ({ \
304 __typeof__(ptr) __ptr = (ptr); \
305 __typeof__(*(ptr)) __old = (old); \
306 __typeof__(*(ptr)) __new = (new); \
307 __typeof__(*(ptr)) __ret; \
308 register unsigned int __rc; \
309 switch (size) { \
310 case 4: \
311 __asm__ __volatile__ ( \
312 "0: lr.w %0, %2\n" \
313 " bne %0, %z3, 1f\n" \
314 " sc.w.rl %1, %z4, %2\n" \
315 " bnez %1, 0b\n" \
316 " fence rw, rw\n" \
317 "1:\n" \
318 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
319 : "rJ" (__old), "rJ" (__new) \
320 : "memory"); \
321 break; \
322 case 8: \
323 __asm__ __volatile__ ( \
324 "0: lr.d %0, %2\n" \
325 " bne %0, %z3, 1f\n" \
326 " sc.d.rl %1, %z4, %2\n" \
327 " bnez %1, 0b\n" \
328 " fence rw, rw\n" \
329 "1:\n" \
330 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
331 : "rJ" (__old), "rJ" (__new) \
332 : "memory"); \
333 break; \
334 default: \
335 BUILD_BUG(); \
336 } \
337 __ret; \
338 })
339
340 #define cmpxchg(ptr, o, n) \
341 ({ \
342 __typeof__(*(ptr)) _o_ = (o); \
343 __typeof__(*(ptr)) _n_ = (n); \
344 (__typeof__(*(ptr))) __cmpxchg((ptr), \
345 _o_, _n_, sizeof(*(ptr))); \
346 })
347
348 #define cmpxchg_local(ptr, o, n) \
349 (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
350
351 #define cmpxchg32(ptr, o, n) \
352 ({ \
353 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
354 cmpxchg((ptr), (o), (n)); \
355 })
356
357 #define cmpxchg32_local(ptr, o, n) \
358 ({ \
359 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
360 cmpxchg_relaxed((ptr), (o), (n)) \
361 })
362
363 #define cmpxchg64(ptr, o, n) \
364 ({ \
365 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
366 cmpxchg((ptr), (o), (n)); \
367 })
368
369 #define cmpxchg64_local(ptr, o, n) \
370 ({ \
371 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
372 cmpxchg_relaxed((ptr), (o), (n)); \
373 })
374
375 #endif /* _ASM_RISCV_CMPXCHG_H */