This source file includes following definitions.
- rseq_cmpeqv_storev
- rseq_cmpnev_storeoffp_load
- rseq_addv
- rseq_cmpeqv_trystorev_storev
- rseq_cmpeqv_trystorev_storev_release
- rseq_cmpeqv_cmpeqv_storev
- rseq_cmpeqv_trymemcpy_storev
- rseq_cmpeqv_trymemcpy_storev_release
1
2
3
4
5
6
7
8
9
10
11
12
13
14 #define RSEQ_SIG_CODE 0xd428bc00
15
16 #ifdef __AARCH64EB__
17 #define RSEQ_SIG_DATA 0x00bc28d4
18 #else
19 #define RSEQ_SIG_DATA RSEQ_SIG_CODE
20 #endif
21
22 #define RSEQ_SIG RSEQ_SIG_DATA
23
24 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
27
28 #define rseq_smp_load_acquire(p) \
29 __extension__ ({ \
30 __typeof(*p) ____p1; \
31 switch (sizeof(*p)) { \
32 case 1: \
33 asm volatile ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)p) \
35 : "Q" (*p) : "memory"); \
36 break; \
37 case 2: \
38 asm volatile ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)p) \
40 : "Q" (*p) : "memory"); \
41 break; \
42 case 4: \
43 asm volatile ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)p) \
45 : "Q" (*p) : "memory"); \
46 break; \
47 case 8: \
48 asm volatile ("ldar %0, %1" \
49 : "=r" (*(__u64 *)p) \
50 : "Q" (*p) : "memory"); \
51 break; \
52 } \
53 ____p1; \
54 })
55
56 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
57
58 #define rseq_smp_store_release(p, v) \
59 do { \
60 switch (sizeof(*p)) { \
61 case 1: \
62 asm volatile ("stlrb %w1, %0" \
63 : "=Q" (*p) \
64 : "r" ((__u8)v) \
65 : "memory"); \
66 break; \
67 case 2: \
68 asm volatile ("stlrh %w1, %0" \
69 : "=Q" (*p) \
70 : "r" ((__u16)v) \
71 : "memory"); \
72 break; \
73 case 4: \
74 asm volatile ("stlr %w1, %0" \
75 : "=Q" (*p) \
76 : "r" ((__u32)v) \
77 : "memory"); \
78 break; \
79 case 8: \
80 asm volatile ("stlr %1, %0" \
81 : "=Q" (*p) \
82 : "r" ((__u64)v) \
83 : "memory"); \
84 break; \
85 } \
86 } while (0)
87
88 #ifdef RSEQ_SKIP_FASTPATH
89 #include "rseq-skip.h"
90 #else
91
92 #define RSEQ_ASM_TMP_REG32 "w15"
93 #define RSEQ_ASM_TMP_REG "x15"
94 #define RSEQ_ASM_TMP_REG_2 "x14"
95
96 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
97 post_commit_offset, abort_ip) \
98 " .pushsection __rseq_cs, \"aw\"\n" \
99 " .balign 32\n" \
100 __rseq_str(label) ":\n" \
101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
102 " .quad " __rseq_str(start_ip) ", " \
103 __rseq_str(post_commit_offset) ", " \
104 __rseq_str(abort_ip) "\n" \
105 " .popsection\n\t" \
106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
107 " .quad " __rseq_str(label) "b\n" \
108 " .popsection\n"
109
110 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
112 (post_commit_ip - start_ip), abort_ip)
113
114
115
116
117
118
119
120
121
122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
125 " .popsection\n"
126
127 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
128 RSEQ_INJECT_ASM(1) \
129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
130 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
131 ", :lo12:" __rseq_str(cs_label) "\n" \
132 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
133 __rseq_str(label) ":\n"
134
135 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
136 " b 222f\n" \
137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
138 __rseq_str(label) ":\n" \
139 " b %l[" __rseq_str(abort_label) "]\n" \
140 "222:\n"
141
142 #define RSEQ_ASM_OP_STORE(value, var) \
143 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
144
145 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
146 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
147
148 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
149 RSEQ_ASM_OP_STORE(value, var) \
150 __rseq_str(post_commit_label) ":\n"
151
152 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
153 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
154 __rseq_str(post_commit_label) ":\n"
155
156 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
157 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
158 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
159 ", %[" __rseq_str(expect) "]\n" \
160 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
161
162 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
163 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
164 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
165 ", %w[" __rseq_str(expect) "]\n" \
166 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
167
168 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \
169 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
170 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
171 ", %[" __rseq_str(expect) "]\n" \
172 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
173
174 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
175 RSEQ_INJECT_ASM(2) \
176 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
177
178 #define RSEQ_ASM_OP_R_LOAD(var) \
179 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
180
181 #define RSEQ_ASM_OP_R_STORE(var) \
182 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
183
184 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
185 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
186 ", %[" __rseq_str(offset) "]]\n"
187
188 #define RSEQ_ASM_OP_R_ADD(count) \
189 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
190 ", %[" __rseq_str(count) "]\n"
191
192 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
193 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
194 __rseq_str(post_commit_label) ":\n"
195
196 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
197 " cbz %[" __rseq_str(len) "], 333f\n" \
198 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
199 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
200 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
202 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
205 "333:\n"
206
207 static inline __attribute__((always_inline))
208 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
209 {
210 RSEQ_INJECT_C(9)
211
212 __asm__ __volatile__ goto (
213 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
215 #ifdef RSEQ_COMPARE_TWICE
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
217 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
218 #endif
219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221 RSEQ_INJECT_ASM(3)
222 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
223 RSEQ_INJECT_ASM(4)
224 #ifdef RSEQ_COMPARE_TWICE
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
226 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
227 #endif
228 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
229 RSEQ_INJECT_ASM(5)
230 RSEQ_ASM_DEFINE_ABORT(4, abort)
231 :
232 : [cpu_id] "r" (cpu),
233 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
234 [rseq_cs] "m" (__rseq_abi.rseq_cs),
235 [v] "Qo" (*v),
236 [expect] "r" (expect),
237 [newv] "r" (newv)
238 RSEQ_INJECT_INPUT
239 : "memory", RSEQ_ASM_TMP_REG
240 : abort, cmpfail
241 #ifdef RSEQ_COMPARE_TWICE
242 , error1, error2
243 #endif
244 );
245
246 return 0;
247 abort:
248 RSEQ_INJECT_FAILED
249 return -1;
250 cmpfail:
251 return 1;
252 #ifdef RSEQ_COMPARE_TWICE
253 error1:
254 rseq_bug("cpu_id comparison failed");
255 error2:
256 rseq_bug("expected value comparison failed");
257 #endif
258 }
259
260 static inline __attribute__((always_inline))
261 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
262 off_t voffp, intptr_t *load, int cpu)
263 {
264 RSEQ_INJECT_C(9)
265
266 __asm__ __volatile__ goto (
267 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
268 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
269 #ifdef RSEQ_COMPARE_TWICE
270 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
271 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
272 #endif
273 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
275 RSEQ_INJECT_ASM(3)
276 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
277 RSEQ_INJECT_ASM(4)
278 #ifdef RSEQ_COMPARE_TWICE
279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
280 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
281 #endif
282 RSEQ_ASM_OP_R_LOAD(v)
283 RSEQ_ASM_OP_R_STORE(load)
284 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
285 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
286 RSEQ_INJECT_ASM(5)
287 RSEQ_ASM_DEFINE_ABORT(4, abort)
288 :
289 : [cpu_id] "r" (cpu),
290 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
291 [rseq_cs] "m" (__rseq_abi.rseq_cs),
292 [v] "Qo" (*v),
293 [expectnot] "r" (expectnot),
294 [load] "Qo" (*load),
295 [voffp] "r" (voffp)
296 RSEQ_INJECT_INPUT
297 : "memory", RSEQ_ASM_TMP_REG
298 : abort, cmpfail
299 #ifdef RSEQ_COMPARE_TWICE
300 , error1, error2
301 #endif
302 );
303 return 0;
304 abort:
305 RSEQ_INJECT_FAILED
306 return -1;
307 cmpfail:
308 return 1;
309 #ifdef RSEQ_COMPARE_TWICE
310 error1:
311 rseq_bug("cpu_id comparison failed");
312 error2:
313 rseq_bug("expected value comparison failed");
314 #endif
315 }
316
317 static inline __attribute__((always_inline))
318 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
319 {
320 RSEQ_INJECT_C(9)
321
322 __asm__ __volatile__ goto (
323 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
324 #ifdef RSEQ_COMPARE_TWICE
325 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
326 #endif
327 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
329 RSEQ_INJECT_ASM(3)
330 #ifdef RSEQ_COMPARE_TWICE
331 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
332 #endif
333 RSEQ_ASM_OP_R_LOAD(v)
334 RSEQ_ASM_OP_R_ADD(count)
335 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
336 RSEQ_INJECT_ASM(4)
337 RSEQ_ASM_DEFINE_ABORT(4, abort)
338 :
339 : [cpu_id] "r" (cpu),
340 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
341 [rseq_cs] "m" (__rseq_abi.rseq_cs),
342 [v] "Qo" (*v),
343 [count] "r" (count)
344 RSEQ_INJECT_INPUT
345 : "memory", RSEQ_ASM_TMP_REG
346 : abort
347 #ifdef RSEQ_COMPARE_TWICE
348 , error1
349 #endif
350 );
351 return 0;
352 abort:
353 RSEQ_INJECT_FAILED
354 return -1;
355 #ifdef RSEQ_COMPARE_TWICE
356 error1:
357 rseq_bug("cpu_id comparison failed");
358 #endif
359 }
360
361 static inline __attribute__((always_inline))
362 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
363 intptr_t *v2, intptr_t newv2,
364 intptr_t newv, int cpu)
365 {
366 RSEQ_INJECT_C(9)
367
368 __asm__ __volatile__ goto (
369 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
370 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
371 #ifdef RSEQ_COMPARE_TWICE
372 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
373 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
374 #endif
375 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
376 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
377 RSEQ_INJECT_ASM(3)
378 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
379 RSEQ_INJECT_ASM(4)
380 #ifdef RSEQ_COMPARE_TWICE
381 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
382 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
383 #endif
384 RSEQ_ASM_OP_STORE(newv2, v2)
385 RSEQ_INJECT_ASM(5)
386 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
387 RSEQ_INJECT_ASM(6)
388 RSEQ_ASM_DEFINE_ABORT(4, abort)
389 :
390 : [cpu_id] "r" (cpu),
391 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
392 [rseq_cs] "m" (__rseq_abi.rseq_cs),
393 [expect] "r" (expect),
394 [v] "Qo" (*v),
395 [newv] "r" (newv),
396 [v2] "Qo" (*v2),
397 [newv2] "r" (newv2)
398 RSEQ_INJECT_INPUT
399 : "memory", RSEQ_ASM_TMP_REG
400 : abort, cmpfail
401 #ifdef RSEQ_COMPARE_TWICE
402 , error1, error2
403 #endif
404 );
405
406 return 0;
407 abort:
408 RSEQ_INJECT_FAILED
409 return -1;
410 cmpfail:
411 return 1;
412 #ifdef RSEQ_COMPARE_TWICE
413 error1:
414 rseq_bug("cpu_id comparison failed");
415 error2:
416 rseq_bug("expected value comparison failed");
417 #endif
418 }
419
420 static inline __attribute__((always_inline))
421 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
422 intptr_t *v2, intptr_t newv2,
423 intptr_t newv, int cpu)
424 {
425 RSEQ_INJECT_C(9)
426
427 __asm__ __volatile__ goto (
428 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
429 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
430 #ifdef RSEQ_COMPARE_TWICE
431 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
432 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
433 #endif
434 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
436 RSEQ_INJECT_ASM(3)
437 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
438 RSEQ_INJECT_ASM(4)
439 #ifdef RSEQ_COMPARE_TWICE
440 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
441 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
442 #endif
443 RSEQ_ASM_OP_STORE(newv2, v2)
444 RSEQ_INJECT_ASM(5)
445 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
446 RSEQ_INJECT_ASM(6)
447 RSEQ_ASM_DEFINE_ABORT(4, abort)
448 :
449 : [cpu_id] "r" (cpu),
450 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
451 [rseq_cs] "m" (__rseq_abi.rseq_cs),
452 [expect] "r" (expect),
453 [v] "Qo" (*v),
454 [newv] "r" (newv),
455 [v2] "Qo" (*v2),
456 [newv2] "r" (newv2)
457 RSEQ_INJECT_INPUT
458 : "memory", RSEQ_ASM_TMP_REG
459 : abort, cmpfail
460 #ifdef RSEQ_COMPARE_TWICE
461 , error1, error2
462 #endif
463 );
464
465 return 0;
466 abort:
467 RSEQ_INJECT_FAILED
468 return -1;
469 cmpfail:
470 return 1;
471 #ifdef RSEQ_COMPARE_TWICE
472 error1:
473 rseq_bug("cpu_id comparison failed");
474 error2:
475 rseq_bug("expected value comparison failed");
476 #endif
477 }
478
479 static inline __attribute__((always_inline))
480 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
481 intptr_t *v2, intptr_t expect2,
482 intptr_t newv, int cpu)
483 {
484 RSEQ_INJECT_C(9)
485
486 __asm__ __volatile__ goto (
487 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
488 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
489 #ifdef RSEQ_COMPARE_TWICE
490 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
491 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
492 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
493 #endif
494 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
495 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
496 RSEQ_INJECT_ASM(3)
497 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
498 RSEQ_INJECT_ASM(4)
499 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
500 RSEQ_INJECT_ASM(5)
501 #ifdef RSEQ_COMPARE_TWICE
502 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
503 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
504 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
505 #endif
506 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
507 RSEQ_INJECT_ASM(6)
508 RSEQ_ASM_DEFINE_ABORT(4, abort)
509 :
510 : [cpu_id] "r" (cpu),
511 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
512 [rseq_cs] "m" (__rseq_abi.rseq_cs),
513 [v] "Qo" (*v),
514 [expect] "r" (expect),
515 [v2] "Qo" (*v2),
516 [expect2] "r" (expect2),
517 [newv] "r" (newv)
518 RSEQ_INJECT_INPUT
519 : "memory", RSEQ_ASM_TMP_REG
520 : abort, cmpfail
521 #ifdef RSEQ_COMPARE_TWICE
522 , error1, error2, error3
523 #endif
524 );
525
526 return 0;
527 abort:
528 RSEQ_INJECT_FAILED
529 return -1;
530 cmpfail:
531 return 1;
532 #ifdef RSEQ_COMPARE_TWICE
533 error1:
534 rseq_bug("cpu_id comparison failed");
535 error2:
536 rseq_bug("expected value comparison failed");
537 error3:
538 rseq_bug("2nd expected value comparison failed");
539 #endif
540 }
541
542 static inline __attribute__((always_inline))
543 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
544 void *dst, void *src, size_t len,
545 intptr_t newv, int cpu)
546 {
547 RSEQ_INJECT_C(9)
548
549 __asm__ __volatile__ goto (
550 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
551 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
552 #ifdef RSEQ_COMPARE_TWICE
553 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
554 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
555 #endif
556 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
557 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
558 RSEQ_INJECT_ASM(3)
559 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
560 RSEQ_INJECT_ASM(4)
561 #ifdef RSEQ_COMPARE_TWICE
562 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
563 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
564 #endif
565 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
566 RSEQ_INJECT_ASM(5)
567 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
568 RSEQ_INJECT_ASM(6)
569 RSEQ_ASM_DEFINE_ABORT(4, abort)
570 :
571 : [cpu_id] "r" (cpu),
572 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
573 [rseq_cs] "m" (__rseq_abi.rseq_cs),
574 [expect] "r" (expect),
575 [v] "Qo" (*v),
576 [newv] "r" (newv),
577 [dst] "r" (dst),
578 [src] "r" (src),
579 [len] "r" (len)
580 RSEQ_INJECT_INPUT
581 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
582 : abort, cmpfail
583 #ifdef RSEQ_COMPARE_TWICE
584 , error1, error2
585 #endif
586 );
587
588 return 0;
589 abort:
590 RSEQ_INJECT_FAILED
591 return -1;
592 cmpfail:
593 return 1;
594 #ifdef RSEQ_COMPARE_TWICE
595 error1:
596 rseq_bug("cpu_id comparison failed");
597 error2:
598 rseq_bug("expected value comparison failed");
599 #endif
600 }
601
602 static inline __attribute__((always_inline))
603 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
604 void *dst, void *src, size_t len,
605 intptr_t newv, int cpu)
606 {
607 RSEQ_INJECT_C(9)
608
609 __asm__ __volatile__ goto (
610 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
611 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
612 #ifdef RSEQ_COMPARE_TWICE
613 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
614 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
615 #endif
616 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 RSEQ_INJECT_ASM(3)
619 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
620 RSEQ_INJECT_ASM(4)
621 #ifdef RSEQ_COMPARE_TWICE
622 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
623 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
624 #endif
625 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
626 RSEQ_INJECT_ASM(5)
627 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
628 RSEQ_INJECT_ASM(6)
629 RSEQ_ASM_DEFINE_ABORT(4, abort)
630 :
631 : [cpu_id] "r" (cpu),
632 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
633 [rseq_cs] "m" (__rseq_abi.rseq_cs),
634 [expect] "r" (expect),
635 [v] "Qo" (*v),
636 [newv] "r" (newv),
637 [dst] "r" (dst),
638 [src] "r" (src),
639 [len] "r" (len)
640 RSEQ_INJECT_INPUT
641 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
642 : abort, cmpfail
643 #ifdef RSEQ_COMPARE_TWICE
644 , error1, error2
645 #endif
646 );
647
648 return 0;
649 abort:
650 RSEQ_INJECT_FAILED
651 return -1;
652 cmpfail:
653 return 1;
654 #ifdef RSEQ_COMPARE_TWICE
655 error1:
656 rseq_bug("cpu_id comparison failed");
657 error2:
658 rseq_bug("expected value comparison failed");
659 #endif
660 }
661
662 #endif