This source file includes following definitions.
- rseq_cmpeqv_storev
- rseq_cmpnev_storeoffp_load
- rseq_addv
- rseq_cmpeqv_trystorev_storev
- rseq_cmpeqv_trystorev_storev_release
- rseq_cmpeqv_cmpeqv_storev
- rseq_cmpeqv_trymemcpy_storev
- rseq_cmpeqv_trymemcpy_storev_release
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 #define RSEQ_SIG 0x0fe5000b
18
19 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
20 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
21 #define rseq_smp_rmb() rseq_smp_lwsync()
22 #define rseq_smp_wmb() rseq_smp_lwsync()
23
24 #define rseq_smp_load_acquire(p) \
25 __extension__ ({ \
26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
27 rseq_smp_lwsync(); \
28 ____p1; \
29 })
30
31 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
32
33 #define rseq_smp_store_release(p, v) \
34 do { \
35 rseq_smp_lwsync(); \
36 RSEQ_WRITE_ONCE(*p, v); \
37 } while (0)
38
39 #ifdef RSEQ_SKIP_FASTPATH
40 #include "rseq-skip.h"
41 #else
42
43
44
45
46
47
48 #ifdef __PPC64__
49
50 #define STORE_WORD "std "
51 #define LOAD_WORD "ld "
52 #define LOADX_WORD "ldx "
53 #define CMP_WORD "cmpd "
54
55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
56 start_ip, post_commit_offset, abort_ip) \
57 ".pushsection __rseq_cs, \"aw\"\n\t" \
58 ".balign 32\n\t" \
59 __rseq_str(label) ":\n\t" \
60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 ".popsection\n\t" \
63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
64 ".quad " __rseq_str(label) "b\n\t" \
65 ".popsection\n\t"
66
67 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
68 RSEQ_INJECT_ASM(1) \
69 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
70 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
71 "rldicr %%r17, %%r17, 32, 31\n\t" \
72 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
73 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
74 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
75 __rseq_str(label) ":\n\t"
76
77
78
79
80
81
82
83
84
85 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
86 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
87 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
88 ".popsection\n\t"
89
90 #else
91
92 #define STORE_WORD "stw "
93 #define LOAD_WORD "lwz "
94 #define LOADX_WORD "lwzx "
95 #define CMP_WORD "cmpw "
96
97 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
98 start_ip, post_commit_offset, abort_ip) \
99 ".pushsection __rseq_cs, \"aw\"\n\t" \
100 ".balign 32\n\t" \
101 __rseq_str(label) ":\n\t" \
102 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
103 \
104 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
105 ".popsection\n\t" \
106 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
107 ".long 0x0, " __rseq_str(label) "b\n\t" \
108 ".popsection\n\t"
109
110
111
112
113
114
115
116
117
118 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
119 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
120 \
121 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
122 ".popsection\n\t"
123
124 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
125 RSEQ_INJECT_ASM(1) \
126 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
127 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
128 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
129 __rseq_str(label) ":\n\t"
130
131 #endif
132
133 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
134 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
135 (post_commit_ip - start_ip), abort_ip)
136
137 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
138 RSEQ_INJECT_ASM(2) \
139 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
140 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
141 "bne- cr7, " __rseq_str(label) "\n\t"
142
143 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
144 ".pushsection __rseq_failure, \"ax\"\n\t" \
145 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
146 __rseq_str(label) ":\n\t" \
147 "b %l[" __rseq_str(abort_label) "]\n\t" \
148 ".popsection\n\t"
149
150
151
152
153
154
155 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
156 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
157 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
158 "bne- cr7, " __rseq_str(label) "\n\t"
159
160 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
161 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
162 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
163 "beq- cr7, " __rseq_str(label) "\n\t"
164
165 #define RSEQ_ASM_OP_STORE(value, var) \
166 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
167
168
169 #define RSEQ_ASM_OP_R_LOAD(var) \
170 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
171
172
173 #define RSEQ_ASM_OP_R_STORE(var) \
174 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
175
176
177 #define RSEQ_ASM_OP_R_ADD(count) \
178 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
179
180
181 #define RSEQ_ASM_OP_R_LOADX(voffp) \
182 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
183
184
185 #define RSEQ_ASM_OP_R_MEMCPY() \
186 "cmpdi %%r19, 0\n\t" \
187 "beq 333f\n\t" \
188 "addi %%r20, %%r20, -1\n\t" \
189 "addi %%r21, %%r21, -1\n\t" \
190 "222:\n\t" \
191 "lbzu %%r18, 1(%%r20)\n\t" \
192 "stbu %%r18, 1(%%r21)\n\t" \
193 "addi %%r19, %%r19, -1\n\t" \
194 "cmpdi %%r19, 0\n\t" \
195 "bne 222b\n\t" \
196 "333:\n\t" \
197
198 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
199 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
200 __rseq_str(post_commit_label) ":\n\t"
201
202 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
203 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
204 __rseq_str(post_commit_label) ":\n\t"
205
206 static inline __attribute__((always_inline))
207 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
208 {
209 RSEQ_INJECT_C(9)
210
211 __asm__ __volatile__ goto (
212 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
213 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
214 #ifdef RSEQ_COMPARE_TWICE
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
216 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
217 #endif
218
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
220
221 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
222 RSEQ_INJECT_ASM(3)
223
224 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
225 RSEQ_INJECT_ASM(4)
226 #ifdef RSEQ_COMPARE_TWICE
227
228 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
229
230 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
231 #endif
232
233 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
234 RSEQ_INJECT_ASM(5)
235 RSEQ_ASM_DEFINE_ABORT(4, abort)
236 :
237 : [cpu_id] "r" (cpu),
238 [current_cpu_id] "m" (__rseq_abi.cpu_id),
239 [rseq_cs] "m" (__rseq_abi.rseq_cs),
240 [v] "m" (*v),
241 [expect] "r" (expect),
242 [newv] "r" (newv)
243 RSEQ_INJECT_INPUT
244 : "memory", "cc", "r17"
245 RSEQ_INJECT_CLOBBER
246 : abort, cmpfail
247 #ifdef RSEQ_COMPARE_TWICE
248 , error1, error2
249 #endif
250 );
251 return 0;
252 abort:
253 RSEQ_INJECT_FAILED
254 return -1;
255 cmpfail:
256 return 1;
257 #ifdef RSEQ_COMPARE_TWICE
258 error1:
259 rseq_bug("cpu_id comparison failed");
260 error2:
261 rseq_bug("expected value comparison failed");
262 #endif
263 }
264
265 static inline __attribute__((always_inline))
266 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
267 off_t voffp, intptr_t *load, int cpu)
268 {
269 RSEQ_INJECT_C(9)
270
271 __asm__ __volatile__ goto (
272 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
274 #ifdef RSEQ_COMPARE_TWICE
275 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
276 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
277 #endif
278
279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
280
281 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
282 RSEQ_INJECT_ASM(3)
283
284 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
285 RSEQ_INJECT_ASM(4)
286 #ifdef RSEQ_COMPARE_TWICE
287
288 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
289
290 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
291 #endif
292
293 RSEQ_ASM_OP_R_LOAD(v)
294
295 RSEQ_ASM_OP_R_STORE(load)
296
297 RSEQ_ASM_OP_R_LOADX(voffp)
298
299 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
300 RSEQ_INJECT_ASM(5)
301 RSEQ_ASM_DEFINE_ABORT(4, abort)
302 :
303 : [cpu_id] "r" (cpu),
304 [current_cpu_id] "m" (__rseq_abi.cpu_id),
305 [rseq_cs] "m" (__rseq_abi.rseq_cs),
306
307 [v] "m" (*v),
308 [expectnot] "r" (expectnot),
309 [voffp] "b" (voffp),
310 [load] "m" (*load)
311 RSEQ_INJECT_INPUT
312 : "memory", "cc", "r17"
313 RSEQ_INJECT_CLOBBER
314 : abort, cmpfail
315 #ifdef RSEQ_COMPARE_TWICE
316 , error1, error2
317 #endif
318 );
319 return 0;
320 abort:
321 RSEQ_INJECT_FAILED
322 return -1;
323 cmpfail:
324 return 1;
325 #ifdef RSEQ_COMPARE_TWICE
326 error1:
327 rseq_bug("cpu_id comparison failed");
328 error2:
329 rseq_bug("expected value comparison failed");
330 #endif
331 }
332
333 static inline __attribute__((always_inline))
334 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
335 {
336 RSEQ_INJECT_C(9)
337
338 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
340 #ifdef RSEQ_COMPARE_TWICE
341 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
342 #endif
343
344 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
345
346 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
347 RSEQ_INJECT_ASM(3)
348 #ifdef RSEQ_COMPARE_TWICE
349
350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
351 #endif
352
353 RSEQ_ASM_OP_R_LOAD(v)
354
355 RSEQ_ASM_OP_R_ADD(count)
356
357 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
358 RSEQ_INJECT_ASM(4)
359 RSEQ_ASM_DEFINE_ABORT(4, abort)
360 :
361 : [cpu_id] "r" (cpu),
362 [current_cpu_id] "m" (__rseq_abi.cpu_id),
363 [rseq_cs] "m" (__rseq_abi.rseq_cs),
364
365 [v] "m" (*v),
366 [count] "r" (count)
367 RSEQ_INJECT_INPUT
368 : "memory", "cc", "r17"
369 RSEQ_INJECT_CLOBBER
370 : abort
371 #ifdef RSEQ_COMPARE_TWICE
372 , error1
373 #endif
374 );
375 return 0;
376 abort:
377 RSEQ_INJECT_FAILED
378 return -1;
379 #ifdef RSEQ_COMPARE_TWICE
380 error1:
381 rseq_bug("cpu_id comparison failed");
382 #endif
383 }
384
385 static inline __attribute__((always_inline))
386 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
387 intptr_t *v2, intptr_t newv2,
388 intptr_t newv, int cpu)
389 {
390 RSEQ_INJECT_C(9)
391
392 __asm__ __volatile__ goto (
393 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
394 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
395 #ifdef RSEQ_COMPARE_TWICE
396 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
397 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
398 #endif
399
400 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
401
402 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
403 RSEQ_INJECT_ASM(3)
404
405 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
406 RSEQ_INJECT_ASM(4)
407 #ifdef RSEQ_COMPARE_TWICE
408
409 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
410
411 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
412 #endif
413
414 RSEQ_ASM_OP_STORE(newv2, v2)
415 RSEQ_INJECT_ASM(5)
416
417 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
418 RSEQ_INJECT_ASM(6)
419 RSEQ_ASM_DEFINE_ABORT(4, abort)
420 :
421 : [cpu_id] "r" (cpu),
422 [current_cpu_id] "m" (__rseq_abi.cpu_id),
423 [rseq_cs] "m" (__rseq_abi.rseq_cs),
424
425 [v2] "m" (*v2),
426 [newv2] "r" (newv2),
427
428 [v] "m" (*v),
429 [expect] "r" (expect),
430 [newv] "r" (newv)
431 RSEQ_INJECT_INPUT
432 : "memory", "cc", "r17"
433 RSEQ_INJECT_CLOBBER
434 : abort, cmpfail
435 #ifdef RSEQ_COMPARE_TWICE
436 , error1, error2
437 #endif
438 );
439 return 0;
440 abort:
441 RSEQ_INJECT_FAILED
442 return -1;
443 cmpfail:
444 return 1;
445 #ifdef RSEQ_COMPARE_TWICE
446 error1:
447 rseq_bug("cpu_id comparison failed");
448 error2:
449 rseq_bug("expected value comparison failed");
450 #endif
451 }
452
453 static inline __attribute__((always_inline))
454 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
455 intptr_t *v2, intptr_t newv2,
456 intptr_t newv, int cpu)
457 {
458 RSEQ_INJECT_C(9)
459
460 __asm__ __volatile__ goto (
461 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
462 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
463 #ifdef RSEQ_COMPARE_TWICE
464 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
465 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
466 #endif
467
468 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
469
470 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
471 RSEQ_INJECT_ASM(3)
472
473 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
474 RSEQ_INJECT_ASM(4)
475 #ifdef RSEQ_COMPARE_TWICE
476
477 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
478
479 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
480 #endif
481
482 RSEQ_ASM_OP_STORE(newv2, v2)
483 RSEQ_INJECT_ASM(5)
484
485 "lwsync\n\t"
486
487 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
488 RSEQ_INJECT_ASM(6)
489 RSEQ_ASM_DEFINE_ABORT(4, abort)
490 :
491 : [cpu_id] "r" (cpu),
492 [current_cpu_id] "m" (__rseq_abi.cpu_id),
493 [rseq_cs] "m" (__rseq_abi.rseq_cs),
494
495 [v2] "m" (*v2),
496 [newv2] "r" (newv2),
497
498 [v] "m" (*v),
499 [expect] "r" (expect),
500 [newv] "r" (newv)
501 RSEQ_INJECT_INPUT
502 : "memory", "cc", "r17"
503 RSEQ_INJECT_CLOBBER
504 : abort, cmpfail
505 #ifdef RSEQ_COMPARE_TWICE
506 , error1, error2
507 #endif
508 );
509 return 0;
510 abort:
511 RSEQ_INJECT_FAILED
512 return -1;
513 cmpfail:
514 return 1;
515 #ifdef RSEQ_COMPARE_TWICE
516 error1:
517 rseq_bug("cpu_id comparison failed");
518 error2:
519 rseq_bug("expected value comparison failed");
520 #endif
521 }
522
523 static inline __attribute__((always_inline))
524 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
525 intptr_t *v2, intptr_t expect2,
526 intptr_t newv, int cpu)
527 {
528 RSEQ_INJECT_C(9)
529
530 __asm__ __volatile__ goto (
531 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
532 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
533 #ifdef RSEQ_COMPARE_TWICE
534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
535 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
537 #endif
538
539 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
540
541 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
542 RSEQ_INJECT_ASM(3)
543
544 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
545 RSEQ_INJECT_ASM(4)
546
547 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
548 RSEQ_INJECT_ASM(5)
549 #ifdef RSEQ_COMPARE_TWICE
550
551 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
552
553 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
554
555 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
556 #endif
557
558 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
559 RSEQ_INJECT_ASM(6)
560 RSEQ_ASM_DEFINE_ABORT(4, abort)
561 :
562 : [cpu_id] "r" (cpu),
563 [current_cpu_id] "m" (__rseq_abi.cpu_id),
564 [rseq_cs] "m" (__rseq_abi.rseq_cs),
565
566 [v2] "m" (*v2),
567 [expect2] "r" (expect2),
568
569 [v] "m" (*v),
570 [expect] "r" (expect),
571 [newv] "r" (newv)
572 RSEQ_INJECT_INPUT
573 : "memory", "cc", "r17"
574 RSEQ_INJECT_CLOBBER
575 : abort, cmpfail
576 #ifdef RSEQ_COMPARE_TWICE
577 , error1, error2, error3
578 #endif
579 );
580 return 0;
581 abort:
582 RSEQ_INJECT_FAILED
583 return -1;
584 cmpfail:
585 return 1;
586 #ifdef RSEQ_COMPARE_TWICE
587 error1:
588 rseq_bug("cpu_id comparison failed");
589 error2:
590 rseq_bug("1st expected value comparison failed");
591 error3:
592 rseq_bug("2nd expected value comparison failed");
593 #endif
594 }
595
596 static inline __attribute__((always_inline))
597 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
598 void *dst, void *src, size_t len,
599 intptr_t newv, int cpu)
600 {
601 RSEQ_INJECT_C(9)
602
603 __asm__ __volatile__ goto (
604 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
605 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
606 #ifdef RSEQ_COMPARE_TWICE
607 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
608 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
609 #endif
610
611 "mr %%r19, %[len]\n\t"
612 "mr %%r20, %[src]\n\t"
613 "mr %%r21, %[dst]\n\t"
614
615 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
616
617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 RSEQ_INJECT_ASM(3)
619
620 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
621 RSEQ_INJECT_ASM(4)
622 #ifdef RSEQ_COMPARE_TWICE
623
624 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
625
626 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
627 #endif
628
629 RSEQ_ASM_OP_R_MEMCPY()
630 RSEQ_INJECT_ASM(5)
631
632 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
633 RSEQ_INJECT_ASM(6)
634
635 RSEQ_ASM_DEFINE_ABORT(4, abort)
636 :
637 : [cpu_id] "r" (cpu),
638 [current_cpu_id] "m" (__rseq_abi.cpu_id),
639 [rseq_cs] "m" (__rseq_abi.rseq_cs),
640
641 [v] "m" (*v),
642 [expect] "r" (expect),
643 [newv] "r" (newv),
644
645 [dst] "r" (dst),
646 [src] "r" (src),
647 [len] "r" (len)
648 RSEQ_INJECT_INPUT
649 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
650 RSEQ_INJECT_CLOBBER
651 : abort, cmpfail
652 #ifdef RSEQ_COMPARE_TWICE
653 , error1, error2
654 #endif
655 );
656 return 0;
657 abort:
658 RSEQ_INJECT_FAILED
659 return -1;
660 cmpfail:
661 return 1;
662 #ifdef RSEQ_COMPARE_TWICE
663 error1:
664 rseq_bug("cpu_id comparison failed");
665 error2:
666 rseq_bug("expected value comparison failed");
667 #endif
668 }
669
670 static inline __attribute__((always_inline))
671 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
672 void *dst, void *src, size_t len,
673 intptr_t newv, int cpu)
674 {
675 RSEQ_INJECT_C(9)
676
677 __asm__ __volatile__ goto (
678 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f)
679 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
680 #ifdef RSEQ_COMPARE_TWICE
681 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
682 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
683 #endif
684
685 "mr %%r19, %[len]\n\t"
686 "mr %%r20, %[src]\n\t"
687 "mr %%r21, %[dst]\n\t"
688
689 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
690
691 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
692 RSEQ_INJECT_ASM(3)
693
694 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
695 RSEQ_INJECT_ASM(4)
696 #ifdef RSEQ_COMPARE_TWICE
697
698 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
699
700 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
701 #endif
702
703 RSEQ_ASM_OP_R_MEMCPY()
704 RSEQ_INJECT_ASM(5)
705
706 "lwsync\n\t"
707
708 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
709 RSEQ_INJECT_ASM(6)
710
711 RSEQ_ASM_DEFINE_ABORT(4, abort)
712 :
713 : [cpu_id] "r" (cpu),
714 [current_cpu_id] "m" (__rseq_abi.cpu_id),
715 [rseq_cs] "m" (__rseq_abi.rseq_cs),
716
717 [v] "m" (*v),
718 [expect] "r" (expect),
719 [newv] "r" (newv),
720
721 [dst] "r" (dst),
722 [src] "r" (src),
723 [len] "r" (len)
724 RSEQ_INJECT_INPUT
725 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
726 RSEQ_INJECT_CLOBBER
727 : abort, cmpfail
728 #ifdef RSEQ_COMPARE_TWICE
729 , error1, error2
730 #endif
731 );
732 return 0;
733 abort:
734 RSEQ_INJECT_FAILED
735 return -1;
736 cmpfail:
737 return 1;
738 #ifdef RSEQ_COMPARE_TWICE
739 error1:
740 rseq_bug("cpu_id comparison failed");
741 error2:
742 rseq_bug("expected value comparison failed");
743 #endif
744 }
745
746 #undef STORE_WORD
747 #undef LOAD_WORD
748 #undef LOADX_WORD
749 #undef CMP_WORD
750
751 #endif