1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4 */
5
6 #ifndef _ASM_ARC_ATOMIC_H
7 #define _ASM_ARC_ATOMIC_H
8
9 #ifndef __ASSEMBLY__
10
11 #include <linux/types.h>
12 #include <linux/compiler.h>
13 #include <asm/cmpxchg.h>
14 #include <asm/barrier.h>
15 #include <asm/smp.h>
16
17 #define ATOMIC_INIT(i) { (i) }
18
19 #ifndef CONFIG_ARC_PLAT_EZNPS
20
21 #define atomic_read(v) READ_ONCE((v)->counter)
22
23 #ifdef CONFIG_ARC_HAS_LLSC
24
25 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
26
27 #define ATOMIC_OP(op, c_op, asm_op) \
28 static inline void atomic_##op(int i, atomic_t *v) \
29 { \
30 unsigned int val; \
31 \
32 __asm__ __volatile__( \
33 "1: llock %[val], [%[ctr]] \n" \
34 " " #asm_op " %[val], %[val], %[i] \n" \
35 " scond %[val], [%[ctr]] \n" \
36 " bnz 1b \n" \
37 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \
38 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \
39 [i] "ir" (i) \
40 : "cc"); \
41 } \
42
43 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
44 static inline int atomic_##op##_return(int i, atomic_t *v) \
45 { \
46 unsigned int val; \
47 \
48 /* \
49 * Explicit full memory barrier needed before/after as \
50 * LLOCK/SCOND thmeselves don't provide any such semantics \
51 */ \
52 smp_mb(); \
53 \
54 __asm__ __volatile__( \
55 "1: llock %[val], [%[ctr]] \n" \
56 " " #asm_op " %[val], %[val], %[i] \n" \
57 " scond %[val], [%[ctr]] \n" \
58 " bnz 1b \n" \
59 : [val] "=&r" (val) \
60 : [ctr] "r" (&v->counter), \
61 [i] "ir" (i) \
62 : "cc"); \
63 \
64 smp_mb(); \
65 \
66 return val; \
67 }
68
69 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
70 static inline int atomic_fetch_##op(int i, atomic_t *v) \
71 { \
72 unsigned int val, orig; \
73 \
74 /* \
75 * Explicit full memory barrier needed before/after as \
76 * LLOCK/SCOND thmeselves don't provide any such semantics \
77 */ \
78 smp_mb(); \
79 \
80 __asm__ __volatile__( \
81 "1: llock %[orig], [%[ctr]] \n" \
82 " " #asm_op " %[val], %[orig], %[i] \n" \
83 " scond %[val], [%[ctr]] \n" \
84 " bnz 1b \n" \
85 : [val] "=&r" (val), \
86 [orig] "=&r" (orig) \
87 : [ctr] "r" (&v->counter), \
88 [i] "ir" (i) \
89 : "cc"); \
90 \
91 smp_mb(); \
92 \
93 return orig; \
94 }
95
96 #else /* !CONFIG_ARC_HAS_LLSC */
97
98 #ifndef CONFIG_SMP
99
100 /* violating atomic_xxx API locking protocol in UP for optimization sake */
101 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
102
103 #else
104
105 static inline void atomic_set(atomic_t *v, int i)
106 {
107 /*
108 * Independent of hardware support, all of the atomic_xxx() APIs need
109 * to follow the same locking rules to make sure that a "hardware"
110 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
111 * sequence
112 *
113 * Thus atomic_set() despite being 1 insn (and seemingly atomic)
114 * requires the locking.
115 */
116 unsigned long flags;
117
118 atomic_ops_lock(flags);
119 WRITE_ONCE(v->counter, i);
120 atomic_ops_unlock(flags);
121 }
122
123 #define atomic_set_release(v, i) atomic_set((v), (i))
124
125 #endif
126
127 /*
128 * Non hardware assisted Atomic-R-M-W
129 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
130 */
131
132 #define ATOMIC_OP(op, c_op, asm_op) \
133 static inline void atomic_##op(int i, atomic_t *v) \
134 { \
135 unsigned long flags; \
136 \
137 atomic_ops_lock(flags); \
138 v->counter c_op i; \
139 atomic_ops_unlock(flags); \
140 }
141
142 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
143 static inline int atomic_##op##_return(int i, atomic_t *v) \
144 { \
145 unsigned long flags; \
146 unsigned long temp; \
147 \
148 /* \
149 * spin lock/unlock provides the needed smp_mb() before/after \
150 */ \
151 atomic_ops_lock(flags); \
152 temp = v->counter; \
153 temp c_op i; \
154 v->counter = temp; \
155 atomic_ops_unlock(flags); \
156 \
157 return temp; \
158 }
159
160 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
161 static inline int atomic_fetch_##op(int i, atomic_t *v) \
162 { \
163 unsigned long flags; \
164 unsigned long orig; \
165 \
166 /* \
167 * spin lock/unlock provides the needed smp_mb() before/after \
168 */ \
169 atomic_ops_lock(flags); \
170 orig = v->counter; \
171 v->counter c_op i; \
172 atomic_ops_unlock(flags); \
173 \
174 return orig; \
175 }
176
177 #endif /* !CONFIG_ARC_HAS_LLSC */
178
179 #define ATOMIC_OPS(op, c_op, asm_op) \
180 ATOMIC_OP(op, c_op, asm_op) \
181 ATOMIC_OP_RETURN(op, c_op, asm_op) \
182 ATOMIC_FETCH_OP(op, c_op, asm_op)
183
184 ATOMIC_OPS(add, +=, add)
185 ATOMIC_OPS(sub, -=, sub)
186
187 #define atomic_andnot atomic_andnot
188 #define atomic_fetch_andnot atomic_fetch_andnot
189
190 #undef ATOMIC_OPS
191 #define ATOMIC_OPS(op, c_op, asm_op) \
192 ATOMIC_OP(op, c_op, asm_op) \
193 ATOMIC_FETCH_OP(op, c_op, asm_op)
194
195 ATOMIC_OPS(and, &=, and)
196 ATOMIC_OPS(andnot, &= ~, bic)
197 ATOMIC_OPS(or, |=, or)
198 ATOMIC_OPS(xor, ^=, xor)
199
200 #else /* CONFIG_ARC_PLAT_EZNPS */
201
202 static inline int atomic_read(const atomic_t *v)
203 {
204 int temp;
205
206 __asm__ __volatile__(
207 " ld.di %0, [%1]"
208 : "=r"(temp)
209 : "r"(&v->counter)
210 : "memory");
211 return temp;
212 }
213
214 static inline void atomic_set(atomic_t *v, int i)
215 {
216 __asm__ __volatile__(
217 " st.di %0,[%1]"
218 :
219 : "r"(i), "r"(&v->counter)
220 : "memory");
221 }
222
223 #define ATOMIC_OP(op, c_op, asm_op) \
224 static inline void atomic_##op(int i, atomic_t *v) \
225 { \
226 __asm__ __volatile__( \
227 " mov r2, %0\n" \
228 " mov r3, %1\n" \
229 " .word %2\n" \
230 : \
231 : "r"(i), "r"(&v->counter), "i"(asm_op) \
232 : "r2", "r3", "memory"); \
233 } \
234
235 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
236 static inline int atomic_##op##_return(int i, atomic_t *v) \
237 { \
238 unsigned int temp = i; \
239 \
240 /* Explicit full memory barrier needed before/after */ \
241 smp_mb(); \
242 \
243 __asm__ __volatile__( \
244 " mov r2, %0\n" \
245 " mov r3, %1\n" \
246 " .word %2\n" \
247 " mov %0, r2" \
248 : "+r"(temp) \
249 : "r"(&v->counter), "i"(asm_op) \
250 : "r2", "r3", "memory"); \
251 \
252 smp_mb(); \
253 \
254 temp c_op i; \
255 \
256 return temp; \
257 }
258
259 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
260 static inline int atomic_fetch_##op(int i, atomic_t *v) \
261 { \
262 unsigned int temp = i; \
263 \
264 /* Explicit full memory barrier needed before/after */ \
265 smp_mb(); \
266 \
267 __asm__ __volatile__( \
268 " mov r2, %0\n" \
269 " mov r3, %1\n" \
270 " .word %2\n" \
271 " mov %0, r2" \
272 : "+r"(temp) \
273 : "r"(&v->counter), "i"(asm_op) \
274 : "r2", "r3", "memory"); \
275 \
276 smp_mb(); \
277 \
278 return temp; \
279 }
280
281 #define ATOMIC_OPS(op, c_op, asm_op) \
282 ATOMIC_OP(op, c_op, asm_op) \
283 ATOMIC_OP_RETURN(op, c_op, asm_op) \
284 ATOMIC_FETCH_OP(op, c_op, asm_op)
285
286 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
287 #define atomic_sub(i, v) atomic_add(-(i), (v))
288 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
289 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
290
291 #undef ATOMIC_OPS
292 #define ATOMIC_OPS(op, c_op, asm_op) \
293 ATOMIC_OP(op, c_op, asm_op) \
294 ATOMIC_FETCH_OP(op, c_op, asm_op)
295
296 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
297 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
298 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
299
300 #endif /* CONFIG_ARC_PLAT_EZNPS */
301
302 #undef ATOMIC_OPS
303 #undef ATOMIC_FETCH_OP
304 #undef ATOMIC_OP_RETURN
305 #undef ATOMIC_OP
306
307 #ifdef CONFIG_GENERIC_ATOMIC64
308
309 #include <asm-generic/atomic64.h>
310
311 #else /* Kconfig ensures this is only enabled with needed h/w assist */
312
313 /*
314 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
315 * - The address HAS to be 64-bit aligned
316 * - There are 2 semantics involved here:
317 * = exclusive implies no interim update between load/store to same addr
318 * = both words are observed/updated together: this is guaranteed even
319 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
320 * is NOT required to use LLOCKD+SCONDD, STD suffices
321 */
322
323 typedef struct {
324 s64 __aligned(8) counter;
325 } atomic64_t;
326
327 #define ATOMIC64_INIT(a) { (a) }
328
329 static inline s64 atomic64_read(const atomic64_t *v)
330 {
331 s64 val;
332
333 __asm__ __volatile__(
334 " ldd %0, [%1] \n"
335 : "=r"(val)
336 : "r"(&v->counter));
337
338 return val;
339 }
340
341 static inline void atomic64_set(atomic64_t *v, s64 a)
342 {
343 /*
344 * This could have been a simple assignment in "C" but would need
345 * explicit volatile. Otherwise gcc optimizers could elide the store
346 * which borked atomic64 self-test
347 * In the inline asm version, memory clobber needed for exact same
348 * reason, to tell gcc about the store.
349 *
350 * This however is not needed for sibling atomic64_add() etc since both
351 * load/store are explicitly done in inline asm. As long as API is used
352 * for each access, gcc has no way to optimize away any load/store
353 */
354 __asm__ __volatile__(
355 " std %0, [%1] \n"
356 :
357 : "r"(a), "r"(&v->counter)
358 : "memory");
359 }
360
361 #define ATOMIC64_OP(op, op1, op2) \
362 static inline void atomic64_##op(s64 a, atomic64_t *v) \
363 { \
364 s64 val; \
365 \
366 __asm__ __volatile__( \
367 "1: \n" \
368 " llockd %0, [%1] \n" \
369 " " #op1 " %L0, %L0, %L2 \n" \
370 " " #op2 " %H0, %H0, %H2 \n" \
371 " scondd %0, [%1] \n" \
372 " bnz 1b \n" \
373 : "=&r"(val) \
374 : "r"(&v->counter), "ir"(a) \
375 : "cc"); \
376 } \
377
378 #define ATOMIC64_OP_RETURN(op, op1, op2) \
379 static inline s64 atomic64_##op##_return(s64 a, atomic64_t *v) \
380 { \
381 s64 val; \
382 \
383 smp_mb(); \
384 \
385 __asm__ __volatile__( \
386 "1: \n" \
387 " llockd %0, [%1] \n" \
388 " " #op1 " %L0, %L0, %L2 \n" \
389 " " #op2 " %H0, %H0, %H2 \n" \
390 " scondd %0, [%1] \n" \
391 " bnz 1b \n" \
392 : [val] "=&r"(val) \
393 : "r"(&v->counter), "ir"(a) \
394 : "cc"); /* memory clobber comes from smp_mb() */ \
395 \
396 smp_mb(); \
397 \
398 return val; \
399 }
400
401 #define ATOMIC64_FETCH_OP(op, op1, op2) \
402 static inline s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \
403 { \
404 s64 val, orig; \
405 \
406 smp_mb(); \
407 \
408 __asm__ __volatile__( \
409 "1: \n" \
410 " llockd %0, [%2] \n" \
411 " " #op1 " %L1, %L0, %L3 \n" \
412 " " #op2 " %H1, %H0, %H3 \n" \
413 " scondd %1, [%2] \n" \
414 " bnz 1b \n" \
415 : "=&r"(orig), "=&r"(val) \
416 : "r"(&v->counter), "ir"(a) \
417 : "cc"); /* memory clobber comes from smp_mb() */ \
418 \
419 smp_mb(); \
420 \
421 return orig; \
422 }
423
424 #define ATOMIC64_OPS(op, op1, op2) \
425 ATOMIC64_OP(op, op1, op2) \
426 ATOMIC64_OP_RETURN(op, op1, op2) \
427 ATOMIC64_FETCH_OP(op, op1, op2)
428
429 #define atomic64_andnot atomic64_andnot
430 #define atomic64_fetch_andnot atomic64_fetch_andnot
431
432 ATOMIC64_OPS(add, add.f, adc)
433 ATOMIC64_OPS(sub, sub.f, sbc)
434 ATOMIC64_OPS(and, and, and)
435 ATOMIC64_OPS(andnot, bic, bic)
436 ATOMIC64_OPS(or, or, or)
437 ATOMIC64_OPS(xor, xor, xor)
438
439 #undef ATOMIC64_OPS
440 #undef ATOMIC64_FETCH_OP
441 #undef ATOMIC64_OP_RETURN
442 #undef ATOMIC64_OP
443
444 static inline s64
445 atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)
446 {
447 s64 prev;
448
449 smp_mb();
450
451 __asm__ __volatile__(
452 "1: llockd %0, [%1] \n"
453 " brne %L0, %L2, 2f \n"
454 " brne %H0, %H2, 2f \n"
455 " scondd %3, [%1] \n"
456 " bnz 1b \n"
457 "2: \n"
458 : "=&r"(prev)
459 : "r"(ptr), "ir"(expected), "r"(new)
460 : "cc"); /* memory clobber comes from smp_mb() */
461
462 smp_mb();
463
464 return prev;
465 }
466
467 static inline s64 atomic64_xchg(atomic64_t *ptr, s64 new)
468 {
469 s64 prev;
470
471 smp_mb();
472
473 __asm__ __volatile__(
474 "1: llockd %0, [%1] \n"
475 " scondd %2, [%1] \n"
476 " bnz 1b \n"
477 "2: \n"
478 : "=&r"(prev)
479 : "r"(ptr), "r"(new)
480 : "cc"); /* memory clobber comes from smp_mb() */
481
482 smp_mb();
483
484 return prev;
485 }
486
487 /**
488 * atomic64_dec_if_positive - decrement by 1 if old value positive
489 * @v: pointer of type atomic64_t
490 *
491 * The function returns the old value of *v minus 1, even if
492 * the atomic variable, v, was not decremented.
493 */
494
495 static inline s64 atomic64_dec_if_positive(atomic64_t *v)
496 {
497 s64 val;
498
499 smp_mb();
500
501 __asm__ __volatile__(
502 "1: llockd %0, [%1] \n"
503 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n"
504 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n"
505 " brlt %H0, 0, 2f \n"
506 " scondd %0, [%1] \n"
507 " bnz 1b \n"
508 "2: \n"
509 : "=&r"(val)
510 : "r"(&v->counter)
511 : "cc"); /* memory clobber comes from smp_mb() */
512
513 smp_mb();
514
515 return val;
516 }
517 #define atomic64_dec_if_positive atomic64_dec_if_positive
518
519 /**
520 * atomic64_fetch_add_unless - add unless the number is a given value
521 * @v: pointer of type atomic64_t
522 * @a: the amount to add to v...
523 * @u: ...unless v is equal to u.
524 *
525 * Atomically adds @a to @v, if it was not @u.
526 * Returns the old value of @v
527 */
528 static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
529 {
530 s64 old, temp;
531
532 smp_mb();
533
534 __asm__ __volatile__(
535 "1: llockd %0, [%2] \n"
536 " brne %L0, %L4, 2f # continue to add since v != u \n"
537 " breq.d %H0, %H4, 3f # return since v == u \n"
538 "2: \n"
539 " add.f %L1, %L0, %L3 \n"
540 " adc %H1, %H0, %H3 \n"
541 " scondd %1, [%2] \n"
542 " bnz 1b \n"
543 "3: \n"
544 : "=&r"(old), "=&r" (temp)
545 : "r"(&v->counter), "r"(a), "r"(u)
546 : "cc"); /* memory clobber comes from smp_mb() */
547
548 smp_mb();
549
550 return old;
551 }
552 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
553
554 #endif /* !CONFIG_GENERIC_ATOMIC64 */
555
556 #endif /* !__ASSEMBLY__ */
557
558 #endif