This source file includes following definitions.
- __xchg_u32_local
- __xchg_u32_relaxed
- __xchg_u64_local
- __xchg_u64_relaxed
- __xchg_local
- __xchg_relaxed
- __cmpxchg_u32
- __cmpxchg_u32_local
- __cmpxchg_u32_relaxed
- __cmpxchg_u32_acquire
- __cmpxchg_u64
- __cmpxchg_u64_local
- __cmpxchg_u64_relaxed
- __cmpxchg_u64_acquire
- __cmpxchg
- __cmpxchg_local
- __cmpxchg_relaxed
- __cmpxchg_acquire
1
2 #ifndef _ASM_POWERPC_CMPXCHG_H_
3 #define _ASM_POWERPC_CMPXCHG_H_
4
5 #ifdef __KERNEL__
6 #include <linux/compiler.h>
7 #include <asm/synch.h>
8 #include <linux/bug.h>
9 #include <asm/asm-405.h>
10
11 #ifdef __BIG_ENDIAN
12 #define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
13 #else
14 #define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
15 #endif
16
17 #define XCHG_GEN(type, sfx, cl) \
18 static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
19 { \
20 unsigned int prev, prev_mask, tmp, bitoff, off; \
21 \
22 off = (unsigned long)p % sizeof(u32); \
23 bitoff = BITOFF_CAL(sizeof(type), off); \
24 p -= off; \
25 val <<= bitoff; \
26 prev_mask = (u32)(type)-1 << bitoff; \
27 \
28 __asm__ __volatile__( \
29 "1: lwarx %0,0,%3\n" \
30 " andc %1,%0,%5\n" \
31 " or %1,%1,%4\n" \
32 PPC405_ERR77(0,%3) \
33 " stwcx. %1,0,%3\n" \
34 " bne- 1b\n" \
35 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
36 : "r" (p), "r" (val), "r" (prev_mask) \
37 : "cc", cl); \
38 \
39 return prev >> bitoff; \
40 }
41
42 #define CMPXCHG_GEN(type, sfx, br, br2, cl) \
43 static inline \
44 u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
45 { \
46 unsigned int prev, prev_mask, tmp, bitoff, off; \
47 \
48 off = (unsigned long)p % sizeof(u32); \
49 bitoff = BITOFF_CAL(sizeof(type), off); \
50 p -= off; \
51 old <<= bitoff; \
52 new <<= bitoff; \
53 prev_mask = (u32)(type)-1 << bitoff; \
54 \
55 __asm__ __volatile__( \
56 br \
57 "1: lwarx %0,0,%3\n" \
58 " and %1,%0,%6\n" \
59 " cmpw 0,%1,%4\n" \
60 " bne- 2f\n" \
61 " andc %1,%0,%6\n" \
62 " or %1,%1,%5\n" \
63 PPC405_ERR77(0,%3) \
64 " stwcx. %1,0,%3\n" \
65 " bne- 1b\n" \
66 br2 \
67 "\n" \
68 "2:" \
69 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
70 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
71 : "cc", cl); \
72 \
73 return prev >> bitoff; \
74 }
75
76
77
78
79
80
81
82
83 XCHG_GEN(u8, _local, "memory");
84 XCHG_GEN(u8, _relaxed, "cc");
85 XCHG_GEN(u16, _local, "memory");
86 XCHG_GEN(u16, _relaxed, "cc");
87
88 static __always_inline unsigned long
89 __xchg_u32_local(volatile void *p, unsigned long val)
90 {
91 unsigned long prev;
92
93 __asm__ __volatile__(
94 "1: lwarx %0,0,%2 \n"
95 PPC405_ERR77(0,%2)
96 " stwcx. %3,0,%2 \n\
97 bne- 1b"
98 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
99 : "r" (p), "r" (val)
100 : "cc", "memory");
101
102 return prev;
103 }
104
105 static __always_inline unsigned long
106 __xchg_u32_relaxed(u32 *p, unsigned long val)
107 {
108 unsigned long prev;
109
110 __asm__ __volatile__(
111 "1: lwarx %0,0,%2\n"
112 PPC405_ERR77(0, %2)
113 " stwcx. %3,0,%2\n"
114 " bne- 1b"
115 : "=&r" (prev), "+m" (*p)
116 : "r" (p), "r" (val)
117 : "cc");
118
119 return prev;
120 }
121
122 #ifdef CONFIG_PPC64
123 static __always_inline unsigned long
124 __xchg_u64_local(volatile void *p, unsigned long val)
125 {
126 unsigned long prev;
127
128 __asm__ __volatile__(
129 "1: ldarx %0,0,%2 \n"
130 PPC405_ERR77(0,%2)
131 " stdcx. %3,0,%2 \n\
132 bne- 1b"
133 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
134 : "r" (p), "r" (val)
135 : "cc", "memory");
136
137 return prev;
138 }
139
140 static __always_inline unsigned long
141 __xchg_u64_relaxed(u64 *p, unsigned long val)
142 {
143 unsigned long prev;
144
145 __asm__ __volatile__(
146 "1: ldarx %0,0,%2\n"
147 PPC405_ERR77(0, %2)
148 " stdcx. %3,0,%2\n"
149 " bne- 1b"
150 : "=&r" (prev), "+m" (*p)
151 : "r" (p), "r" (val)
152 : "cc");
153
154 return prev;
155 }
156 #endif
157
158 static __always_inline unsigned long
159 __xchg_local(void *ptr, unsigned long x, unsigned int size)
160 {
161 switch (size) {
162 case 1:
163 return __xchg_u8_local(ptr, x);
164 case 2:
165 return __xchg_u16_local(ptr, x);
166 case 4:
167 return __xchg_u32_local(ptr, x);
168 #ifdef CONFIG_PPC64
169 case 8:
170 return __xchg_u64_local(ptr, x);
171 #endif
172 }
173 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
174 return x;
175 }
176
177 static __always_inline unsigned long
178 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
179 {
180 switch (size) {
181 case 1:
182 return __xchg_u8_relaxed(ptr, x);
183 case 2:
184 return __xchg_u16_relaxed(ptr, x);
185 case 4:
186 return __xchg_u32_relaxed(ptr, x);
187 #ifdef CONFIG_PPC64
188 case 8:
189 return __xchg_u64_relaxed(ptr, x);
190 #endif
191 }
192 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
193 return x;
194 }
195 #define xchg_local(ptr,x) \
196 ({ \
197 __typeof__(*(ptr)) _x_ = (x); \
198 (__typeof__(*(ptr))) __xchg_local((ptr), \
199 (unsigned long)_x_, sizeof(*(ptr))); \
200 })
201
202 #define xchg_relaxed(ptr, x) \
203 ({ \
204 __typeof__(*(ptr)) _x_ = (x); \
205 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
206 (unsigned long)_x_, sizeof(*(ptr))); \
207 })
208
209
210
211
212
213 CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
214 CMPXCHG_GEN(u8, _local, , , "memory");
215 CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
216 CMPXCHG_GEN(u8, _relaxed, , , "cc");
217 CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
218 CMPXCHG_GEN(u16, _local, , , "memory");
219 CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
220 CMPXCHG_GEN(u16, _relaxed, , , "cc");
221
222 static __always_inline unsigned long
223 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
224 {
225 unsigned int prev;
226
227 __asm__ __volatile__ (
228 PPC_ATOMIC_ENTRY_BARRIER
229 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
230 cmpw 0,%0,%3\n\
231 bne- 2f\n"
232 PPC405_ERR77(0,%2)
233 " stwcx. %4,0,%2\n\
234 bne- 1b"
235 PPC_ATOMIC_EXIT_BARRIER
236 "\n\
237 2:"
238 : "=&r" (prev), "+m" (*p)
239 : "r" (p), "r" (old), "r" (new)
240 : "cc", "memory");
241
242 return prev;
243 }
244
245 static __always_inline unsigned long
246 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
247 unsigned long new)
248 {
249 unsigned int prev;
250
251 __asm__ __volatile__ (
252 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
253 cmpw 0,%0,%3\n\
254 bne- 2f\n"
255 PPC405_ERR77(0,%2)
256 " stwcx. %4,0,%2\n\
257 bne- 1b"
258 "\n\
259 2:"
260 : "=&r" (prev), "+m" (*p)
261 : "r" (p), "r" (old), "r" (new)
262 : "cc", "memory");
263
264 return prev;
265 }
266
267 static __always_inline unsigned long
268 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
269 {
270 unsigned long prev;
271
272 __asm__ __volatile__ (
273 "1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
274 " cmpw 0,%0,%3\n"
275 " bne- 2f\n"
276 PPC405_ERR77(0, %2)
277 " stwcx. %4,0,%2\n"
278 " bne- 1b\n"
279 "2:"
280 : "=&r" (prev), "+m" (*p)
281 : "r" (p), "r" (old), "r" (new)
282 : "cc");
283
284 return prev;
285 }
286
287
288
289
290
291
292
293
294
295 static __always_inline unsigned long
296 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
297 {
298 unsigned long prev;
299
300 __asm__ __volatile__ (
301 "1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
302 " cmpw 0,%0,%3\n"
303 " bne- 2f\n"
304 PPC405_ERR77(0, %2)
305 " stwcx. %4,0,%2\n"
306 " bne- 1b\n"
307 PPC_ACQUIRE_BARRIER
308 "\n"
309 "2:"
310 : "=&r" (prev), "+m" (*p)
311 : "r" (p), "r" (old), "r" (new)
312 : "cc", "memory");
313
314 return prev;
315 }
316
317 #ifdef CONFIG_PPC64
318 static __always_inline unsigned long
319 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
320 {
321 unsigned long prev;
322
323 __asm__ __volatile__ (
324 PPC_ATOMIC_ENTRY_BARRIER
325 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
326 cmpd 0,%0,%3\n\
327 bne- 2f\n\
328 stdcx. %4,0,%2\n\
329 bne- 1b"
330 PPC_ATOMIC_EXIT_BARRIER
331 "\n\
332 2:"
333 : "=&r" (prev), "+m" (*p)
334 : "r" (p), "r" (old), "r" (new)
335 : "cc", "memory");
336
337 return prev;
338 }
339
340 static __always_inline unsigned long
341 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
342 unsigned long new)
343 {
344 unsigned long prev;
345
346 __asm__ __volatile__ (
347 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
348 cmpd 0,%0,%3\n\
349 bne- 2f\n\
350 stdcx. %4,0,%2\n\
351 bne- 1b"
352 "\n\
353 2:"
354 : "=&r" (prev), "+m" (*p)
355 : "r" (p), "r" (old), "r" (new)
356 : "cc", "memory");
357
358 return prev;
359 }
360
361 static __always_inline unsigned long
362 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
363 {
364 unsigned long prev;
365
366 __asm__ __volatile__ (
367 "1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
368 " cmpd 0,%0,%3\n"
369 " bne- 2f\n"
370 " stdcx. %4,0,%2\n"
371 " bne- 1b\n"
372 "2:"
373 : "=&r" (prev), "+m" (*p)
374 : "r" (p), "r" (old), "r" (new)
375 : "cc");
376
377 return prev;
378 }
379
380 static __always_inline unsigned long
381 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
382 {
383 unsigned long prev;
384
385 __asm__ __volatile__ (
386 "1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
387 " cmpd 0,%0,%3\n"
388 " bne- 2f\n"
389 " stdcx. %4,0,%2\n"
390 " bne- 1b\n"
391 PPC_ACQUIRE_BARRIER
392 "\n"
393 "2:"
394 : "=&r" (prev), "+m" (*p)
395 : "r" (p), "r" (old), "r" (new)
396 : "cc", "memory");
397
398 return prev;
399 }
400 #endif
401
402 static __always_inline unsigned long
403 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
404 unsigned int size)
405 {
406 switch (size) {
407 case 1:
408 return __cmpxchg_u8(ptr, old, new);
409 case 2:
410 return __cmpxchg_u16(ptr, old, new);
411 case 4:
412 return __cmpxchg_u32(ptr, old, new);
413 #ifdef CONFIG_PPC64
414 case 8:
415 return __cmpxchg_u64(ptr, old, new);
416 #endif
417 }
418 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
419 return old;
420 }
421
422 static __always_inline unsigned long
423 __cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
424 unsigned int size)
425 {
426 switch (size) {
427 case 1:
428 return __cmpxchg_u8_local(ptr, old, new);
429 case 2:
430 return __cmpxchg_u16_local(ptr, old, new);
431 case 4:
432 return __cmpxchg_u32_local(ptr, old, new);
433 #ifdef CONFIG_PPC64
434 case 8:
435 return __cmpxchg_u64_local(ptr, old, new);
436 #endif
437 }
438 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
439 return old;
440 }
441
442 static __always_inline unsigned long
443 __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
444 unsigned int size)
445 {
446 switch (size) {
447 case 1:
448 return __cmpxchg_u8_relaxed(ptr, old, new);
449 case 2:
450 return __cmpxchg_u16_relaxed(ptr, old, new);
451 case 4:
452 return __cmpxchg_u32_relaxed(ptr, old, new);
453 #ifdef CONFIG_PPC64
454 case 8:
455 return __cmpxchg_u64_relaxed(ptr, old, new);
456 #endif
457 }
458 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
459 return old;
460 }
461
462 static __always_inline unsigned long
463 __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
464 unsigned int size)
465 {
466 switch (size) {
467 case 1:
468 return __cmpxchg_u8_acquire(ptr, old, new);
469 case 2:
470 return __cmpxchg_u16_acquire(ptr, old, new);
471 case 4:
472 return __cmpxchg_u32_acquire(ptr, old, new);
473 #ifdef CONFIG_PPC64
474 case 8:
475 return __cmpxchg_u64_acquire(ptr, old, new);
476 #endif
477 }
478 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
479 return old;
480 }
481 #define cmpxchg(ptr, o, n) \
482 ({ \
483 __typeof__(*(ptr)) _o_ = (o); \
484 __typeof__(*(ptr)) _n_ = (n); \
485 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
486 (unsigned long)_n_, sizeof(*(ptr))); \
487 })
488
489
490 #define cmpxchg_local(ptr, o, n) \
491 ({ \
492 __typeof__(*(ptr)) _o_ = (o); \
493 __typeof__(*(ptr)) _n_ = (n); \
494 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
495 (unsigned long)_n_, sizeof(*(ptr))); \
496 })
497
498 #define cmpxchg_relaxed(ptr, o, n) \
499 ({ \
500 __typeof__(*(ptr)) _o_ = (o); \
501 __typeof__(*(ptr)) _n_ = (n); \
502 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
503 (unsigned long)_o_, (unsigned long)_n_, \
504 sizeof(*(ptr))); \
505 })
506
507 #define cmpxchg_acquire(ptr, o, n) \
508 ({ \
509 __typeof__(*(ptr)) _o_ = (o); \
510 __typeof__(*(ptr)) _n_ = (n); \
511 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
512 (unsigned long)_o_, (unsigned long)_n_, \
513 sizeof(*(ptr))); \
514 })
515 #ifdef CONFIG_PPC64
516 #define cmpxchg64(ptr, o, n) \
517 ({ \
518 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
519 cmpxchg((ptr), (o), (n)); \
520 })
521 #define cmpxchg64_local(ptr, o, n) \
522 ({ \
523 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
524 cmpxchg_local((ptr), (o), (n)); \
525 })
526 #define cmpxchg64_relaxed(ptr, o, n) \
527 ({ \
528 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
529 cmpxchg_relaxed((ptr), (o), (n)); \
530 })
531 #define cmpxchg64_acquire(ptr, o, n) \
532 ({ \
533 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
534 cmpxchg_acquire((ptr), (o), (n)); \
535 })
536 #else
537 #include <asm-generic/cmpxchg-local.h>
538 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
539 #endif
540
541 #endif
542 #endif