1#ifndef _ASM_POWERPC_CMPXCHG_H_
2#define _ASM_POWERPC_CMPXCHG_H_
3
4#ifdef __KERNEL__
5#include <linux/compiler.h>
6#include <asm/synch.h>
7#include <asm/asm-compat.h>
8
9/*
10 * Atomic exchange
11 *
12 * Changes the memory location '*ptr' to be val and returns
13 * the previous value stored there.
14 */
15static __always_inline unsigned long
16__xchg_u32(volatile void *p, unsigned long val)
17{
18	unsigned long prev;
19
20	__asm__ __volatile__(
21	PPC_ATOMIC_ENTRY_BARRIER
22"1:	lwarx	%0,0,%2 \n"
23	PPC405_ERR77(0,%2)
24"	stwcx.	%3,0,%2 \n\
25	bne-	1b"
26	PPC_ATOMIC_EXIT_BARRIER
27	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
28	: "r" (p), "r" (val)
29	: "cc", "memory");
30
31	return prev;
32}
33
34/*
35 * Atomic exchange
36 *
37 * Changes the memory location '*ptr' to be val and returns
38 * the previous value stored there.
39 */
40static __always_inline unsigned long
41__xchg_u32_local(volatile void *p, unsigned long val)
42{
43	unsigned long prev;
44
45	__asm__ __volatile__(
46"1:	lwarx	%0,0,%2 \n"
47	PPC405_ERR77(0,%2)
48"	stwcx.	%3,0,%2 \n\
49	bne-	1b"
50	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
51	: "r" (p), "r" (val)
52	: "cc", "memory");
53
54	return prev;
55}
56
57#ifdef CONFIG_PPC64
58static __always_inline unsigned long
59__xchg_u64(volatile void *p, unsigned long val)
60{
61	unsigned long prev;
62
63	__asm__ __volatile__(
64	PPC_ATOMIC_ENTRY_BARRIER
65"1:	ldarx	%0,0,%2 \n"
66	PPC405_ERR77(0,%2)
67"	stdcx.	%3,0,%2 \n\
68	bne-	1b"
69	PPC_ATOMIC_EXIT_BARRIER
70	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
71	: "r" (p), "r" (val)
72	: "cc", "memory");
73
74	return prev;
75}
76
77static __always_inline unsigned long
78__xchg_u64_local(volatile void *p, unsigned long val)
79{
80	unsigned long prev;
81
82	__asm__ __volatile__(
83"1:	ldarx	%0,0,%2 \n"
84	PPC405_ERR77(0,%2)
85"	stdcx.	%3,0,%2 \n\
86	bne-	1b"
87	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
88	: "r" (p), "r" (val)
89	: "cc", "memory");
90
91	return prev;
92}
93#endif
94
95/*
96 * This function doesn't exist, so you'll get a linker error
97 * if something tries to do an invalid xchg().
98 */
99extern void __xchg_called_with_bad_pointer(void);
100
101static __always_inline unsigned long
102__xchg(volatile void *ptr, unsigned long x, unsigned int size)
103{
104	switch (size) {
105	case 4:
106		return __xchg_u32(ptr, x);
107#ifdef CONFIG_PPC64
108	case 8:
109		return __xchg_u64(ptr, x);
110#endif
111	}
112	__xchg_called_with_bad_pointer();
113	return x;
114}
115
116static __always_inline unsigned long
117__xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
118{
119	switch (size) {
120	case 4:
121		return __xchg_u32_local(ptr, x);
122#ifdef CONFIG_PPC64
123	case 8:
124		return __xchg_u64_local(ptr, x);
125#endif
126	}
127	__xchg_called_with_bad_pointer();
128	return x;
129}
130#define xchg(ptr,x)							     \
131  ({									     \
132     __typeof__(*(ptr)) _x_ = (x);					     \
133     (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
134  })
135
136#define xchg_local(ptr,x)						     \
137  ({									     \
138     __typeof__(*(ptr)) _x_ = (x);					     \
139     (__typeof__(*(ptr))) __xchg_local((ptr),				     \
140     		(unsigned long)_x_, sizeof(*(ptr))); 			     \
141  })
142
143/*
144 * Compare and exchange - if *p == old, set it to new,
145 * and return the old value of *p.
146 */
147#define __HAVE_ARCH_CMPXCHG	1
148
149static __always_inline unsigned long
150__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
151{
152	unsigned int prev;
153
154	__asm__ __volatile__ (
155	PPC_ATOMIC_ENTRY_BARRIER
156"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
157	cmpw	0,%0,%3\n\
158	bne-	2f\n"
159	PPC405_ERR77(0,%2)
160"	stwcx.	%4,0,%2\n\
161	bne-	1b"
162	PPC_ATOMIC_EXIT_BARRIER
163	"\n\
1642:"
165	: "=&r" (prev), "+m" (*p)
166	: "r" (p), "r" (old), "r" (new)
167	: "cc", "memory");
168
169	return prev;
170}
171
172static __always_inline unsigned long
173__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
174			unsigned long new)
175{
176	unsigned int prev;
177
178	__asm__ __volatile__ (
179"1:	lwarx	%0,0,%2		# __cmpxchg_u32\n\
180	cmpw	0,%0,%3\n\
181	bne-	2f\n"
182	PPC405_ERR77(0,%2)
183"	stwcx.	%4,0,%2\n\
184	bne-	1b"
185	"\n\
1862:"
187	: "=&r" (prev), "+m" (*p)
188	: "r" (p), "r" (old), "r" (new)
189	: "cc", "memory");
190
191	return prev;
192}
193
194#ifdef CONFIG_PPC64
195static __always_inline unsigned long
196__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
197{
198	unsigned long prev;
199
200	__asm__ __volatile__ (
201	PPC_ATOMIC_ENTRY_BARRIER
202"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
203	cmpd	0,%0,%3\n\
204	bne-	2f\n\
205	stdcx.	%4,0,%2\n\
206	bne-	1b"
207	PPC_ATOMIC_EXIT_BARRIER
208	"\n\
2092:"
210	: "=&r" (prev), "+m" (*p)
211	: "r" (p), "r" (old), "r" (new)
212	: "cc", "memory");
213
214	return prev;
215}
216
217static __always_inline unsigned long
218__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
219			unsigned long new)
220{
221	unsigned long prev;
222
223	__asm__ __volatile__ (
224"1:	ldarx	%0,0,%2		# __cmpxchg_u64\n\
225	cmpd	0,%0,%3\n\
226	bne-	2f\n\
227	stdcx.	%4,0,%2\n\
228	bne-	1b"
229	"\n\
2302:"
231	: "=&r" (prev), "+m" (*p)
232	: "r" (p), "r" (old), "r" (new)
233	: "cc", "memory");
234
235	return prev;
236}
237#endif
238
239/* This function doesn't exist, so you'll get a linker error
240   if something tries to do an invalid cmpxchg().  */
241extern void __cmpxchg_called_with_bad_pointer(void);
242
243static __always_inline unsigned long
244__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
245	  unsigned int size)
246{
247	switch (size) {
248	case 4:
249		return __cmpxchg_u32(ptr, old, new);
250#ifdef CONFIG_PPC64
251	case 8:
252		return __cmpxchg_u64(ptr, old, new);
253#endif
254	}
255	__cmpxchg_called_with_bad_pointer();
256	return old;
257}
258
259static __always_inline unsigned long
260__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
261	  unsigned int size)
262{
263	switch (size) {
264	case 4:
265		return __cmpxchg_u32_local(ptr, old, new);
266#ifdef CONFIG_PPC64
267	case 8:
268		return __cmpxchg_u64_local(ptr, old, new);
269#endif
270	}
271	__cmpxchg_called_with_bad_pointer();
272	return old;
273}
274
275#define cmpxchg(ptr, o, n)						 \
276  ({									 \
277     __typeof__(*(ptr)) _o_ = (o);					 \
278     __typeof__(*(ptr)) _n_ = (n);					 \
279     (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,		 \
280				    (unsigned long)_n_, sizeof(*(ptr))); \
281  })
282
283
284#define cmpxchg_local(ptr, o, n)					 \
285  ({									 \
286     __typeof__(*(ptr)) _o_ = (o);					 \
287     __typeof__(*(ptr)) _n_ = (n);					 \
288     (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,	 \
289				    (unsigned long)_n_, sizeof(*(ptr))); \
290  })
291
292#ifdef CONFIG_PPC64
293#define cmpxchg64(ptr, o, n)						\
294  ({									\
295	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
296	cmpxchg((ptr), (o), (n));					\
297  })
298#define cmpxchg64_local(ptr, o, n)					\
299  ({									\
300	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
301	cmpxchg_local((ptr), (o), (n));					\
302  })
303#define cmpxchg64_relaxed	cmpxchg64_local
304#else
305#include <asm-generic/cmpxchg-local.h>
306#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
307#endif
308
309#endif /* __KERNEL__ */
310#endif /* _ASM_POWERPC_CMPXCHG_H_ */
311