1#ifndef __ASM_ARM_CMPXCHG_H
2#define __ASM_ARM_CMPXCHG_H
3
4#include <linux/irqflags.h>
5#include <linux/prefetch.h>
6#include <asm/barrier.h>
7
8#if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
9/*
10 * On the StrongARM, "swp" is terminally broken since it bypasses the
11 * cache totally.  This means that the cache becomes inconsistent, and,
12 * since we use normal loads/stores as well, this is really bad.
13 * Typically, this causes oopsen in filp_close, but could have other,
14 * more disastrous effects.  There are two work-arounds:
15 *  1. Disable interrupts and emulate the atomic swap
16 *  2. Clean the cache, perform atomic swap, flush the cache
17 *
18 * We choose (1) since its the "easiest" to achieve here and is not
19 * dependent on the processor type.
20 *
21 * NOTE that this solution won't work on an SMP system, so explcitly
22 * forbid it here.
23 */
24#define swp_is_buggy
25#endif
26
27static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
28{
29	extern void __bad_xchg(volatile void *, int);
30	unsigned long ret;
31#ifdef swp_is_buggy
32	unsigned long flags;
33#endif
34#if __LINUX_ARM_ARCH__ >= 6
35	unsigned int tmp;
36#endif
37
38	prefetchw((const void *)ptr);
39
40	switch (size) {
41#if __LINUX_ARM_ARCH__ >= 6
42#ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */
43	case 1:
44		asm volatile("@	__xchg1\n"
45		"1:	ldrexb	%0, [%3]\n"
46		"	strexb	%1, %2, [%3]\n"
47		"	teq	%1, #0\n"
48		"	bne	1b"
49			: "=&r" (ret), "=&r" (tmp)
50			: "r" (x), "r" (ptr)
51			: "memory", "cc");
52		break;
53	case 2:
54		asm volatile("@	__xchg2\n"
55		"1:	ldrexh	%0, [%3]\n"
56		"	strexh	%1, %2, [%3]\n"
57		"	teq	%1, #0\n"
58		"	bne	1b"
59			: "=&r" (ret), "=&r" (tmp)
60			: "r" (x), "r" (ptr)
61			: "memory", "cc");
62		break;
63#endif
64	case 4:
65		asm volatile("@	__xchg4\n"
66		"1:	ldrex	%0, [%3]\n"
67		"	strex	%1, %2, [%3]\n"
68		"	teq	%1, #0\n"
69		"	bne	1b"
70			: "=&r" (ret), "=&r" (tmp)
71			: "r" (x), "r" (ptr)
72			: "memory", "cc");
73		break;
74#elif defined(swp_is_buggy)
75#ifdef CONFIG_SMP
76#error SMP is not supported on this platform
77#endif
78	case 1:
79		raw_local_irq_save(flags);
80		ret = *(volatile unsigned char *)ptr;
81		*(volatile unsigned char *)ptr = x;
82		raw_local_irq_restore(flags);
83		break;
84
85	case 4:
86		raw_local_irq_save(flags);
87		ret = *(volatile unsigned long *)ptr;
88		*(volatile unsigned long *)ptr = x;
89		raw_local_irq_restore(flags);
90		break;
91#else
92	case 1:
93		asm volatile("@	__xchg1\n"
94		"	swpb	%0, %1, [%2]"
95			: "=&r" (ret)
96			: "r" (x), "r" (ptr)
97			: "memory", "cc");
98		break;
99	case 4:
100		asm volatile("@	__xchg4\n"
101		"	swp	%0, %1, [%2]"
102			: "=&r" (ret)
103			: "r" (x), "r" (ptr)
104			: "memory", "cc");
105		break;
106#endif
107	default:
108		/* Cause a link-time error, the xchg() size is not supported */
109		__bad_xchg(ptr, size), ret = 0;
110		break;
111	}
112
113	return ret;
114}
115
116#define xchg_relaxed(ptr, x) ({						\
117	(__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr),		\
118				   sizeof(*(ptr)));			\
119})
120
121#include <asm-generic/cmpxchg-local.h>
122
123#if __LINUX_ARM_ARCH__ < 6
124/* min ARCH < ARMv6 */
125
126#ifdef CONFIG_SMP
127#error "SMP is not supported on this platform"
128#endif
129
130#define xchg xchg_relaxed
131
132/*
133 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
134 * them available.
135 */
136#define cmpxchg_local(ptr, o, n) ({					\
137	(__typeof(*ptr))__cmpxchg_local_generic((ptr),			\
138					        (unsigned long)(o),	\
139					        (unsigned long)(n),	\
140					        sizeof(*(ptr)));	\
141})
142
143#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
144
145#include <asm-generic/cmpxchg.h>
146
147#else	/* min ARCH >= ARMv6 */
148
149extern void __bad_cmpxchg(volatile void *ptr, int size);
150
151/*
152 * cmpxchg only support 32-bits operands on ARMv6.
153 */
154
155static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
156				      unsigned long new, int size)
157{
158	unsigned long oldval, res;
159
160	prefetchw((const void *)ptr);
161
162	switch (size) {
163#ifndef CONFIG_CPU_V6	/* min ARCH >= ARMv6K */
164	case 1:
165		do {
166			asm volatile("@ __cmpxchg1\n"
167			"	ldrexb	%1, [%2]\n"
168			"	mov	%0, #0\n"
169			"	teq	%1, %3\n"
170			"	strexbeq %0, %4, [%2]\n"
171				: "=&r" (res), "=&r" (oldval)
172				: "r" (ptr), "Ir" (old), "r" (new)
173				: "memory", "cc");
174		} while (res);
175		break;
176	case 2:
177		do {
178			asm volatile("@ __cmpxchg1\n"
179			"	ldrexh	%1, [%2]\n"
180			"	mov	%0, #0\n"
181			"	teq	%1, %3\n"
182			"	strexheq %0, %4, [%2]\n"
183				: "=&r" (res), "=&r" (oldval)
184				: "r" (ptr), "Ir" (old), "r" (new)
185				: "memory", "cc");
186		} while (res);
187		break;
188#endif
189	case 4:
190		do {
191			asm volatile("@ __cmpxchg4\n"
192			"	ldrex	%1, [%2]\n"
193			"	mov	%0, #0\n"
194			"	teq	%1, %3\n"
195			"	strexeq %0, %4, [%2]\n"
196				: "=&r" (res), "=&r" (oldval)
197				: "r" (ptr), "Ir" (old), "r" (new)
198				: "memory", "cc");
199		} while (res);
200		break;
201	default:
202		__bad_cmpxchg(ptr, size);
203		oldval = 0;
204	}
205
206	return oldval;
207}
208
209#define cmpxchg_relaxed(ptr,o,n) ({					\
210	(__typeof__(*(ptr)))__cmpxchg((ptr),				\
211				      (unsigned long)(o),		\
212				      (unsigned long)(n),		\
213				      sizeof(*(ptr)));			\
214})
215
216static inline unsigned long __cmpxchg_local(volatile void *ptr,
217					    unsigned long old,
218					    unsigned long new, int size)
219{
220	unsigned long ret;
221
222	switch (size) {
223#ifdef CONFIG_CPU_V6	/* min ARCH == ARMv6 */
224	case 1:
225	case 2:
226		ret = __cmpxchg_local_generic(ptr, old, new, size);
227		break;
228#endif
229	default:
230		ret = __cmpxchg(ptr, old, new, size);
231	}
232
233	return ret;
234}
235
236#define cmpxchg_local(ptr, o, n) ({					\
237	(__typeof(*ptr))__cmpxchg_local((ptr),				\
238				        (unsigned long)(o),		\
239				        (unsigned long)(n),		\
240				        sizeof(*(ptr)));		\
241})
242
243static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
244					     unsigned long long old,
245					     unsigned long long new)
246{
247	unsigned long long oldval;
248	unsigned long res;
249
250	prefetchw(ptr);
251
252	__asm__ __volatile__(
253"1:	ldrexd		%1, %H1, [%3]\n"
254"	teq		%1, %4\n"
255"	teqeq		%H1, %H4\n"
256"	bne		2f\n"
257"	strexd		%0, %5, %H5, [%3]\n"
258"	teq		%0, #0\n"
259"	bne		1b\n"
260"2:"
261	: "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
262	: "r" (ptr), "r" (old), "r" (new)
263	: "cc");
264
265	return oldval;
266}
267
268#define cmpxchg64_relaxed(ptr, o, n) ({					\
269	(__typeof__(*(ptr)))__cmpxchg64((ptr),				\
270					(unsigned long long)(o),	\
271					(unsigned long long)(n));	\
272})
273
274#define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n))
275
276#endif	/* __LINUX_ARM_ARCH__ >= 6 */
277
278#endif /* __ASM_ARM_CMPXCHG_H */
279