1#ifndef _ASM_X86_CMPXCHG_32_H
2#define _ASM_X86_CMPXCHG_32_H
3
4/*
5 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
6 *       you need to test for the feature in boot_cpu_data.
7 */
8
9/*
10 * CMPXCHG8B only writes to the target if we had the previous
11 * value in registers, otherwise it acts as a read and gives us the
12 * "new previous" value.  That is why there is a loop.  Preloading
13 * EDX:EAX is a performance optimization: in the common case it means
14 * we need only one locked operation.
15 *
16 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
17 * least an FPU save and/or %cr0.ts manipulation.
18 *
19 * cmpxchg8b must be used with the lock prefix here to allow the
20 * instruction to be executed atomically.  We need to have the reader
21 * side to see the coherent 64bit value.
22 */
23static inline void set_64bit(volatile u64 *ptr, u64 value)
24{
25	u32 low  = value;
26	u32 high = value >> 32;
27	u64 prev = *ptr;
28
29	asm volatile("\n1:\t"
30		     LOCK_PREFIX "cmpxchg8b %0\n\t"
31		     "jnz 1b"
32		     : "=m" (*ptr), "+A" (prev)
33		     : "b" (low), "c" (high)
34		     : "memory");
35}
36
37#ifdef CONFIG_X86_CMPXCHG64
38#define cmpxchg64(ptr, o, n)						\
39	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
40					 (unsigned long long)(n)))
41#define cmpxchg64_local(ptr, o, n)					\
42	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
43					       (unsigned long long)(n)))
44#endif
45
46static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
47{
48	u64 prev;
49	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
50		     : "=A" (prev),
51		       "+m" (*ptr)
52		     : "b" ((u32)new),
53		       "c" ((u32)(new >> 32)),
54		       "0" (old)
55		     : "memory");
56	return prev;
57}
58
59static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
60{
61	u64 prev;
62	asm volatile("cmpxchg8b %1"
63		     : "=A" (prev),
64		       "+m" (*ptr)
65		     : "b" ((u32)new),
66		       "c" ((u32)(new >> 32)),
67		       "0" (old)
68		     : "memory");
69	return prev;
70}
71
72#ifndef CONFIG_X86_CMPXCHG64
73/*
74 * Building a kernel capable running on 80386 and 80486. It may be necessary
75 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
76 */
77
78#define cmpxchg64(ptr, o, n)					\
79({								\
80	__typeof__(*(ptr)) __ret;				\
81	__typeof__(*(ptr)) __old = (o);				\
82	__typeof__(*(ptr)) __new = (n);				\
83	alternative_io(LOCK_PREFIX_HERE				\
84			"call cmpxchg8b_emu",			\
85			"lock; cmpxchg8b (%%esi)" ,		\
86		       X86_FEATURE_CX8,				\
87		       "=A" (__ret),				\
88		       "S" ((ptr)), "0" (__old),		\
89		       "b" ((unsigned int)__new),		\
90		       "c" ((unsigned int)(__new>>32))		\
91		       : "memory");				\
92	__ret; })
93
94
95#define cmpxchg64_local(ptr, o, n)				\
96({								\
97	__typeof__(*(ptr)) __ret;				\
98	__typeof__(*(ptr)) __old = (o);				\
99	__typeof__(*(ptr)) __new = (n);				\
100	alternative_io("call cmpxchg8b_emu",			\
101		       "cmpxchg8b (%%esi)" ,			\
102		       X86_FEATURE_CX8,				\
103		       "=A" (__ret),				\
104		       "S" ((ptr)), "0" (__old),		\
105		       "b" ((unsigned int)__new),		\
106		       "c" ((unsigned int)(__new>>32))		\
107		       : "memory");				\
108	__ret; })
109
110#endif
111
112#define system_has_cmpxchg_double() cpu_has_cx8
113
114#endif /* _ASM_X86_CMPXCHG_32_H */
115