1#ifndef __ARCH_S390_PERCPU__
2#define __ARCH_S390_PERCPU__
3
4#include <linux/preempt.h>
5#include <asm/cmpxchg.h>
6
7/*
8 * s390 uses its own implementation for per cpu data, the offset of
9 * the cpu local data area is cached in the cpu's lowcore memory.
10 */
11#define __my_cpu_offset S390_lowcore.percpu_offset
12
13/*
14 * For 64 bit module code, the module may be more than 4G above the
15 * per cpu area, use weak definitions to force the compiler to
16 * generate external references.
17 */
18#if defined(CONFIG_SMP) && defined(MODULE)
19#define ARCH_NEEDS_WEAK_PER_CPU
20#endif
21
22/*
23 * We use a compare-and-swap loop since that uses less cpu cycles than
24 * disabling and enabling interrupts like the generic variant would do.
25 */
26#define arch_this_cpu_to_op_simple(pcp, val, op)			\
27({									\
28	typedef typeof(pcp) pcp_op_T__;					\
29	pcp_op_T__ old__, new__, prev__;				\
30	pcp_op_T__ *ptr__;						\
31	preempt_disable();						\
32	ptr__ = raw_cpu_ptr(&(pcp));					\
33	prev__ = *ptr__;						\
34	do {								\
35		old__ = prev__;						\
36		new__ = old__ op (val);					\
37		prev__ = cmpxchg(ptr__, old__, new__);			\
38	} while (prev__ != old__);					\
39	preempt_enable();						\
40	new__;								\
41})
42
43#define this_cpu_add_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
44#define this_cpu_add_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
45#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
46#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
47#define this_cpu_and_1(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
48#define this_cpu_and_2(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
49#define this_cpu_or_1(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
50#define this_cpu_or_2(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
51
52#ifndef CONFIG_HAVE_MARCH_Z196_FEATURES
53
54#define this_cpu_add_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
55#define this_cpu_add_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
56#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
57#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
58#define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
59#define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, &)
60#define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
61#define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op_simple(pcp, val, |)
62
63#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
64
65#define arch_this_cpu_add(pcp, val, op1, op2, szcast)			\
66{									\
67	typedef typeof(pcp) pcp_op_T__; 				\
68	pcp_op_T__ val__ = (val);					\
69	pcp_op_T__ old__, *ptr__;					\
70	preempt_disable();						\
71	ptr__ = raw_cpu_ptr(&(pcp)); 				\
72	if (__builtin_constant_p(val__) &&				\
73	    ((szcast)val__ > -129) && ((szcast)val__ < 128)) {		\
74		asm volatile(						\
75			op2 "   %[ptr__],%[val__]\n"			\
76			: [ptr__] "+Q" (*ptr__) 			\
77			: [val__] "i" ((szcast)val__)			\
78			: "cc");					\
79	} else {							\
80		asm volatile(						\
81			op1 "   %[old__],%[val__],%[ptr__]\n"		\
82			: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)	\
83			: [val__] "d" (val__)				\
84			: "cc");					\
85	}								\
86	preempt_enable();						\
87}
88
89#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
90#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
91
92#define arch_this_cpu_add_return(pcp, val, op)				\
93({									\
94	typedef typeof(pcp) pcp_op_T__; 				\
95	pcp_op_T__ val__ = (val);					\
96	pcp_op_T__ old__, *ptr__;					\
97	preempt_disable();						\
98	ptr__ = raw_cpu_ptr(&(pcp));	 				\
99	asm volatile(							\
100		op "    %[old__],%[val__],%[ptr__]\n"			\
101		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
102		: [val__] "d" (val__)					\
103		: "cc");						\
104	preempt_enable();						\
105	old__ + val__;							\
106})
107
108#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
109#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
110
111#define arch_this_cpu_to_op(pcp, val, op)				\
112{									\
113	typedef typeof(pcp) pcp_op_T__; 				\
114	pcp_op_T__ val__ = (val);					\
115	pcp_op_T__ old__, *ptr__;					\
116	preempt_disable();						\
117	ptr__ = raw_cpu_ptr(&(pcp));	 				\
118	asm volatile(							\
119		op "    %[old__],%[val__],%[ptr__]\n"			\
120		: [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)		\
121		: [val__] "d" (val__)					\
122		: "cc");						\
123	preempt_enable();						\
124}
125
126#define this_cpu_and_4(pcp, val)	arch_this_cpu_to_op(pcp, val, "lan")
127#define this_cpu_and_8(pcp, val)	arch_this_cpu_to_op(pcp, val, "lang")
128#define this_cpu_or_4(pcp, val)		arch_this_cpu_to_op(pcp, val, "lao")
129#define this_cpu_or_8(pcp, val)		arch_this_cpu_to_op(pcp, val, "laog")
130
131#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
132
133#define arch_this_cpu_cmpxchg(pcp, oval, nval)				\
134({									\
135	typedef typeof(pcp) pcp_op_T__;					\
136	pcp_op_T__ ret__;						\
137	pcp_op_T__ *ptr__;						\
138	preempt_disable();						\
139	ptr__ = raw_cpu_ptr(&(pcp));					\
140	ret__ = cmpxchg(ptr__, oval, nval);				\
141	preempt_enable();						\
142	ret__;								\
143})
144
145#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
146#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
147#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
148#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149
150#define arch_this_cpu_xchg(pcp, nval)					\
151({									\
152	typeof(pcp) *ptr__;						\
153	typeof(pcp) ret__;						\
154	preempt_disable();						\
155	ptr__ = raw_cpu_ptr(&(pcp));					\
156	ret__ = xchg(ptr__, nval);					\
157	preempt_enable();						\
158	ret__;								\
159})
160
161#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
162#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
163#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
164#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
165
166#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2)	\
167({									\
168	typeof(pcp1) o1__ = (o1), n1__ = (n1);				\
169	typeof(pcp2) o2__ = (o2), n2__ = (n2);				\
170	typeof(pcp1) *p1__;						\
171	typeof(pcp2) *p2__;						\
172	int ret__;							\
173	preempt_disable();						\
174	p1__ = raw_cpu_ptr(&(pcp1));					\
175	p2__ = raw_cpu_ptr(&(pcp2));					\
176	ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__);	\
177	preempt_enable();						\
178	ret__;								\
179})
180
181#define this_cpu_cmpxchg_double_4 arch_this_cpu_cmpxchg_double
182#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
183
184#include <asm-generic/percpu.h>
185
186#endif /* __ARCH_S390_PERCPU__ */
187