1#ifndef _ASM_GENERIC_PERCPU_H_
2#define _ASM_GENERIC_PERCPU_H_
3
4#include <linux/compiler.h>
5#include <linux/threads.h>
6#include <linux/percpu-defs.h>
7
8#ifdef CONFIG_SMP
9
10/*
11 * per_cpu_offset() is the offset that has to be added to a
12 * percpu variable to get to the instance for a certain processor.
13 *
14 * Most arches use the __per_cpu_offset array for those offsets but
15 * some arches have their own ways of determining the offset (x86_64, s390).
16 */
17#ifndef __per_cpu_offset
18extern unsigned long __per_cpu_offset[NR_CPUS];
19
20#define per_cpu_offset(x) (__per_cpu_offset[x])
21#endif
22
23/*
24 * Determine the offset for the currently active processor.
25 * An arch may define __my_cpu_offset to provide a more effective
26 * means of obtaining the offset to the per cpu variables of the
27 * current processor.
28 */
29#ifndef __my_cpu_offset
30#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
31#endif
32#ifdef CONFIG_DEBUG_PREEMPT
33#define my_cpu_offset per_cpu_offset(smp_processor_id())
34#else
35#define my_cpu_offset __my_cpu_offset
36#endif
37
38/*
39 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
40 * translations for raw_cpu_ptr().
41 */
42#ifndef arch_raw_cpu_ptr
43#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
44#endif
45
46#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
47extern void setup_per_cpu_areas(void);
48#endif
49
50#endif	/* SMP */
51
52#ifndef PER_CPU_BASE_SECTION
53#ifdef CONFIG_SMP
54#define PER_CPU_BASE_SECTION ".data..percpu"
55#else
56#define PER_CPU_BASE_SECTION ".data"
57#endif
58#endif
59
60#ifndef PER_CPU_ATTRIBUTES
61#define PER_CPU_ATTRIBUTES
62#endif
63
64#ifndef PER_CPU_DEF_ATTRIBUTES
65#define PER_CPU_DEF_ATTRIBUTES
66#endif
67
68#define raw_cpu_generic_to_op(pcp, val, op)				\
69do {									\
70	*raw_cpu_ptr(&(pcp)) op val;					\
71} while (0)
72
73#define raw_cpu_generic_add_return(pcp, val)				\
74({									\
75	raw_cpu_add(pcp, val);						\
76	raw_cpu_read(pcp);						\
77})
78
79#define raw_cpu_generic_xchg(pcp, nval)					\
80({									\
81	typeof(pcp) __ret;						\
82	__ret = raw_cpu_read(pcp);					\
83	raw_cpu_write(pcp, nval);					\
84	__ret;								\
85})
86
87#define raw_cpu_generic_cmpxchg(pcp, oval, nval)			\
88({									\
89	typeof(pcp) __ret;						\
90	__ret = raw_cpu_read(pcp);					\
91	if (__ret == (oval))						\
92		raw_cpu_write(pcp, nval);				\
93	__ret;								\
94})
95
96#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
97({									\
98	int __ret = 0;							\
99	if (raw_cpu_read(pcp1) == (oval1) &&				\
100			 raw_cpu_read(pcp2)  == (oval2)) {		\
101		raw_cpu_write(pcp1, nval1);				\
102		raw_cpu_write(pcp2, nval2);				\
103		__ret = 1;						\
104	}								\
105	(__ret);							\
106})
107
108#define this_cpu_generic_read(pcp)					\
109({									\
110	typeof(pcp) __ret;						\
111	preempt_disable();						\
112	__ret = *this_cpu_ptr(&(pcp));					\
113	preempt_enable();						\
114	__ret;								\
115})
116
117#define this_cpu_generic_to_op(pcp, val, op)				\
118do {									\
119	unsigned long __flags;						\
120	raw_local_irq_save(__flags);					\
121	*raw_cpu_ptr(&(pcp)) op val;					\
122	raw_local_irq_restore(__flags);					\
123} while (0)
124
125#define this_cpu_generic_add_return(pcp, val)				\
126({									\
127	typeof(pcp) __ret;						\
128	unsigned long __flags;						\
129	raw_local_irq_save(__flags);					\
130	raw_cpu_add(pcp, val);						\
131	__ret = raw_cpu_read(pcp);					\
132	raw_local_irq_restore(__flags);					\
133	__ret;								\
134})
135
136#define this_cpu_generic_xchg(pcp, nval)				\
137({									\
138	typeof(pcp) __ret;						\
139	unsigned long __flags;						\
140	raw_local_irq_save(__flags);					\
141	__ret = raw_cpu_read(pcp);					\
142	raw_cpu_write(pcp, nval);					\
143	raw_local_irq_restore(__flags);					\
144	__ret;								\
145})
146
147#define this_cpu_generic_cmpxchg(pcp, oval, nval)			\
148({									\
149	typeof(pcp) __ret;						\
150	unsigned long __flags;						\
151	raw_local_irq_save(__flags);					\
152	__ret = raw_cpu_read(pcp);					\
153	if (__ret == (oval))						\
154		raw_cpu_write(pcp, nval);				\
155	raw_local_irq_restore(__flags);					\
156	__ret;								\
157})
158
159#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)	\
160({									\
161	int __ret;							\
162	unsigned long __flags;						\
163	raw_local_irq_save(__flags);					\
164	__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,		\
165			oval1, oval2, nval1, nval2);			\
166	raw_local_irq_restore(__flags);					\
167	__ret;								\
168})
169
170#ifndef raw_cpu_read_1
171#define raw_cpu_read_1(pcp)		(*raw_cpu_ptr(&(pcp)))
172#endif
173#ifndef raw_cpu_read_2
174#define raw_cpu_read_2(pcp)		(*raw_cpu_ptr(&(pcp)))
175#endif
176#ifndef raw_cpu_read_4
177#define raw_cpu_read_4(pcp)		(*raw_cpu_ptr(&(pcp)))
178#endif
179#ifndef raw_cpu_read_8
180#define raw_cpu_read_8(pcp)		(*raw_cpu_ptr(&(pcp)))
181#endif
182
183#ifndef raw_cpu_write_1
184#define raw_cpu_write_1(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
185#endif
186#ifndef raw_cpu_write_2
187#define raw_cpu_write_2(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
188#endif
189#ifndef raw_cpu_write_4
190#define raw_cpu_write_4(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
191#endif
192#ifndef raw_cpu_write_8
193#define raw_cpu_write_8(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
194#endif
195
196#ifndef raw_cpu_add_1
197#define raw_cpu_add_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
198#endif
199#ifndef raw_cpu_add_2
200#define raw_cpu_add_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
201#endif
202#ifndef raw_cpu_add_4
203#define raw_cpu_add_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
204#endif
205#ifndef raw_cpu_add_8
206#define raw_cpu_add_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
207#endif
208
209#ifndef raw_cpu_and_1
210#define raw_cpu_and_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
211#endif
212#ifndef raw_cpu_and_2
213#define raw_cpu_and_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
214#endif
215#ifndef raw_cpu_and_4
216#define raw_cpu_and_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
217#endif
218#ifndef raw_cpu_and_8
219#define raw_cpu_and_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
220#endif
221
222#ifndef raw_cpu_or_1
223#define raw_cpu_or_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
224#endif
225#ifndef raw_cpu_or_2
226#define raw_cpu_or_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
227#endif
228#ifndef raw_cpu_or_4
229#define raw_cpu_or_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
230#endif
231#ifndef raw_cpu_or_8
232#define raw_cpu_or_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
233#endif
234
235#ifndef raw_cpu_add_return_1
236#define raw_cpu_add_return_1(pcp, val)	raw_cpu_generic_add_return(pcp, val)
237#endif
238#ifndef raw_cpu_add_return_2
239#define raw_cpu_add_return_2(pcp, val)	raw_cpu_generic_add_return(pcp, val)
240#endif
241#ifndef raw_cpu_add_return_4
242#define raw_cpu_add_return_4(pcp, val)	raw_cpu_generic_add_return(pcp, val)
243#endif
244#ifndef raw_cpu_add_return_8
245#define raw_cpu_add_return_8(pcp, val)	raw_cpu_generic_add_return(pcp, val)
246#endif
247
248#ifndef raw_cpu_xchg_1
249#define raw_cpu_xchg_1(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
250#endif
251#ifndef raw_cpu_xchg_2
252#define raw_cpu_xchg_2(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
253#endif
254#ifndef raw_cpu_xchg_4
255#define raw_cpu_xchg_4(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
256#endif
257#ifndef raw_cpu_xchg_8
258#define raw_cpu_xchg_8(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
259#endif
260
261#ifndef raw_cpu_cmpxchg_1
262#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
263	raw_cpu_generic_cmpxchg(pcp, oval, nval)
264#endif
265#ifndef raw_cpu_cmpxchg_2
266#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
267	raw_cpu_generic_cmpxchg(pcp, oval, nval)
268#endif
269#ifndef raw_cpu_cmpxchg_4
270#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
271	raw_cpu_generic_cmpxchg(pcp, oval, nval)
272#endif
273#ifndef raw_cpu_cmpxchg_8
274#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
275	raw_cpu_generic_cmpxchg(pcp, oval, nval)
276#endif
277
278#ifndef raw_cpu_cmpxchg_double_1
279#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
280	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
281#endif
282#ifndef raw_cpu_cmpxchg_double_2
283#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
284	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
285#endif
286#ifndef raw_cpu_cmpxchg_double_4
287#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
288	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
289#endif
290#ifndef raw_cpu_cmpxchg_double_8
291#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
292	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
293#endif
294
295#ifndef this_cpu_read_1
296#define this_cpu_read_1(pcp)		this_cpu_generic_read(pcp)
297#endif
298#ifndef this_cpu_read_2
299#define this_cpu_read_2(pcp)		this_cpu_generic_read(pcp)
300#endif
301#ifndef this_cpu_read_4
302#define this_cpu_read_4(pcp)		this_cpu_generic_read(pcp)
303#endif
304#ifndef this_cpu_read_8
305#define this_cpu_read_8(pcp)		this_cpu_generic_read(pcp)
306#endif
307
308#ifndef this_cpu_write_1
309#define this_cpu_write_1(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
310#endif
311#ifndef this_cpu_write_2
312#define this_cpu_write_2(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
313#endif
314#ifndef this_cpu_write_4
315#define this_cpu_write_4(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
316#endif
317#ifndef this_cpu_write_8
318#define this_cpu_write_8(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
319#endif
320
321#ifndef this_cpu_add_1
322#define this_cpu_add_1(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
323#endif
324#ifndef this_cpu_add_2
325#define this_cpu_add_2(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
326#endif
327#ifndef this_cpu_add_4
328#define this_cpu_add_4(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
329#endif
330#ifndef this_cpu_add_8
331#define this_cpu_add_8(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
332#endif
333
334#ifndef this_cpu_and_1
335#define this_cpu_and_1(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
336#endif
337#ifndef this_cpu_and_2
338#define this_cpu_and_2(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
339#endif
340#ifndef this_cpu_and_4
341#define this_cpu_and_4(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
342#endif
343#ifndef this_cpu_and_8
344#define this_cpu_and_8(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
345#endif
346
347#ifndef this_cpu_or_1
348#define this_cpu_or_1(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
349#endif
350#ifndef this_cpu_or_2
351#define this_cpu_or_2(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
352#endif
353#ifndef this_cpu_or_4
354#define this_cpu_or_4(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
355#endif
356#ifndef this_cpu_or_8
357#define this_cpu_or_8(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
358#endif
359
360#ifndef this_cpu_add_return_1
361#define this_cpu_add_return_1(pcp, val)	this_cpu_generic_add_return(pcp, val)
362#endif
363#ifndef this_cpu_add_return_2
364#define this_cpu_add_return_2(pcp, val)	this_cpu_generic_add_return(pcp, val)
365#endif
366#ifndef this_cpu_add_return_4
367#define this_cpu_add_return_4(pcp, val)	this_cpu_generic_add_return(pcp, val)
368#endif
369#ifndef this_cpu_add_return_8
370#define this_cpu_add_return_8(pcp, val)	this_cpu_generic_add_return(pcp, val)
371#endif
372
373#ifndef this_cpu_xchg_1
374#define this_cpu_xchg_1(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
375#endif
376#ifndef this_cpu_xchg_2
377#define this_cpu_xchg_2(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
378#endif
379#ifndef this_cpu_xchg_4
380#define this_cpu_xchg_4(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
381#endif
382#ifndef this_cpu_xchg_8
383#define this_cpu_xchg_8(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
384#endif
385
386#ifndef this_cpu_cmpxchg_1
387#define this_cpu_cmpxchg_1(pcp, oval, nval) \
388	this_cpu_generic_cmpxchg(pcp, oval, nval)
389#endif
390#ifndef this_cpu_cmpxchg_2
391#define this_cpu_cmpxchg_2(pcp, oval, nval) \
392	this_cpu_generic_cmpxchg(pcp, oval, nval)
393#endif
394#ifndef this_cpu_cmpxchg_4
395#define this_cpu_cmpxchg_4(pcp, oval, nval) \
396	this_cpu_generic_cmpxchg(pcp, oval, nval)
397#endif
398#ifndef this_cpu_cmpxchg_8
399#define this_cpu_cmpxchg_8(pcp, oval, nval) \
400	this_cpu_generic_cmpxchg(pcp, oval, nval)
401#endif
402
403#ifndef this_cpu_cmpxchg_double_1
404#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
405	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
406#endif
407#ifndef this_cpu_cmpxchg_double_2
408#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
409	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
410#endif
411#ifndef this_cpu_cmpxchg_double_4
412#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
413	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
414#endif
415#ifndef this_cpu_cmpxchg_double_8
416#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
417	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
418#endif
419
420#endif /* _ASM_GENERIC_PERCPU_H_ */
421