1#ifndef _ASM_GENERIC_PERCPU_H_ 2#define _ASM_GENERIC_PERCPU_H_ 3 4#include <linux/compiler.h> 5#include <linux/threads.h> 6#include <linux/percpu-defs.h> 7 8#ifdef CONFIG_SMP 9 10/* 11 * per_cpu_offset() is the offset that has to be added to a 12 * percpu variable to get to the instance for a certain processor. 13 * 14 * Most arches use the __per_cpu_offset array for those offsets but 15 * some arches have their own ways of determining the offset (x86_64, s390). 16 */ 17#ifndef __per_cpu_offset 18extern unsigned long __per_cpu_offset[NR_CPUS]; 19 20#define per_cpu_offset(x) (__per_cpu_offset[x]) 21#endif 22 23/* 24 * Determine the offset for the currently active processor. 25 * An arch may define __my_cpu_offset to provide a more effective 26 * means of obtaining the offset to the per cpu variables of the 27 * current processor. 28 */ 29#ifndef __my_cpu_offset 30#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id()) 31#endif 32#ifdef CONFIG_DEBUG_PREEMPT 33#define my_cpu_offset per_cpu_offset(smp_processor_id()) 34#else 35#define my_cpu_offset __my_cpu_offset 36#endif 37 38/* 39 * Arch may define arch_raw_cpu_ptr() to provide more efficient address 40 * translations for raw_cpu_ptr(). 41 */ 42#ifndef arch_raw_cpu_ptr 43#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) 44#endif 45 46#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA 47extern void setup_per_cpu_areas(void); 48#endif 49 50#endif /* SMP */ 51 52#ifndef PER_CPU_BASE_SECTION 53#ifdef CONFIG_SMP 54#define PER_CPU_BASE_SECTION ".data..percpu" 55#else 56#define PER_CPU_BASE_SECTION ".data" 57#endif 58#endif 59 60#ifndef PER_CPU_ATTRIBUTES 61#define PER_CPU_ATTRIBUTES 62#endif 63 64#ifndef PER_CPU_DEF_ATTRIBUTES 65#define PER_CPU_DEF_ATTRIBUTES 66#endif 67 68#define raw_cpu_generic_to_op(pcp, val, op) \ 69do { \ 70 *raw_cpu_ptr(&(pcp)) op val; \ 71} while (0) 72 73#define raw_cpu_generic_add_return(pcp, val) \ 74({ \ 75 raw_cpu_add(pcp, val); \ 76 raw_cpu_read(pcp); \ 77}) 78 79#define raw_cpu_generic_xchg(pcp, nval) \ 80({ \ 81 typeof(pcp) __ret; \ 82 __ret = raw_cpu_read(pcp); \ 83 raw_cpu_write(pcp, nval); \ 84 __ret; \ 85}) 86 87#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ 88({ \ 89 typeof(pcp) __ret; \ 90 __ret = raw_cpu_read(pcp); \ 91 if (__ret == (oval)) \ 92 raw_cpu_write(pcp, nval); \ 93 __ret; \ 94}) 95 96#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 97({ \ 98 int __ret = 0; \ 99 if (raw_cpu_read(pcp1) == (oval1) && \ 100 raw_cpu_read(pcp2) == (oval2)) { \ 101 raw_cpu_write(pcp1, nval1); \ 102 raw_cpu_write(pcp2, nval2); \ 103 __ret = 1; \ 104 } \ 105 (__ret); \ 106}) 107 108#define this_cpu_generic_read(pcp) \ 109({ \ 110 typeof(pcp) __ret; \ 111 preempt_disable(); \ 112 __ret = *this_cpu_ptr(&(pcp)); \ 113 preempt_enable(); \ 114 __ret; \ 115}) 116 117#define this_cpu_generic_to_op(pcp, val, op) \ 118do { \ 119 unsigned long __flags; \ 120 raw_local_irq_save(__flags); \ 121 *raw_cpu_ptr(&(pcp)) op val; \ 122 raw_local_irq_restore(__flags); \ 123} while (0) 124 125#define this_cpu_generic_add_return(pcp, val) \ 126({ \ 127 typeof(pcp) __ret; \ 128 unsigned long __flags; \ 129 raw_local_irq_save(__flags); \ 130 raw_cpu_add(pcp, val); \ 131 __ret = raw_cpu_read(pcp); \ 132 raw_local_irq_restore(__flags); \ 133 __ret; \ 134}) 135 136#define this_cpu_generic_xchg(pcp, nval) \ 137({ \ 138 typeof(pcp) __ret; \ 139 unsigned long __flags; \ 140 raw_local_irq_save(__flags); \ 141 __ret = raw_cpu_read(pcp); \ 142 raw_cpu_write(pcp, nval); \ 143 raw_local_irq_restore(__flags); \ 144 __ret; \ 145}) 146 147#define this_cpu_generic_cmpxchg(pcp, oval, nval) \ 148({ \ 149 typeof(pcp) __ret; \ 150 unsigned long __flags; \ 151 raw_local_irq_save(__flags); \ 152 __ret = raw_cpu_read(pcp); \ 153 if (__ret == (oval)) \ 154 raw_cpu_write(pcp, nval); \ 155 raw_local_irq_restore(__flags); \ 156 __ret; \ 157}) 158 159#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 160({ \ 161 int __ret; \ 162 unsigned long __flags; \ 163 raw_local_irq_save(__flags); \ 164 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \ 165 oval1, oval2, nval1, nval2); \ 166 raw_local_irq_restore(__flags); \ 167 __ret; \ 168}) 169 170#ifndef raw_cpu_read_1 171#define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp))) 172#endif 173#ifndef raw_cpu_read_2 174#define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp))) 175#endif 176#ifndef raw_cpu_read_4 177#define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp))) 178#endif 179#ifndef raw_cpu_read_8 180#define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp))) 181#endif 182 183#ifndef raw_cpu_write_1 184#define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 185#endif 186#ifndef raw_cpu_write_2 187#define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 188#endif 189#ifndef raw_cpu_write_4 190#define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 191#endif 192#ifndef raw_cpu_write_8 193#define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =) 194#endif 195 196#ifndef raw_cpu_add_1 197#define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 198#endif 199#ifndef raw_cpu_add_2 200#define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 201#endif 202#ifndef raw_cpu_add_4 203#define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 204#endif 205#ifndef raw_cpu_add_8 206#define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=) 207#endif 208 209#ifndef raw_cpu_and_1 210#define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 211#endif 212#ifndef raw_cpu_and_2 213#define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 214#endif 215#ifndef raw_cpu_and_4 216#define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 217#endif 218#ifndef raw_cpu_and_8 219#define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=) 220#endif 221 222#ifndef raw_cpu_or_1 223#define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 224#endif 225#ifndef raw_cpu_or_2 226#define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 227#endif 228#ifndef raw_cpu_or_4 229#define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 230#endif 231#ifndef raw_cpu_or_8 232#define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=) 233#endif 234 235#ifndef raw_cpu_add_return_1 236#define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val) 237#endif 238#ifndef raw_cpu_add_return_2 239#define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val) 240#endif 241#ifndef raw_cpu_add_return_4 242#define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val) 243#endif 244#ifndef raw_cpu_add_return_8 245#define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val) 246#endif 247 248#ifndef raw_cpu_xchg_1 249#define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 250#endif 251#ifndef raw_cpu_xchg_2 252#define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 253#endif 254#ifndef raw_cpu_xchg_4 255#define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 256#endif 257#ifndef raw_cpu_xchg_8 258#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval) 259#endif 260 261#ifndef raw_cpu_cmpxchg_1 262#define raw_cpu_cmpxchg_1(pcp, oval, nval) \ 263 raw_cpu_generic_cmpxchg(pcp, oval, nval) 264#endif 265#ifndef raw_cpu_cmpxchg_2 266#define raw_cpu_cmpxchg_2(pcp, oval, nval) \ 267 raw_cpu_generic_cmpxchg(pcp, oval, nval) 268#endif 269#ifndef raw_cpu_cmpxchg_4 270#define raw_cpu_cmpxchg_4(pcp, oval, nval) \ 271 raw_cpu_generic_cmpxchg(pcp, oval, nval) 272#endif 273#ifndef raw_cpu_cmpxchg_8 274#define raw_cpu_cmpxchg_8(pcp, oval, nval) \ 275 raw_cpu_generic_cmpxchg(pcp, oval, nval) 276#endif 277 278#ifndef raw_cpu_cmpxchg_double_1 279#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 280 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 281#endif 282#ifndef raw_cpu_cmpxchg_double_2 283#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 284 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 285#endif 286#ifndef raw_cpu_cmpxchg_double_4 287#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 288 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 289#endif 290#ifndef raw_cpu_cmpxchg_double_8 291#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 292 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 293#endif 294 295#ifndef this_cpu_read_1 296#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp) 297#endif 298#ifndef this_cpu_read_2 299#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp) 300#endif 301#ifndef this_cpu_read_4 302#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp) 303#endif 304#ifndef this_cpu_read_8 305#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp) 306#endif 307 308#ifndef this_cpu_write_1 309#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =) 310#endif 311#ifndef this_cpu_write_2 312#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =) 313#endif 314#ifndef this_cpu_write_4 315#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =) 316#endif 317#ifndef this_cpu_write_8 318#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =) 319#endif 320 321#ifndef this_cpu_add_1 322#define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 323#endif 324#ifndef this_cpu_add_2 325#define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 326#endif 327#ifndef this_cpu_add_4 328#define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 329#endif 330#ifndef this_cpu_add_8 331#define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=) 332#endif 333 334#ifndef this_cpu_and_1 335#define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 336#endif 337#ifndef this_cpu_and_2 338#define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 339#endif 340#ifndef this_cpu_and_4 341#define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 342#endif 343#ifndef this_cpu_and_8 344#define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=) 345#endif 346 347#ifndef this_cpu_or_1 348#define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 349#endif 350#ifndef this_cpu_or_2 351#define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 352#endif 353#ifndef this_cpu_or_4 354#define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 355#endif 356#ifndef this_cpu_or_8 357#define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=) 358#endif 359 360#ifndef this_cpu_add_return_1 361#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val) 362#endif 363#ifndef this_cpu_add_return_2 364#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val) 365#endif 366#ifndef this_cpu_add_return_4 367#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val) 368#endif 369#ifndef this_cpu_add_return_8 370#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val) 371#endif 372 373#ifndef this_cpu_xchg_1 374#define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval) 375#endif 376#ifndef this_cpu_xchg_2 377#define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval) 378#endif 379#ifndef this_cpu_xchg_4 380#define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval) 381#endif 382#ifndef this_cpu_xchg_8 383#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval) 384#endif 385 386#ifndef this_cpu_cmpxchg_1 387#define this_cpu_cmpxchg_1(pcp, oval, nval) \ 388 this_cpu_generic_cmpxchg(pcp, oval, nval) 389#endif 390#ifndef this_cpu_cmpxchg_2 391#define this_cpu_cmpxchg_2(pcp, oval, nval) \ 392 this_cpu_generic_cmpxchg(pcp, oval, nval) 393#endif 394#ifndef this_cpu_cmpxchg_4 395#define this_cpu_cmpxchg_4(pcp, oval, nval) \ 396 this_cpu_generic_cmpxchg(pcp, oval, nval) 397#endif 398#ifndef this_cpu_cmpxchg_8 399#define this_cpu_cmpxchg_8(pcp, oval, nval) \ 400 this_cpu_generic_cmpxchg(pcp, oval, nval) 401#endif 402 403#ifndef this_cpu_cmpxchg_double_1 404#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 405 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 406#endif 407#ifndef this_cpu_cmpxchg_double_2 408#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 409 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 410#endif 411#ifndef this_cpu_cmpxchg_double_4 412#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 413 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 414#endif 415#ifndef this_cpu_cmpxchg_double_8 416#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ 417 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) 418#endif 419 420#endif /* _ASM_GENERIC_PERCPU_H_ */ 421