1#ifndef __ARCH_S390_PERCPU__ 2#define __ARCH_S390_PERCPU__ 3 4#include <linux/preempt.h> 5#include <asm/cmpxchg.h> 6 7/* 8 * s390 uses its own implementation for per cpu data, the offset of 9 * the cpu local data area is cached in the cpu's lowcore memory. 10 */ 11#define __my_cpu_offset S390_lowcore.percpu_offset 12 13/* 14 * For 64 bit module code, the module may be more than 4G above the 15 * per cpu area, use weak definitions to force the compiler to 16 * generate external references. 17 */ 18#if defined(CONFIG_SMP) && defined(MODULE) 19#define ARCH_NEEDS_WEAK_PER_CPU 20#endif 21 22/* 23 * We use a compare-and-swap loop since that uses less cpu cycles than 24 * disabling and enabling interrupts like the generic variant would do. 25 */ 26#define arch_this_cpu_to_op_simple(pcp, val, op) \ 27({ \ 28 typedef typeof(pcp) pcp_op_T__; \ 29 pcp_op_T__ old__, new__, prev__; \ 30 pcp_op_T__ *ptr__; \ 31 preempt_disable(); \ 32 ptr__ = raw_cpu_ptr(&(pcp)); \ 33 prev__ = *ptr__; \ 34 do { \ 35 old__ = prev__; \ 36 new__ = old__ op (val); \ 37 prev__ = cmpxchg(ptr__, old__, new__); \ 38 } while (prev__ != old__); \ 39 preempt_enable(); \ 40 new__; \ 41}) 42 43#define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 44#define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 45#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 46#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 47#define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 48#define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 49#define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 50#define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 51 52#ifndef CONFIG_HAVE_MARCH_Z196_FEATURES 53 54#define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 55#define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 56#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 57#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 58#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 59#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 60#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 61#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 62 63#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 64 65#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ 66{ \ 67 typedef typeof(pcp) pcp_op_T__; \ 68 pcp_op_T__ val__ = (val); \ 69 pcp_op_T__ old__, *ptr__; \ 70 preempt_disable(); \ 71 ptr__ = raw_cpu_ptr(&(pcp)); \ 72 if (__builtin_constant_p(val__) && \ 73 ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ 74 asm volatile( \ 75 op2 " %[ptr__],%[val__]\n" \ 76 : [ptr__] "+Q" (*ptr__) \ 77 : [val__] "i" ((szcast)val__) \ 78 : "cc"); \ 79 } else { \ 80 asm volatile( \ 81 op1 " %[old__],%[val__],%[ptr__]\n" \ 82 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 83 : [val__] "d" (val__) \ 84 : "cc"); \ 85 } \ 86 preempt_enable(); \ 87} 88 89#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) 90#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) 91 92#define arch_this_cpu_add_return(pcp, val, op) \ 93({ \ 94 typedef typeof(pcp) pcp_op_T__; \ 95 pcp_op_T__ val__ = (val); \ 96 pcp_op_T__ old__, *ptr__; \ 97 preempt_disable(); \ 98 ptr__ = raw_cpu_ptr(&(pcp)); \ 99 asm volatile( \ 100 op " %[old__],%[val__],%[ptr__]\n" \ 101 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 102 : [val__] "d" (val__) \ 103 : "cc"); \ 104 preempt_enable(); \ 105 old__ + val__; \ 106}) 107 108#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa") 109#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") 110 111#define arch_this_cpu_to_op(pcp, val, op) \ 112{ \ 113 typedef typeof(pcp) pcp_op_T__; \ 114 pcp_op_T__ val__ = (val); \ 115 pcp_op_T__ old__, *ptr__; \ 116 preempt_disable(); \ 117 ptr__ = raw_cpu_ptr(&(pcp)); \ 118 asm volatile( \ 119 op " %[old__],%[val__],%[ptr__]\n" \ 120 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 121 : [val__] "d" (val__) \ 122 : "cc"); \ 123 preempt_enable(); \ 124} 125 126#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") 127#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") 128#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") 129#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") 130 131#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 132 133#define arch_this_cpu_cmpxchg(pcp, oval, nval) \ 134({ \ 135 typedef typeof(pcp) pcp_op_T__; \ 136 pcp_op_T__ ret__; \ 137 pcp_op_T__ *ptr__; \ 138 preempt_disable(); \ 139 ptr__ = raw_cpu_ptr(&(pcp)); \ 140 ret__ = cmpxchg(ptr__, oval, nval); \ 141 preempt_enable(); \ 142 ret__; \ 143}) 144 145#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 146#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 147#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 148#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 149 150#define arch_this_cpu_xchg(pcp, nval) \ 151({ \ 152 typeof(pcp) *ptr__; \ 153 typeof(pcp) ret__; \ 154 preempt_disable(); \ 155 ptr__ = raw_cpu_ptr(&(pcp)); \ 156 ret__ = xchg(ptr__, nval); \ 157 preempt_enable(); \ 158 ret__; \ 159}) 160 161#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) 162#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) 163#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) 164#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) 165 166#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \ 167({ \ 168 typeof(pcp1) o1__ = (o1), n1__ = (n1); \ 169 typeof(pcp2) o2__ = (o2), n2__ = (n2); \ 170 typeof(pcp1) *p1__; \ 171 typeof(pcp2) *p2__; \ 172 int ret__; \ 173 preempt_disable(); \ 174 p1__ = raw_cpu_ptr(&(pcp1)); \ 175 p2__ = raw_cpu_ptr(&(pcp2)); \ 176 ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__); \ 177 preempt_enable(); \ 178 ret__; \ 179}) 180 181#define this_cpu_cmpxchg_double_4 arch_this_cpu_cmpxchg_double 182#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double 183 184#include <asm-generic/percpu.h> 185 186#endif /* __ARCH_S390_PERCPU__ */ 187