1/* futex.c: futex operations 2 * 3 * Copyright (C) 2005 Red Hat, Inc. All Rights Reserved. 4 * Written by David Howells (dhowells@redhat.com) 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 9 * 2 of the License, or (at your option) any later version. 10 */ 11 12#include <linux/futex.h> 13#include <linux/uaccess.h> 14#include <asm/futex.h> 15#include <asm/errno.h> 16 17/* 18 * the various futex operations; MMU fault checking is ignored under no-MMU 19 * conditions 20 */ 21static inline int atomic_futex_op_xchg_set(int oparg, u32 __user *uaddr, int *_oldval) 22{ 23 int oldval, ret; 24 25 asm("0: \n" 26 " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ 27 " ckeq icc3,cc7 \n" 28 "1: ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ 29 " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ 30 "2: cst.p %3,%M0 ,cc3,#1 \n" 31 " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ 32 " beq icc3,#0,0b \n" 33 " setlos 0,%2 \n" 34 "3: \n" 35 ".subsection 2 \n" 36 "4: setlos %5,%2 \n" 37 " bra 3b \n" 38 ".previous \n" 39 ".section __ex_table,\"a\" \n" 40 " .balign 8 \n" 41 " .long 1b,4b \n" 42 " .long 2b,4b \n" 43 ".previous" 44 : "+U"(*uaddr), "=&r"(oldval), "=&r"(ret), "=r"(oparg) 45 : "3"(oparg), "i"(-EFAULT) 46 : "memory", "cc7", "cc3", "icc3" 47 ); 48 49 *_oldval = oldval; 50 return ret; 51} 52 53static inline int atomic_futex_op_xchg_add(int oparg, u32 __user *uaddr, int *_oldval) 54{ 55 int oldval, ret; 56 57 asm("0: \n" 58 " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ 59 " ckeq icc3,cc7 \n" 60 "1: ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ 61 " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ 62 " add %1,%3,%3 \n" 63 "2: cst.p %3,%M0 ,cc3,#1 \n" 64 " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ 65 " beq icc3,#0,0b \n" 66 " setlos 0,%2 \n" 67 "3: \n" 68 ".subsection 2 \n" 69 "4: setlos %5,%2 \n" 70 " bra 3b \n" 71 ".previous \n" 72 ".section __ex_table,\"a\" \n" 73 " .balign 8 \n" 74 " .long 1b,4b \n" 75 " .long 2b,4b \n" 76 ".previous" 77 : "+U"(*uaddr), "=&r"(oldval), "=&r"(ret), "=r"(oparg) 78 : "3"(oparg), "i"(-EFAULT) 79 : "memory", "cc7", "cc3", "icc3" 80 ); 81 82 *_oldval = oldval; 83 return ret; 84} 85 86static inline int atomic_futex_op_xchg_or(int oparg, u32 __user *uaddr, int *_oldval) 87{ 88 int oldval, ret; 89 90 asm("0: \n" 91 " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ 92 " ckeq icc3,cc7 \n" 93 "1: ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ 94 " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ 95 " or %1,%3,%3 \n" 96 "2: cst.p %3,%M0 ,cc3,#1 \n" 97 " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ 98 " beq icc3,#0,0b \n" 99 " setlos 0,%2 \n" 100 "3: \n" 101 ".subsection 2 \n" 102 "4: setlos %5,%2 \n" 103 " bra 3b \n" 104 ".previous \n" 105 ".section __ex_table,\"a\" \n" 106 " .balign 8 \n" 107 " .long 1b,4b \n" 108 " .long 2b,4b \n" 109 ".previous" 110 : "+U"(*uaddr), "=&r"(oldval), "=&r"(ret), "=r"(oparg) 111 : "3"(oparg), "i"(-EFAULT) 112 : "memory", "cc7", "cc3", "icc3" 113 ); 114 115 *_oldval = oldval; 116 return ret; 117} 118 119static inline int atomic_futex_op_xchg_and(int oparg, u32 __user *uaddr, int *_oldval) 120{ 121 int oldval, ret; 122 123 asm("0: \n" 124 " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ 125 " ckeq icc3,cc7 \n" 126 "1: ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ 127 " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ 128 " and %1,%3,%3 \n" 129 "2: cst.p %3,%M0 ,cc3,#1 \n" 130 " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ 131 " beq icc3,#0,0b \n" 132 " setlos 0,%2 \n" 133 "3: \n" 134 ".subsection 2 \n" 135 "4: setlos %5,%2 \n" 136 " bra 3b \n" 137 ".previous \n" 138 ".section __ex_table,\"a\" \n" 139 " .balign 8 \n" 140 " .long 1b,4b \n" 141 " .long 2b,4b \n" 142 ".previous" 143 : "+U"(*uaddr), "=&r"(oldval), "=&r"(ret), "=r"(oparg) 144 : "3"(oparg), "i"(-EFAULT) 145 : "memory", "cc7", "cc3", "icc3" 146 ); 147 148 *_oldval = oldval; 149 return ret; 150} 151 152static inline int atomic_futex_op_xchg_xor(int oparg, u32 __user *uaddr, int *_oldval) 153{ 154 int oldval, ret; 155 156 asm("0: \n" 157 " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ 158 " ckeq icc3,cc7 \n" 159 "1: ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ 160 " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ 161 " xor %1,%3,%3 \n" 162 "2: cst.p %3,%M0 ,cc3,#1 \n" 163 " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ 164 " beq icc3,#0,0b \n" 165 " setlos 0,%2 \n" 166 "3: \n" 167 ".subsection 2 \n" 168 "4: setlos %5,%2 \n" 169 " bra 3b \n" 170 ".previous \n" 171 ".section __ex_table,\"a\" \n" 172 " .balign 8 \n" 173 " .long 1b,4b \n" 174 " .long 2b,4b \n" 175 ".previous" 176 : "+U"(*uaddr), "=&r"(oldval), "=&r"(ret), "=r"(oparg) 177 : "3"(oparg), "i"(-EFAULT) 178 : "memory", "cc7", "cc3", "icc3" 179 ); 180 181 *_oldval = oldval; 182 return ret; 183} 184 185/*****************************************************************************/ 186/* 187 * do the futex operations 188 */ 189int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr) 190{ 191 int op = (encoded_op >> 28) & 7; 192 int cmp = (encoded_op >> 24) & 15; 193 int oparg = (encoded_op << 8) >> 20; 194 int cmparg = (encoded_op << 20) >> 20; 195 int oldval = 0, ret; 196 197 if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) 198 oparg = 1 << oparg; 199 200 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) 201 return -EFAULT; 202 203 pagefault_disable(); 204 205 switch (op) { 206 case FUTEX_OP_SET: 207 ret = atomic_futex_op_xchg_set(oparg, uaddr, &oldval); 208 break; 209 case FUTEX_OP_ADD: 210 ret = atomic_futex_op_xchg_add(oparg, uaddr, &oldval); 211 break; 212 case FUTEX_OP_OR: 213 ret = atomic_futex_op_xchg_or(oparg, uaddr, &oldval); 214 break; 215 case FUTEX_OP_ANDN: 216 ret = atomic_futex_op_xchg_and(~oparg, uaddr, &oldval); 217 break; 218 case FUTEX_OP_XOR: 219 ret = atomic_futex_op_xchg_xor(oparg, uaddr, &oldval); 220 break; 221 default: 222 ret = -ENOSYS; 223 break; 224 } 225 226 pagefault_enable(); 227 228 if (!ret) { 229 switch (cmp) { 230 case FUTEX_OP_CMP_EQ: ret = (oldval == cmparg); break; 231 case FUTEX_OP_CMP_NE: ret = (oldval != cmparg); break; 232 case FUTEX_OP_CMP_LT: ret = (oldval < cmparg); break; 233 case FUTEX_OP_CMP_GE: ret = (oldval >= cmparg); break; 234 case FUTEX_OP_CMP_LE: ret = (oldval <= cmparg); break; 235 case FUTEX_OP_CMP_GT: ret = (oldval > cmparg); break; 236 default: ret = -ENOSYS; break; 237 } 238 } 239 240 return ret; 241 242} /* end futex_atomic_op_inuser() */ 243