1/* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+ 2 * 3 * Written by David Howells (dhowells@redhat.com). 4 * 5 * Derived from asm-x86/semaphore.h 6 * 7 * 8 * The MSW of the count is the negated number of active writers and waiting 9 * lockers, and the LSW is the total number of active locks 10 * 11 * The lock count is initialized to 0 (no active and no waiting lockers). 12 * 13 * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an 14 * uncontended lock. This can be determined because XADD returns the old value. 15 * Readers increment by 1 and see a positive value when uncontended, negative 16 * if there are writers (and maybe) readers waiting (in which case it goes to 17 * sleep). 18 * 19 * The value of WAITING_BIAS supports up to 32766 waiting processes. This can 20 * be extended to 65534 by manually checking the whole MSW rather than relying 21 * on the S flag. 22 * 23 * The value of ACTIVE_BIAS supports up to 65535 active processes. 24 * 25 * This should be totally fair - if anything is waiting, a process that wants a 26 * lock will go to the back of the queue. When the currently active lock is 27 * released, if there's a writer at the front of the queue, then that and only 28 * that will be woken up; if there's a bunch of consequtive readers at the 29 * front, then they'll all be woken up, but no other readers will be. 30 */ 31 32#ifndef _ASM_X86_RWSEM_H 33#define _ASM_X86_RWSEM_H 34 35#ifndef _LINUX_RWSEM_H 36#error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" 37#endif 38 39#ifdef __KERNEL__ 40#include <asm/asm.h> 41 42/* 43 * The bias values and the counter type limits the number of 44 * potential readers/writers to 32767 for 32 bits and 2147483647 45 * for 64 bits. 46 */ 47 48#ifdef CONFIG_X86_64 49# define RWSEM_ACTIVE_MASK 0xffffffffL 50#else 51# define RWSEM_ACTIVE_MASK 0x0000ffffL 52#endif 53 54#define RWSEM_UNLOCKED_VALUE 0x00000000L 55#define RWSEM_ACTIVE_BIAS 0x00000001L 56#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) 57#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS 58#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) 59 60/* 61 * lock for reading 62 */ 63static inline void __down_read(struct rw_semaphore *sem) 64{ 65 asm volatile("# beginning down_read\n\t" 66 LOCK_PREFIX _ASM_INC "(%1)\n\t" 67 /* adds 0x00000001 */ 68 " jns 1f\n" 69 " call call_rwsem_down_read_failed\n" 70 "1:\n\t" 71 "# ending down_read\n\t" 72 : "+m" (sem->count) 73 : "a" (sem) 74 : "memory", "cc"); 75} 76 77/* 78 * trylock for reading -- returns 1 if successful, 0 if contention 79 */ 80static inline int __down_read_trylock(struct rw_semaphore *sem) 81{ 82 long result, tmp; 83 asm volatile("# beginning __down_read_trylock\n\t" 84 " mov %0,%1\n\t" 85 "1:\n\t" 86 " mov %1,%2\n\t" 87 " add %3,%2\n\t" 88 " jle 2f\n\t" 89 LOCK_PREFIX " cmpxchg %2,%0\n\t" 90 " jnz 1b\n\t" 91 "2:\n\t" 92 "# ending __down_read_trylock\n\t" 93 : "+m" (sem->count), "=&a" (result), "=&r" (tmp) 94 : "i" (RWSEM_ACTIVE_READ_BIAS) 95 : "memory", "cc"); 96 return result >= 0 ? 1 : 0; 97} 98 99/* 100 * lock for writing 101 */ 102static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) 103{ 104 long tmp; 105 asm volatile("# beginning down_write\n\t" 106 LOCK_PREFIX " xadd %1,(%2)\n\t" 107 /* adds 0xffff0001, returns the old value */ 108 " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" 109 /* was the active mask 0 before? */ 110 " jz 1f\n" 111 " call call_rwsem_down_write_failed\n" 112 "1:\n" 113 "# ending down_write" 114 : "+m" (sem->count), "=d" (tmp) 115 : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS) 116 : "memory", "cc"); 117} 118 119static inline void __down_write(struct rw_semaphore *sem) 120{ 121 __down_write_nested(sem, 0); 122} 123 124/* 125 * trylock for writing -- returns 1 if successful, 0 if contention 126 */ 127static inline int __down_write_trylock(struct rw_semaphore *sem) 128{ 129 long result, tmp; 130 asm volatile("# beginning __down_write_trylock\n\t" 131 " mov %0,%1\n\t" 132 "1:\n\t" 133 " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" 134 /* was the active mask 0 before? */ 135 " jnz 2f\n\t" 136 " mov %1,%2\n\t" 137 " add %3,%2\n\t" 138 LOCK_PREFIX " cmpxchg %2,%0\n\t" 139 " jnz 1b\n\t" 140 "2:\n\t" 141 " sete %b1\n\t" 142 " movzbl %b1, %k1\n\t" 143 "# ending __down_write_trylock\n\t" 144 : "+m" (sem->count), "=&a" (result), "=&r" (tmp) 145 : "er" (RWSEM_ACTIVE_WRITE_BIAS) 146 : "memory", "cc"); 147 return result; 148} 149 150/* 151 * unlock after reading 152 */ 153static inline void __up_read(struct rw_semaphore *sem) 154{ 155 long tmp; 156 asm volatile("# beginning __up_read\n\t" 157 LOCK_PREFIX " xadd %1,(%2)\n\t" 158 /* subtracts 1, returns the old value */ 159 " jns 1f\n\t" 160 " call call_rwsem_wake\n" /* expects old value in %edx */ 161 "1:\n" 162 "# ending __up_read\n" 163 : "+m" (sem->count), "=d" (tmp) 164 : "a" (sem), "1" (-RWSEM_ACTIVE_READ_BIAS) 165 : "memory", "cc"); 166} 167 168/* 169 * unlock after writing 170 */ 171static inline void __up_write(struct rw_semaphore *sem) 172{ 173 long tmp; 174 asm volatile("# beginning __up_write\n\t" 175 LOCK_PREFIX " xadd %1,(%2)\n\t" 176 /* subtracts 0xffff0001, returns the old value */ 177 " jns 1f\n\t" 178 " call call_rwsem_wake\n" /* expects old value in %edx */ 179 "1:\n\t" 180 "# ending __up_write\n" 181 : "+m" (sem->count), "=d" (tmp) 182 : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS) 183 : "memory", "cc"); 184} 185 186/* 187 * downgrade write lock to read lock 188 */ 189static inline void __downgrade_write(struct rw_semaphore *sem) 190{ 191 asm volatile("# beginning __downgrade_write\n\t" 192 LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t" 193 /* 194 * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386) 195 * 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64) 196 */ 197 " jns 1f\n\t" 198 " call call_rwsem_downgrade_wake\n" 199 "1:\n\t" 200 "# ending __downgrade_write\n" 201 : "+m" (sem->count) 202 : "a" (sem), "er" (-RWSEM_WAITING_BIAS) 203 : "memory", "cc"); 204} 205 206/* 207 * implement atomic add functionality 208 */ 209static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) 210{ 211 asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0" 212 : "+m" (sem->count) 213 : "er" (delta)); 214} 215 216/* 217 * implement exchange and add functionality 218 */ 219static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) 220{ 221 return delta + xadd(&sem->count, delta); 222} 223 224#endif /* __KERNEL__ */ 225#endif /* _ASM_X86_RWSEM_H */ 226