1/* 2 * Copyright (C) 2012 ARM Ltd. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16#ifndef __ASM_SPINLOCK_H 17#define __ASM_SPINLOCK_H 18 19#include <asm/spinlock_types.h> 20#include <asm/processor.h> 21 22/* 23 * Spinlock implementation. 24 * 25 * The memory barriers are implicit with the load-acquire and store-release 26 * instructions. 27 */ 28 29#define arch_spin_unlock_wait(lock) \ 30 do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) 31 32#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 33 34static inline void arch_spin_lock(arch_spinlock_t *lock) 35{ 36 unsigned int tmp; 37 arch_spinlock_t lockval, newval; 38 39 asm volatile( 40 /* Atomically increment the next ticket. */ 41" prfm pstl1strm, %3\n" 42"1: ldaxr %w0, %3\n" 43" add %w1, %w0, %w5\n" 44" stxr %w2, %w1, %3\n" 45" cbnz %w2, 1b\n" 46 /* Did we get the lock? */ 47" eor %w1, %w0, %w0, ror #16\n" 48" cbz %w1, 3f\n" 49 /* 50 * No: spin on the owner. Send a local event to avoid missing an 51 * unlock before the exclusive load. 52 */ 53" sevl\n" 54"2: wfe\n" 55" ldaxrh %w2, %4\n" 56" eor %w1, %w2, %w0, lsr #16\n" 57" cbnz %w1, 2b\n" 58 /* We got the lock. Critical section starts here. */ 59"3:" 60 : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 61 : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 62 : "memory"); 63} 64 65static inline int arch_spin_trylock(arch_spinlock_t *lock) 66{ 67 unsigned int tmp; 68 arch_spinlock_t lockval; 69 70 asm volatile( 71" prfm pstl1strm, %2\n" 72"1: ldaxr %w0, %2\n" 73" eor %w1, %w0, %w0, ror #16\n" 74" cbnz %w1, 2f\n" 75" add %w0, %w0, %3\n" 76" stxr %w1, %w0, %2\n" 77" cbnz %w1, 1b\n" 78"2:" 79 : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 80 : "I" (1 << TICKET_SHIFT) 81 : "memory"); 82 83 return !tmp; 84} 85 86static inline void arch_spin_unlock(arch_spinlock_t *lock) 87{ 88 asm volatile( 89" stlrh %w1, %0\n" 90 : "=Q" (lock->owner) 91 : "r" (lock->owner + 1) 92 : "memory"); 93} 94 95static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 96{ 97 return lock.owner == lock.next; 98} 99 100static inline int arch_spin_is_locked(arch_spinlock_t *lock) 101{ 102 return !arch_spin_value_unlocked(READ_ONCE(*lock)); 103} 104 105static inline int arch_spin_is_contended(arch_spinlock_t *lock) 106{ 107 arch_spinlock_t lockval = READ_ONCE(*lock); 108 return (lockval.next - lockval.owner) > 1; 109} 110#define arch_spin_is_contended arch_spin_is_contended 111 112/* 113 * Write lock implementation. 114 * 115 * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 116 * exclusively held. 117 * 118 * The memory barriers are implicit with the load-acquire and store-release 119 * instructions. 120 */ 121 122static inline void arch_write_lock(arch_rwlock_t *rw) 123{ 124 unsigned int tmp; 125 126 asm volatile( 127 " sevl\n" 128 "1: wfe\n" 129 "2: ldaxr %w0, %1\n" 130 " cbnz %w0, 1b\n" 131 " stxr %w0, %w2, %1\n" 132 " cbnz %w0, 2b\n" 133 : "=&r" (tmp), "+Q" (rw->lock) 134 : "r" (0x80000000) 135 : "memory"); 136} 137 138static inline int arch_write_trylock(arch_rwlock_t *rw) 139{ 140 unsigned int tmp; 141 142 asm volatile( 143 " ldaxr %w0, %1\n" 144 " cbnz %w0, 1f\n" 145 " stxr %w0, %w2, %1\n" 146 "1:\n" 147 : "=&r" (tmp), "+Q" (rw->lock) 148 : "r" (0x80000000) 149 : "memory"); 150 151 return !tmp; 152} 153 154static inline void arch_write_unlock(arch_rwlock_t *rw) 155{ 156 asm volatile( 157 " stlr %w1, %0\n" 158 : "=Q" (rw->lock) : "r" (0) : "memory"); 159} 160 161/* write_can_lock - would write_trylock() succeed? */ 162#define arch_write_can_lock(x) ((x)->lock == 0) 163 164/* 165 * Read lock implementation. 166 * 167 * It exclusively loads the lock value, increments it and stores the new value 168 * back if positive and the CPU still exclusively owns the location. If the 169 * value is negative, the lock is already held. 170 * 171 * During unlocking there may be multiple active read locks but no write lock. 172 * 173 * The memory barriers are implicit with the load-acquire and store-release 174 * instructions. 175 */ 176static inline void arch_read_lock(arch_rwlock_t *rw) 177{ 178 unsigned int tmp, tmp2; 179 180 asm volatile( 181 " sevl\n" 182 "1: wfe\n" 183 "2: ldaxr %w0, %2\n" 184 " add %w0, %w0, #1\n" 185 " tbnz %w0, #31, 1b\n" 186 " stxr %w1, %w0, %2\n" 187 " cbnz %w1, 2b\n" 188 : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 189 : 190 : "memory"); 191} 192 193static inline void arch_read_unlock(arch_rwlock_t *rw) 194{ 195 unsigned int tmp, tmp2; 196 197 asm volatile( 198 "1: ldxr %w0, %2\n" 199 " sub %w0, %w0, #1\n" 200 " stlxr %w1, %w0, %2\n" 201 " cbnz %w1, 1b\n" 202 : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 203 : 204 : "memory"); 205} 206 207static inline int arch_read_trylock(arch_rwlock_t *rw) 208{ 209 unsigned int tmp, tmp2 = 1; 210 211 asm volatile( 212 " ldaxr %w0, %2\n" 213 " add %w0, %w0, #1\n" 214 " tbnz %w0, #31, 1f\n" 215 " stxr %w1, %w0, %2\n" 216 "1:\n" 217 : "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock) 218 : 219 : "memory"); 220 221 return !tmp2; 222} 223 224/* read_can_lock - would read_trylock() succeed? */ 225#define arch_read_can_lock(x) ((x)->lock < 0x80000000) 226 227#define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 228#define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 229 230#define arch_spin_relax(lock) cpu_relax() 231#define arch_read_relax(lock) cpu_relax() 232#define arch_write_relax(lock) cpu_relax() 233 234#endif /* __ASM_SPINLOCK_H */ 235