131 static inline int arch_read_trylock_once(arch_rwlock_t *rw)  in arch_read_trylock_once()  argument
133 	unsigned int old = ACCESS_ONCE(rw->lock);  in arch_read_trylock_once()
135 		      _raw_compare_and_swap(&rw->lock, old, old + 1));  in arch_read_trylock_once()
138 static inline int arch_write_trylock_once(arch_rwlock_t *rw)  in arch_write_trylock_once()  argument
140 	unsigned int old = ACCESS_ONCE(rw->lock);  in arch_write_trylock_once()
142 		      _raw_compare_and_swap(&rw->lock, 0, 0x80000000));  in arch_write_trylock_once()
181 static inline void arch_read_lock(arch_rwlock_t *rw)  in arch_read_lock()  argument
185 	old = __RAW_LOCK(&rw->lock, 1, __RAW_OP_ADD);  in arch_read_lock()
187 		_raw_read_lock_wait(rw);  in arch_read_lock()
190 static inline void arch_read_unlock(arch_rwlock_t *rw)  in arch_read_unlock()  argument
192 	__RAW_UNLOCK(&rw->lock, -1, __RAW_OP_ADD);  in arch_read_unlock()
195 static inline void arch_write_lock(arch_rwlock_t *rw)  in arch_write_lock()  argument
199 	old = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);  in arch_write_lock()
201 		_raw_write_lock_wait(rw, old);  in arch_write_lock()
202 	rw->owner = SPINLOCK_LOCKVAL;  in arch_write_lock()
205 static inline void arch_write_unlock(arch_rwlock_t *rw)  in arch_write_unlock()  argument
207 	rw->owner = 0;  in arch_write_unlock()
208 	__RAW_UNLOCK(&rw->lock, 0x7fffffff, __RAW_OP_AND);  in arch_write_unlock()
216 static inline void arch_read_lock(arch_rwlock_t *rw)  in arch_read_lock()  argument
218 	if (!arch_read_trylock_once(rw))  in arch_read_lock()
219 		_raw_read_lock_wait(rw);  in arch_read_lock()
222 static inline void arch_read_unlock(arch_rwlock_t *rw)  in arch_read_unlock()  argument
227 		old = ACCESS_ONCE(rw->lock);  in arch_read_unlock()
228 	} while (!_raw_compare_and_swap(&rw->lock, old, old - 1));  in arch_read_unlock()
231 static inline void arch_write_lock(arch_rwlock_t *rw)  in arch_write_lock()  argument
233 	if (!arch_write_trylock_once(rw))  in arch_write_lock()
234 		_raw_write_lock_wait(rw);  in arch_write_lock()
235 	rw->owner = SPINLOCK_LOCKVAL;  in arch_write_lock()
238 static inline void arch_write_unlock(arch_rwlock_t *rw)  in arch_write_unlock()  argument
240 	typecheck(unsigned int, rw->lock);  in arch_write_unlock()
242 	rw->owner = 0;  in arch_write_unlock()
245 		: "+Q" (rw->lock)  in arch_write_unlock()
252 static inline int arch_read_trylock(arch_rwlock_t *rw)  in arch_read_trylock()  argument
254 	if (!arch_read_trylock_once(rw))  in arch_read_trylock()
255 		return _raw_read_trylock_retry(rw);  in arch_read_trylock()
259 static inline int arch_write_trylock(arch_rwlock_t *rw)  in arch_write_trylock()  argument
261 	if (!arch_write_trylock_once(rw) && !_raw_write_trylock_retry(rw))  in arch_write_trylock()
263 	rw->owner = SPINLOCK_LOCKVAL;  in arch_write_trylock()
267 static inline void arch_read_relax(arch_rwlock_t *rw)  in arch_read_relax()  argument
269 	arch_lock_relax(rw->owner);  in arch_read_relax()
272 static inline void arch_write_relax(arch_rwlock_t *rw)  in arch_write_relax()  argument
274 	arch_lock_relax(rw->owner);  in arch_write_relax()