132 static inline int arch_read_trylock_once(arch_rwlock_t *rw)  in arch_read_trylock_once()  argument
134 	unsigned int old = ACCESS_ONCE(rw->lock);  in arch_read_trylock_once()
136 		      _raw_compare_and_swap(&rw->lock, old, old + 1));  in arch_read_trylock_once()
139 static inline int arch_write_trylock_once(arch_rwlock_t *rw)  in arch_write_trylock_once()  argument
141 	unsigned int old = ACCESS_ONCE(rw->lock);  in arch_write_trylock_once()
143 		      _raw_compare_and_swap(&rw->lock, 0, 0x80000000));  in arch_write_trylock_once()
183 static inline void arch_read_lock(arch_rwlock_t *rw)  in arch_read_lock()  argument
187 	old = __RAW_LOCK(&rw->lock, 1, __RAW_OP_ADD);  in arch_read_lock()
189 		_raw_read_lock_wait(rw);  in arch_read_lock()
192 static inline void arch_read_unlock(arch_rwlock_t *rw)  in arch_read_unlock()  argument
194 	__RAW_UNLOCK(&rw->lock, -1, __RAW_OP_ADD);  in arch_read_unlock()
197 static inline void arch_write_lock(arch_rwlock_t *rw)  in arch_write_lock()  argument
201 	old = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);  in arch_write_lock()
203 		_raw_write_lock_wait(rw, old);  in arch_write_lock()
204 	rw->owner = SPINLOCK_LOCKVAL;  in arch_write_lock()
207 static inline void arch_write_unlock(arch_rwlock_t *rw)  in arch_write_unlock()  argument
209 	rw->owner = 0;  in arch_write_unlock()
210 	__RAW_UNLOCK(&rw->lock, 0x7fffffff, __RAW_OP_AND);  in arch_write_unlock()
218 static inline void arch_read_lock(arch_rwlock_t *rw)  in arch_read_lock()  argument
220 	if (!arch_read_trylock_once(rw))  in arch_read_lock()
221 		_raw_read_lock_wait(rw);  in arch_read_lock()
224 static inline void arch_read_unlock(arch_rwlock_t *rw)  in arch_read_unlock()  argument
229 		old = ACCESS_ONCE(rw->lock);  in arch_read_unlock()
230 	} while (!_raw_compare_and_swap(&rw->lock, old, old - 1));  in arch_read_unlock()
233 static inline void arch_write_lock(arch_rwlock_t *rw)  in arch_write_lock()  argument
235 	if (!arch_write_trylock_once(rw))  in arch_write_lock()
236 		_raw_write_lock_wait(rw);  in arch_write_lock()
237 	rw->owner = SPINLOCK_LOCKVAL;  in arch_write_lock()
240 static inline void arch_write_unlock(arch_rwlock_t *rw)  in arch_write_unlock()  argument
242 	typecheck(unsigned int, rw->lock);  in arch_write_unlock()
244 	rw->owner = 0;  in arch_write_unlock()
248 		: "+Q" (rw->lock)  in arch_write_unlock()
255 static inline int arch_read_trylock(arch_rwlock_t *rw)  in arch_read_trylock()  argument
257 	if (!arch_read_trylock_once(rw))  in arch_read_trylock()
258 		return _raw_read_trylock_retry(rw);  in arch_read_trylock()
262 static inline int arch_write_trylock(arch_rwlock_t *rw)  in arch_write_trylock()  argument
264 	if (!arch_write_trylock_once(rw) && !_raw_write_trylock_retry(rw))  in arch_write_trylock()
266 	rw->owner = SPINLOCK_LOCKVAL;  in arch_write_trylock()
270 static inline void arch_read_relax(arch_rwlock_t *rw)  in arch_read_relax()  argument
272 	arch_lock_relax(rw->owner);  in arch_read_relax()
275 static inline void arch_write_relax(arch_rwlock_t *rw)  in arch_write_relax()  argument
277 	arch_lock_relax(rw->owner);  in arch_write_relax()