77 static inline long ldsem_atomic_update(long delta, struct ld_semaphore *sem)  in ldsem_atomic_update()  argument
79 	return atomic_long_add_return(delta, (atomic_long_t *)&sem->count);  in ldsem_atomic_update()
87 static inline int ldsem_cmpxchg(long *old, long new, struct ld_semaphore *sem)  in ldsem_cmpxchg()  argument
89 	long tmp = atomic_long_cmpxchg(&sem->count, *old, new);  in ldsem_cmpxchg()
102 void __init_ldsem(struct ld_semaphore *sem, const char *name,  in __init_ldsem()  argument
109 	debug_check_no_locks_freed((void *)sem, sizeof(*sem));  in __init_ldsem()
110 	lockdep_init_map(&sem->dep_map, name, key, 0);  in __init_ldsem()
112 	sem->count = LDSEM_UNLOCKED;  in __init_ldsem()
113 	sem->wait_readers = 0;  in __init_ldsem()
114 	raw_spin_lock_init(&sem->wait_lock);  in __init_ldsem()
115 	INIT_LIST_HEAD(&sem->read_wait);  in __init_ldsem()
116 	INIT_LIST_HEAD(&sem->write_wait);  in __init_ldsem()
119 static void __ldsem_wake_readers(struct ld_semaphore *sem)  in __ldsem_wake_readers()  argument
129 	adjust = sem->wait_readers * (LDSEM_ACTIVE_BIAS - LDSEM_WAIT_BIAS);  in __ldsem_wake_readers()
130 	count = ldsem_atomic_update(adjust, sem);  in __ldsem_wake_readers()
134 		if (ldsem_cmpxchg(&count, count - adjust, sem))  in __ldsem_wake_readers()
138 	list_for_each_entry_safe(waiter, next, &sem->read_wait, list) {  in __ldsem_wake_readers()
145 	INIT_LIST_HEAD(&sem->read_wait);  in __ldsem_wake_readers()
146 	sem->wait_readers = 0;  in __ldsem_wake_readers()
149 static inline int writer_trylock(struct ld_semaphore *sem)  in writer_trylock()  argument
154 	long count = ldsem_atomic_update(LDSEM_ACTIVE_BIAS, sem);  in writer_trylock()
158 		if (ldsem_cmpxchg(&count, count - LDSEM_ACTIVE_BIAS, sem))  in writer_trylock()
163 static void __ldsem_wake_writer(struct ld_semaphore *sem)  in __ldsem_wake_writer()  argument
167 	waiter = list_entry(sem->write_wait.next, struct ldsem_waiter, list);  in __ldsem_wake_writer()
179 static void __ldsem_wake(struct ld_semaphore *sem)  in __ldsem_wake()  argument
181 	if (!list_empty(&sem->write_wait))  in __ldsem_wake()
182 		__ldsem_wake_writer(sem);  in __ldsem_wake()
183 	else if (!list_empty(&sem->read_wait))  in __ldsem_wake()
184 		__ldsem_wake_readers(sem);  in __ldsem_wake()
187 static void ldsem_wake(struct ld_semaphore *sem)  in ldsem_wake()  argument
191 	raw_spin_lock_irqsave(&sem->wait_lock, flags);  in ldsem_wake()
192 	__ldsem_wake(sem);  in ldsem_wake()
193 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);  in ldsem_wake()
200 down_read_failed(struct ld_semaphore *sem, long count, long timeout)  in down_read_failed()  argument
207 	raw_spin_lock_irq(&sem->wait_lock);  in down_read_failed()
213 		if (ldsem_cmpxchg(&count, count + adjust, sem))  in down_read_failed()
216 			raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
217 			return sem;  in down_read_failed()
221 	list_add_tail(&waiter.list, &sem->read_wait);  in down_read_failed()
222 	sem->wait_readers++;  in down_read_failed()
229 		__ldsem_wake(sem);  in down_read_failed()
231 	raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
250 		raw_spin_lock_irq(&sem->wait_lock);  in down_read_failed()
252 			ldsem_atomic_update(-LDSEM_WAIT_BIAS, sem);  in down_read_failed()
254 			raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
258 		raw_spin_unlock_irq(&sem->wait_lock);  in down_read_failed()
261 	return sem;  in down_read_failed()
268 down_write_failed(struct ld_semaphore *sem, long count, long timeout)  in down_write_failed()  argument
276 	raw_spin_lock_irq(&sem->wait_lock);  in down_write_failed()
282 		if (ldsem_cmpxchg(&count, count + adjust, sem))  in down_write_failed()
285 			raw_spin_unlock_irq(&sem->wait_lock);  in down_write_failed()
286 			return sem;  in down_write_failed()
290 	list_add_tail(&waiter.list, &sem->write_wait);  in down_write_failed()
298 		raw_spin_unlock_irq(&sem->wait_lock);  in down_write_failed()
300 		raw_spin_lock_irq(&sem->wait_lock);  in down_write_failed()
302 		if ((locked = writer_trylock(sem)))  in down_write_failed()
307 		ldsem_atomic_update(-LDSEM_WAIT_BIAS, sem);  in down_write_failed()
309 	raw_spin_unlock_irq(&sem->wait_lock);  in down_write_failed()
316 	return sem;  in down_write_failed()
321 static inline int __ldsem_down_read_nested(struct ld_semaphore *sem,  in __ldsem_down_read_nested()  argument
326 	lockdep_acquire_read(sem, subclass, 0, _RET_IP_);  in __ldsem_down_read_nested()
328 	count = ldsem_atomic_update(LDSEM_READ_BIAS, sem);  in __ldsem_down_read_nested()
330 		lock_stat(sem, contended);  in __ldsem_down_read_nested()
331 		if (!down_read_failed(sem, count, timeout)) {  in __ldsem_down_read_nested()
332 			lockdep_release(sem, 1, _RET_IP_);  in __ldsem_down_read_nested()
336 	lock_stat(sem, acquired);  in __ldsem_down_read_nested()
340 static inline int __ldsem_down_write_nested(struct ld_semaphore *sem,  in __ldsem_down_write_nested()  argument
345 	lockdep_acquire(sem, subclass, 0, _RET_IP_);  in __ldsem_down_write_nested()
347 	count = ldsem_atomic_update(LDSEM_WRITE_BIAS, sem);  in __ldsem_down_write_nested()
349 		lock_stat(sem, contended);  in __ldsem_down_write_nested()
350 		if (!down_write_failed(sem, count, timeout)) {  in __ldsem_down_write_nested()
351 			lockdep_release(sem, 1, _RET_IP_);  in __ldsem_down_write_nested()
355 	lock_stat(sem, acquired);  in __ldsem_down_write_nested()
363 int __sched ldsem_down_read(struct ld_semaphore *sem, long timeout)  in ldsem_down_read()  argument
366 	return __ldsem_down_read_nested(sem, 0, timeout);  in ldsem_down_read()
372 int ldsem_down_read_trylock(struct ld_semaphore *sem)  in ldsem_down_read_trylock()  argument
374 	long count = sem->count;  in ldsem_down_read_trylock()
377 		if (ldsem_cmpxchg(&count, count + LDSEM_READ_BIAS, sem)) {  in ldsem_down_read_trylock()
378 			lockdep_acquire_read(sem, 0, 1, _RET_IP_);  in ldsem_down_read_trylock()
379 			lock_stat(sem, acquired);  in ldsem_down_read_trylock()
389 int __sched ldsem_down_write(struct ld_semaphore *sem, long timeout)  in ldsem_down_write()  argument
392 	return __ldsem_down_write_nested(sem, 0, timeout);  in ldsem_down_write()
398 int ldsem_down_write_trylock(struct ld_semaphore *sem)  in ldsem_down_write_trylock()  argument
400 	long count = sem->count;  in ldsem_down_write_trylock()
403 		if (ldsem_cmpxchg(&count, count + LDSEM_WRITE_BIAS, sem)) {  in ldsem_down_write_trylock()
404 			lockdep_acquire(sem, 0, 1, _RET_IP_);  in ldsem_down_write_trylock()
405 			lock_stat(sem, acquired);  in ldsem_down_write_trylock()
415 void ldsem_up_read(struct ld_semaphore *sem)  in ldsem_up_read()  argument
419 	lockdep_release(sem, 1, _RET_IP_);  in ldsem_up_read()
421 	count = ldsem_atomic_update(-LDSEM_READ_BIAS, sem);  in ldsem_up_read()
423 		ldsem_wake(sem);  in ldsem_up_read()
429 void ldsem_up_write(struct ld_semaphore *sem)  in ldsem_up_write()  argument
433 	lockdep_release(sem, 1, _RET_IP_);  in ldsem_up_write()
435 	count = ldsem_atomic_update(-LDSEM_WRITE_BIAS, sem);  in ldsem_up_write()
437 		ldsem_wake(sem);  in ldsem_up_write()
443 int ldsem_down_read_nested(struct ld_semaphore *sem, int subclass, long timeout)  in ldsem_down_read_nested()  argument
446 	return __ldsem_down_read_nested(sem, subclass, timeout);  in ldsem_down_read_nested()
449 int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass,  in ldsem_down_write_nested()  argument
453 	return __ldsem_down_write_nested(sem, subclass, timeout);  in ldsem_down_write_nested()