50 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)  in rt_mutex_set_owner()  argument
54 	if (rt_mutex_has_waiters(lock))  in rt_mutex_set_owner()
57 	lock->owner = (struct task_struct *)val;  in rt_mutex_set_owner()
60 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)  in clear_rt_mutex_waiters()  argument
62 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
63 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
66 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)  in fixup_rt_mutex_waiters()  argument
68 	if (!rt_mutex_has_waiters(lock))  in fixup_rt_mutex_waiters()
69 		clear_rt_mutex_waiters(lock);  in fixup_rt_mutex_waiters()
86 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)  in mark_rt_mutex_waiters()  argument
88 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
102 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)  in unlock_rt_mutex_safe()  argument
103 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
105 	struct task_struct *owner = rt_mutex_owner(lock);  in unlock_rt_mutex_safe()
107 	clear_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
108 	raw_spin_unlock(&lock->wait_lock);  in unlock_rt_mutex_safe()
133 	return rt_mutex_cmpxchg_release(lock, owner, NULL);  in unlock_rt_mutex_safe()
141 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)  in mark_rt_mutex_waiters()  argument
143 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
144 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
150 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)  in unlock_rt_mutex_safe()  argument
151 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
153 	lock->owner = NULL;  in unlock_rt_mutex_safe()
154 	raw_spin_unlock(&lock->wait_lock);  in unlock_rt_mutex_safe()
180 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
182 	struct rb_node **link = &lock->waiters.rb_node;  in rt_mutex_enqueue()
199 		lock->waiters_leftmost = &waiter->tree_entry;  in rt_mutex_enqueue()
202 	rb_insert_color(&waiter->tree_entry, &lock->waiters);  in rt_mutex_enqueue()
206 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
211 	if (lock->waiters_leftmost == &waiter->tree_entry)  in rt_mutex_dequeue()
212 		lock->waiters_leftmost = rb_next(&waiter->tree_entry);  in rt_mutex_dequeue()
214 	rb_erase(&waiter->tree_entry, &lock->waiters);  in rt_mutex_dequeue()
358 	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;  in task_blocked_on_lock()
434 	struct rt_mutex *lock;  in rt_mutex_adjust_prio_chain()  local
514 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
556 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
562 	if (!raw_spin_trylock(&lock->wait_lock)) {  in rt_mutex_adjust_prio_chain()
577 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {  in rt_mutex_adjust_prio_chain()
578 		debug_rt_mutex_deadlock(chwalk, orig_waiter, lock);  in rt_mutex_adjust_prio_chain()
579 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
601 		if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
602 			raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
607 		task = rt_mutex_owner(lock);  in rt_mutex_adjust_prio_chain()
621 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
625 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
638 	prerequeue_top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
641 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
643 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
656 	if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
662 		if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))  in rt_mutex_adjust_prio_chain()
663 			wake_up_process(rt_mutex_top_waiter(lock)->task);  in rt_mutex_adjust_prio_chain()
664 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
669 	task = rt_mutex_owner(lock);  in rt_mutex_adjust_prio_chain()
674 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
697 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
722 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
726 	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
766 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
788 	mark_rt_mutex_waiters(lock);  in try_to_take_rt_mutex()
793 	if (rt_mutex_owner(lock))  in try_to_take_rt_mutex()
806 		if (waiter != rt_mutex_top_waiter(lock))  in try_to_take_rt_mutex()
813 		rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
824 		if (rt_mutex_has_waiters(lock)) {  in try_to_take_rt_mutex()
830 			if (task->prio >= rt_mutex_top_waiter(lock)->prio)  in try_to_take_rt_mutex()
862 	if (rt_mutex_has_waiters(lock))  in try_to_take_rt_mutex()
863 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
868 	debug_rt_mutex_lock(lock);  in try_to_take_rt_mutex()
874 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
876 	rt_mutex_deadlock_account_lock(lock, task);  in try_to_take_rt_mutex()
888 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,  in task_blocks_on_rt_mutex()  argument
893 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()
914 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
918 	if (rt_mutex_has_waiters(lock))  in task_blocks_on_rt_mutex()
919 		top_waiter = rt_mutex_top_waiter(lock);  in task_blocks_on_rt_mutex()
920 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
930 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
960 	raw_spin_unlock(&lock->wait_lock);  in task_blocks_on_rt_mutex()
962 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,  in task_blocks_on_rt_mutex()
965 	raw_spin_lock(&lock->wait_lock);  in task_blocks_on_rt_mutex()
977 				    struct rt_mutex *lock)  in mark_wakeup_next_waiter()  argument
984 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1002 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;  in mark_wakeup_next_waiter()
1015 static void remove_waiter(struct rt_mutex *lock,  in remove_waiter()  argument
1018 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1019 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()
1024 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1039 	if (rt_mutex_has_waiters(lock))  in remove_waiter()
1040 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));  in remove_waiter()
1059 	raw_spin_unlock(&lock->wait_lock);  in remove_waiter()
1061 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,  in remove_waiter()
1064 	raw_spin_lock(&lock->wait_lock);  in remove_waiter()
1086 	next_lock = waiter->lock;  in rt_mutex_adjust_pi()
1107 __rt_mutex_slowlock(struct rt_mutex *lock, int state,  in __rt_mutex_slowlock()  argument
1115 		if (try_to_take_rt_mutex(lock, current, waiter))  in __rt_mutex_slowlock()
1132 		raw_spin_unlock(&lock->wait_lock);  in __rt_mutex_slowlock()
1138 		raw_spin_lock(&lock->wait_lock);  in __rt_mutex_slowlock()
1170 rt_mutex_slowlock(struct rt_mutex *lock, int state,  in rt_mutex_slowlock()  argument
1181 	raw_spin_lock(&lock->wait_lock);  in rt_mutex_slowlock()
1184 	if (try_to_take_rt_mutex(lock, current, NULL)) {  in rt_mutex_slowlock()
1185 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_slowlock()
1195 	ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);  in rt_mutex_slowlock()
1199 		ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);  in rt_mutex_slowlock()
1203 		if (rt_mutex_has_waiters(lock))  in rt_mutex_slowlock()
1204 			remove_waiter(lock, &waiter);  in rt_mutex_slowlock()
1212 	fixup_rt_mutex_waiters(lock);  in rt_mutex_slowlock()
1214 	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_slowlock()
1228 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)  in rt_mutex_slowtrylock()  argument
1237 	if (rt_mutex_owner(lock))  in rt_mutex_slowtrylock()
1244 	raw_spin_lock(&lock->wait_lock);  in rt_mutex_slowtrylock()
1246 	ret = try_to_take_rt_mutex(lock, current, NULL);  in rt_mutex_slowtrylock()
1252 	fixup_rt_mutex_waiters(lock);  in rt_mutex_slowtrylock()
1254 	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_slowtrylock()
1263 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,  in rt_mutex_slowunlock()  argument
1266 	raw_spin_lock(&lock->wait_lock);  in rt_mutex_slowunlock()
1268 	debug_rt_mutex_unlock(lock);  in rt_mutex_slowunlock()
1303 	while (!rt_mutex_has_waiters(lock)) {  in rt_mutex_slowunlock()
1305 		if (unlock_rt_mutex_safe(lock) == true)  in rt_mutex_slowunlock()
1308 		raw_spin_lock(&lock->wait_lock);  in rt_mutex_slowunlock()
1317 	mark_wakeup_next_waiter(wake_q, lock);  in rt_mutex_slowunlock()
1319 	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_slowunlock()
1332 rt_mutex_fastlock(struct rt_mutex *lock, int state,  in rt_mutex_fastlock()  argument
1333 		  int (*slowfn)(struct rt_mutex *lock, int state,  in rt_mutex_fastlock()  argument
1337 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {  in rt_mutex_fastlock()
1338 		rt_mutex_deadlock_account_lock(lock, current);  in rt_mutex_fastlock()
1341 		return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);  in rt_mutex_fastlock()
1345 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,  in rt_mutex_timed_fastlock()  argument
1348 			int (*slowfn)(struct rt_mutex *lock, int state,  in rt_mutex_timed_fastlock()  argument
1353 	    likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {  in rt_mutex_timed_fastlock()
1354 		rt_mutex_deadlock_account_lock(lock, current);  in rt_mutex_timed_fastlock()
1357 		return slowfn(lock, state, timeout, chwalk);  in rt_mutex_timed_fastlock()
1361 rt_mutex_fasttrylock(struct rt_mutex *lock,  in rt_mutex_fasttrylock()  argument
1362 		     int (*slowfn)(struct rt_mutex *lock))  in rt_mutex_fasttrylock()  argument
1364 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) {  in rt_mutex_fasttrylock()
1365 		rt_mutex_deadlock_account_lock(lock, current);  in rt_mutex_fasttrylock()
1368 	return slowfn(lock);  in rt_mutex_fasttrylock()
1372 rt_mutex_fastunlock(struct rt_mutex *lock,  in rt_mutex_fastunlock()  argument
1373 		    bool (*slowfn)(struct rt_mutex *lock,  in rt_mutex_fastunlock()  argument
1378 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) {  in rt_mutex_fastunlock()
1382 		bool deboost = slowfn(lock, &wake_q);  in rt_mutex_fastunlock()
1397 void __sched rt_mutex_lock(struct rt_mutex *lock)  in rt_mutex_lock()  argument
1401 	rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);  in rt_mutex_lock()
1414 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)  in rt_mutex_lock_interruptible()  argument
1418 	return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);  in rt_mutex_lock_interruptible()
1425 int rt_mutex_timed_futex_lock(struct rt_mutex *lock,  in rt_mutex_timed_futex_lock()  argument
1430 	return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,  in rt_mutex_timed_futex_lock()
1449 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)  in rt_mutex_timed_lock()  argument
1453 	return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,  in rt_mutex_timed_lock()
1470 int __sched rt_mutex_trylock(struct rt_mutex *lock)  in rt_mutex_trylock()  argument
1475 	return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);  in rt_mutex_trylock()
1484 void __sched rt_mutex_unlock(struct rt_mutex *lock)  in rt_mutex_unlock()  argument
1486 	rt_mutex_fastunlock(lock, rt_mutex_slowunlock);  in rt_mutex_unlock()
1497 bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock,  in rt_mutex_futex_unlock()  argument
1500 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) {  in rt_mutex_futex_unlock()
1504 	return rt_mutex_slowunlock(lock, wqh);  in rt_mutex_futex_unlock()
1515 void rt_mutex_destroy(struct rt_mutex *lock)  in rt_mutex_destroy()  argument
1517 	WARN_ON(rt_mutex_is_locked(lock));  in rt_mutex_destroy()
1519 	lock->magic = NULL;  in rt_mutex_destroy()
1534 void __rt_mutex_init(struct rt_mutex *lock, const char *name)  in __rt_mutex_init()  argument
1536 	lock->owner = NULL;  in __rt_mutex_init()
1537 	raw_spin_lock_init(&lock->wait_lock);  in __rt_mutex_init()
1538 	lock->waiters = RB_ROOT;  in __rt_mutex_init()
1539 	lock->waiters_leftmost = NULL;  in __rt_mutex_init()
1541 	debug_rt_mutex_init(lock, name);  in __rt_mutex_init()
1555 void rt_mutex_init_proxy_locked(struct rt_mutex *lock,  in rt_mutex_init_proxy_locked()  argument
1558 	__rt_mutex_init(lock, NULL);  in rt_mutex_init_proxy_locked()
1559 	debug_rt_mutex_proxy_lock(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
1560 	rt_mutex_set_owner(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
1561 	rt_mutex_deadlock_account_lock(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
1572 void rt_mutex_proxy_unlock(struct rt_mutex *lock,  in rt_mutex_proxy_unlock()  argument
1575 	debug_rt_mutex_proxy_unlock(lock);  in rt_mutex_proxy_unlock()
1576 	rt_mutex_set_owner(lock, NULL);  in rt_mutex_proxy_unlock()
1593 int rt_mutex_start_proxy_lock(struct rt_mutex *lock,  in rt_mutex_start_proxy_lock()  argument
1599 	raw_spin_lock(&lock->wait_lock);  in rt_mutex_start_proxy_lock()
1601 	if (try_to_take_rt_mutex(lock, task, NULL)) {  in rt_mutex_start_proxy_lock()
1602 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_start_proxy_lock()
1607 	ret = task_blocks_on_rt_mutex(lock, waiter, task,  in rt_mutex_start_proxy_lock()
1610 	if (ret && !rt_mutex_owner(lock)) {  in rt_mutex_start_proxy_lock()
1621 		remove_waiter(lock, waiter);  in rt_mutex_start_proxy_lock()
1623 	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_start_proxy_lock()
1642 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)  in rt_mutex_next_owner()  argument
1644 	if (!rt_mutex_has_waiters(lock))  in rt_mutex_next_owner()
1647 	return rt_mutex_top_waiter(lock)->task;  in rt_mutex_next_owner()
1665 int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,  in rt_mutex_finish_proxy_lock()  argument
1671 	raw_spin_lock(&lock->wait_lock);  in rt_mutex_finish_proxy_lock()
1676 	ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);  in rt_mutex_finish_proxy_lock()
1679 		remove_waiter(lock, waiter);  in rt_mutex_finish_proxy_lock()
1685 	fixup_rt_mutex_waiters(lock);  in rt_mutex_finish_proxy_lock()
1687 	raw_spin_unlock(&lock->wait_lock);  in rt_mutex_finish_proxy_lock()