180 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
190 		if (rt_mutex_waiter_less(waiter, entry)) {  in rt_mutex_enqueue()
199 		lock->waiters_leftmost = &waiter->tree_entry;  in rt_mutex_enqueue()
201 	rb_link_node(&waiter->tree_entry, parent, link);  in rt_mutex_enqueue()
202 	rb_insert_color(&waiter->tree_entry, &lock->waiters);  in rt_mutex_enqueue()
206 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
208 	if (RB_EMPTY_NODE(&waiter->tree_entry))  in rt_mutex_dequeue()
211 	if (lock->waiters_leftmost == &waiter->tree_entry)  in rt_mutex_dequeue()
212 		lock->waiters_leftmost = rb_next(&waiter->tree_entry);  in rt_mutex_dequeue()
214 	rb_erase(&waiter->tree_entry, &lock->waiters);  in rt_mutex_dequeue()
215 	RB_CLEAR_NODE(&waiter->tree_entry);  in rt_mutex_dequeue()
219 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue_pi()  argument
229 		if (rt_mutex_waiter_less(waiter, entry)) {  in rt_mutex_enqueue_pi()
238 		task->pi_waiters_leftmost = &waiter->pi_tree_entry;  in rt_mutex_enqueue_pi()
240 	rb_link_node(&waiter->pi_tree_entry, parent, link);  in rt_mutex_enqueue_pi()
241 	rb_insert_color(&waiter->pi_tree_entry, &task->pi_waiters);  in rt_mutex_enqueue_pi()
245 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue_pi()  argument
247 	if (RB_EMPTY_NODE(&waiter->pi_tree_entry))  in rt_mutex_dequeue_pi()
250 	if (task->pi_waiters_leftmost == &waiter->pi_tree_entry)  in rt_mutex_dequeue_pi()
251 		task->pi_waiters_leftmost = rb_next(&waiter->pi_tree_entry);  in rt_mutex_dequeue_pi()
253 	rb_erase(&waiter->pi_tree_entry, &task->pi_waiters);  in rt_mutex_dequeue_pi()
254 	RB_CLEAR_NODE(&waiter->pi_tree_entry);  in rt_mutex_dequeue_pi()
338 static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,  in rt_mutex_cond_detect_deadlock()  argument
348 	return debug_rt_mutex_detect_deadlock(waiter, chwalk);  in rt_mutex_cond_detect_deadlock()
431 	struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;  in rt_mutex_adjust_prio_chain()  local
484 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_prio_chain()
495 	if (!waiter)  in rt_mutex_adjust_prio_chain()
514 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
546 	if (waiter->prio == task->prio) {  in rt_mutex_adjust_prio_chain()
556 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
641 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
642 	waiter->prio = task->prio;  in rt_mutex_adjust_prio_chain()
643 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
674 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
682 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
685 	} else if (prerequeue_top_waiter == waiter) {  in rt_mutex_adjust_prio_chain()
696 		rt_mutex_dequeue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
697 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
698 		rt_mutex_enqueue_pi(task, waiter);  in rt_mutex_adjust_prio_chain()
743 	if (!detect_deadlock && waiter != top_waiter)  in rt_mutex_adjust_prio_chain()
767 				struct rt_mutex_waiter *waiter)  in try_to_take_rt_mutex()  argument
801 	if (waiter) {  in try_to_take_rt_mutex()
806 		if (waiter != rt_mutex_top_waiter(lock))  in try_to_take_rt_mutex()
813 		rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
889 				   struct rt_mutex_waiter *waiter,  in task_blocks_on_rt_mutex()  argument
894 	struct rt_mutex_waiter *top_waiter = waiter;  in task_blocks_on_rt_mutex()
913 	waiter->task = task;  in task_blocks_on_rt_mutex()
914 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
915 	waiter->prio = task->prio;  in task_blocks_on_rt_mutex()
920 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
922 	task->pi_blocked_on = waiter;  in task_blocks_on_rt_mutex()
930 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
932 		rt_mutex_enqueue_pi(owner, waiter);  in task_blocks_on_rt_mutex()
937 	} else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {  in task_blocks_on_rt_mutex()
963 					 next_lock, waiter, task);  in task_blocks_on_rt_mutex()
979 	struct rt_mutex_waiter *waiter;  in mark_wakeup_next_waiter()  local
984 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
992 	rt_mutex_dequeue_pi(current, waiter);  in mark_wakeup_next_waiter()
1006 	wake_q_add(wake_q, waiter->task);  in mark_wakeup_next_waiter()
1016 			  struct rt_mutex_waiter *waiter)  in remove_waiter()  argument
1018 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1024 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1037 	rt_mutex_dequeue_pi(owner, waiter);  in remove_waiter()
1074 	struct rt_mutex_waiter *waiter;  in rt_mutex_adjust_pi()  local
1080 	waiter = task->pi_blocked_on;  in rt_mutex_adjust_pi()
1081 	if (!waiter || (waiter->prio == task->prio &&  in rt_mutex_adjust_pi()
1086 	next_lock = waiter->lock;  in rt_mutex_adjust_pi()
1109 		    struct rt_mutex_waiter *waiter)  in __rt_mutex_slowlock()  argument
1115 		if (try_to_take_rt_mutex(lock, current, waiter))  in __rt_mutex_slowlock()
1134 		debug_rt_mutex_print_deadlock(waiter);  in __rt_mutex_slowlock()
1174 	struct rt_mutex_waiter waiter;  in rt_mutex_slowlock()  local
1177 	debug_rt_mutex_init_waiter(&waiter);  in rt_mutex_slowlock()
1178 	RB_CLEAR_NODE(&waiter.pi_tree_entry);  in rt_mutex_slowlock()
1179 	RB_CLEAR_NODE(&waiter.tree_entry);  in rt_mutex_slowlock()
1195 	ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);  in rt_mutex_slowlock()
1199 		ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);  in rt_mutex_slowlock()
1204 			remove_waiter(lock, &waiter);  in rt_mutex_slowlock()
1205 		rt_mutex_handle_deadlock(ret, chwalk, &waiter);  in rt_mutex_slowlock()
1220 	debug_rt_mutex_free_waiter(&waiter);  in rt_mutex_slowlock()
1594 			      struct rt_mutex_waiter *waiter,  in rt_mutex_start_proxy_lock()  argument
1607 	ret = task_blocks_on_rt_mutex(lock, waiter, task,  in rt_mutex_start_proxy_lock()
1621 		remove_waiter(lock, waiter);  in rt_mutex_start_proxy_lock()
1625 	debug_rt_mutex_print_deadlock(waiter);  in rt_mutex_start_proxy_lock()
1667 			       struct rt_mutex_waiter *waiter)  in rt_mutex_finish_proxy_lock()  argument
1676 	ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);  in rt_mutex_finish_proxy_lock()
1679 		remove_waiter(lock, waiter);  in rt_mutex_finish_proxy_lock()