1078 static void __unqueue_futex(struct futex_q *q)  in __unqueue_futex()  argument
1082 	if (WARN_ON_SMP(!q->lock_ptr || !spin_is_locked(q->lock_ptr))  in __unqueue_futex()
1083 	    || WARN_ON(plist_node_empty(&q->list)))  in __unqueue_futex()
1086 	hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);  in __unqueue_futex()
1087 	plist_del(&q->list, &hb->chain);  in __unqueue_futex()
1095 static void wake_futex(struct futex_q *q)  in wake_futex()  argument
1097 	struct task_struct *p = q->task;  in wake_futex()
1099 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in wake_futex()
1111 	__unqueue_futex(q);  in wake_futex()
1119 	q->lock_ptr = NULL;  in wake_futex()
1373 void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,  in requeue_futex()  argument
1382 		plist_del(&q->list, &hb1->chain);  in requeue_futex()
1384 		plist_add(&q->list, &hb2->chain);  in requeue_futex()
1386 		q->lock_ptr = &hb2->lock;  in requeue_futex()
1389 	q->key = *key2;  in requeue_futex()
1407 void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,  in requeue_pi_wake_futex()  argument
1411 	q->key = *key;  in requeue_pi_wake_futex()
1413 	__unqueue_futex(q);  in requeue_pi_wake_futex()
1415 	WARN_ON(!q->rt_waiter);  in requeue_pi_wake_futex()
1416 	q->rt_waiter = NULL;  in requeue_pi_wake_futex()
1418 	q->lock_ptr = &hb->lock;  in requeue_pi_wake_futex()
1420 	wake_up_state(q->task, TASK_NORMAL);  in requeue_pi_wake_futex()
1752 static inline struct futex_hash_bucket *queue_lock(struct futex_q *q)  in queue_lock()  argument
1757 	hb = hash_futex(&q->key);  in queue_lock()
1769 	q->lock_ptr = &hb->lock;  in queue_lock()
1795 static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)  in queue_me()  argument
1810 	plist_node_init(&q->list, prio);  in queue_me()
1811 	plist_add(&q->list, &hb->chain);  in queue_me()
1812 	q->task = current;  in queue_me()
1827 static int unqueue_me(struct futex_q *q)  in unqueue_me()  argument
1834 	lock_ptr = q->lock_ptr;  in unqueue_me()
1851 		if (unlikely(lock_ptr != q->lock_ptr)) {  in unqueue_me()
1855 		__unqueue_futex(q);  in unqueue_me()
1857 		BUG_ON(q->pi_state);  in unqueue_me()
1863 	drop_futex_key_refs(&q->key);  in unqueue_me()
1872 static void unqueue_me_pi(struct futex_q *q)  in unqueue_me_pi()  argument
1873 	__releases(q->lock_ptr)  in unqueue_me_pi()
1875 	__unqueue_futex(q);  in unqueue_me_pi()
1877 	BUG_ON(!q->pi_state);  in unqueue_me_pi()
1878 	free_pi_state(q->pi_state);  in unqueue_me_pi()
1879 	q->pi_state = NULL;  in unqueue_me_pi()
1881 	spin_unlock(q->lock_ptr);  in unqueue_me_pi()
1890 static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,  in fixup_pi_state_owner()  argument
1894 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()
1964 	spin_unlock(q->lock_ptr);  in fixup_pi_state_owner()
1968 	spin_lock(q->lock_ptr);  in fixup_pi_state_owner()
1999 static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)  in fixup_owner()  argument
2009 		if (q->pi_state->owner != current)  in fixup_owner()
2010 			ret = fixup_pi_state_owner(uaddr, q, current);  in fixup_owner()
2018 	if (q->pi_state->owner == current) {  in fixup_owner()
2024 		if (rt_mutex_trylock(&q->pi_state->pi_mutex)) {  in fixup_owner()
2034 		raw_spin_lock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2035 		owner = rt_mutex_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2037 			owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2038 		raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2039 		ret = fixup_pi_state_owner(uaddr, q, owner);  in fixup_owner()
2047 	if (rt_mutex_owner(&q->pi_state->pi_mutex) == current)  in fixup_owner()
2050 				q->pi_state->pi_mutex.owner,  in fixup_owner()
2051 				q->pi_state->owner);  in fixup_owner()
2063 static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,  in futex_wait_queue_me()  argument
2073 	queue_me(q, hb);  in futex_wait_queue_me()
2086 	if (likely(!plist_node_empty(&q->list))) {  in futex_wait_queue_me()
2116 			   struct futex_q *q, struct futex_hash_bucket **hb)  in futex_wait_setup()  argument
2140 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, VERIFY_READ);  in futex_wait_setup()
2145 	*hb = queue_lock(q);  in futex_wait_setup()
2159 		put_futex_key(&q->key);  in futex_wait_setup()
2170 		put_futex_key(&q->key);  in futex_wait_setup()
2180 	struct futex_q q = futex_q_init;  in futex_wait()  local
2185 	q.bitset = bitset;  in futex_wait()
2203 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait()
2208 	futex_wait_queue_me(hb, &q, to);  in futex_wait()
2213 	if (!unqueue_me(&q))  in futex_wait()
2276 	struct futex_q q = futex_q_init;  in futex_lock_pi()  local
2291 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, VERIFY_WRITE);  in futex_lock_pi()
2296 	hb = queue_lock(&q);  in futex_lock_pi()
2298 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);  in futex_lock_pi()
2315 			put_futex_key(&q.key);  in futex_lock_pi()
2326 	queue_me(&q, hb);  in futex_lock_pi()
2328 	WARN_ON(!q.pi_state);  in futex_lock_pi()
2333 		ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);  in futex_lock_pi()
2335 		ret = rt_mutex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2340 	spin_lock(q.lock_ptr);  in futex_lock_pi()
2345 	res = fixup_owner(uaddr, &q, !ret);  in futex_lock_pi()
2357 	if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))  in futex_lock_pi()
2358 		rt_mutex_unlock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2361 	unqueue_me_pi(&q);  in futex_lock_pi()
2369 	put_futex_key(&q.key);  in futex_lock_pi()
2385 	put_futex_key(&q.key);  in futex_lock_pi()
2493 				   struct futex_q *q, union futex_key *key2,  in handle_early_requeue_pi_wakeup()  argument
2505 	if (!match_futex(&q->key, key2)) {  in handle_early_requeue_pi_wakeup()
2506 		WARN_ON(q->lock_ptr && (&hb->lock != q->lock_ptr));  in handle_early_requeue_pi_wakeup()
2511 		plist_del(&q->list, &hb->chain);  in handle_early_requeue_pi_wakeup()
2573 	struct futex_q q = futex_q_init;  in futex_wait_requeue_pi()  local
2605 	q.bitset = bitset;  in futex_wait_requeue_pi()
2606 	q.rt_waiter = &rt_waiter;  in futex_wait_requeue_pi()
2607 	q.requeue_pi_key = &key2;  in futex_wait_requeue_pi()
2613 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait_requeue_pi()
2621 	if (match_futex(&q.key, &key2)) {  in futex_wait_requeue_pi()
2628 	futex_wait_queue_me(hb, &q, to);  in futex_wait_requeue_pi()
2631 	ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);  in futex_wait_requeue_pi()
2646 	if (!q.rt_waiter) {  in futex_wait_requeue_pi()
2651 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
2652 			spin_lock(q.lock_ptr);  in futex_wait_requeue_pi()
2653 			ret = fixup_pi_state_owner(uaddr2, &q, current);  in futex_wait_requeue_pi()
2658 			free_pi_state(q.pi_state);  in futex_wait_requeue_pi()
2659 			spin_unlock(q.lock_ptr);  in futex_wait_requeue_pi()
2667 		WARN_ON(!q.pi_state);  in futex_wait_requeue_pi()
2668 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()
2672 		spin_lock(q.lock_ptr);  in futex_wait_requeue_pi()
2677 		res = fixup_owner(uaddr2, &q, !ret);  in futex_wait_requeue_pi()
2686 		unqueue_me_pi(&q);  in futex_wait_requeue_pi()
2708 	put_futex_key(&q.key);  in futex_wait_requeue_pi()