235 	struct futex_pi_state *pi_state;  member
696 	struct futex_pi_state *pi_state;  in refill_pi_state_cache()  local
701 	pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);  in refill_pi_state_cache()
703 	if (!pi_state)  in refill_pi_state_cache()
706 	INIT_LIST_HEAD(&pi_state->list);  in refill_pi_state_cache()
708 	pi_state->owner = NULL;  in refill_pi_state_cache()
709 	atomic_set(&pi_state->refcount, 1);  in refill_pi_state_cache()
710 	pi_state->key = FUTEX_KEY_INIT;  in refill_pi_state_cache()
712 	current->pi_state_cache = pi_state;  in refill_pi_state_cache()
719 	struct futex_pi_state *pi_state = current->pi_state_cache;  in alloc_pi_state()  local
721 	WARN_ON(!pi_state);  in alloc_pi_state()
724 	return pi_state;  in alloc_pi_state()
730 static void free_pi_state(struct futex_pi_state *pi_state)  in free_pi_state()  argument
732 	if (!pi_state)  in free_pi_state()
735 	if (!atomic_dec_and_test(&pi_state->refcount))  in free_pi_state()
742 	if (pi_state->owner) {  in free_pi_state()
743 		raw_spin_lock_irq(&pi_state->owner->pi_lock);  in free_pi_state()
744 		list_del_init(&pi_state->list);  in free_pi_state()
745 		raw_spin_unlock_irq(&pi_state->owner->pi_lock);  in free_pi_state()
747 		rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner);  in free_pi_state()
751 		kfree(pi_state);  in free_pi_state()
758 		pi_state->owner = NULL;  in free_pi_state()
759 		atomic_set(&pi_state->refcount, 1);  in free_pi_state()
760 		current->pi_state_cache = pi_state;  in free_pi_state()
790 	struct futex_pi_state *pi_state;  in exit_pi_state_list()  local
805 		pi_state = list_entry(next, struct futex_pi_state, list);  in exit_pi_state_list()
806 		key = pi_state->key;  in exit_pi_state_list()
822 		WARN_ON(pi_state->owner != curr);  in exit_pi_state_list()
823 		WARN_ON(list_empty(&pi_state->list));  in exit_pi_state_list()
824 		list_del_init(&pi_state->list);  in exit_pi_state_list()
825 		pi_state->owner = NULL;  in exit_pi_state_list()
828 		rt_mutex_unlock(&pi_state->pi_mutex);  in exit_pi_state_list()
892 static int attach_to_pi_state(u32 uval, struct futex_pi_state *pi_state,  in attach_to_pi_state()  argument
900 	if (unlikely(!pi_state))  in attach_to_pi_state()
903 	WARN_ON(!atomic_read(&pi_state->refcount));  in attach_to_pi_state()
914 		if (!pi_state->owner) {  in attach_to_pi_state()
942 		if (!pi_state->owner)  in attach_to_pi_state()
951 	if (pid != task_pid_vnr(pi_state->owner))  in attach_to_pi_state()
954 	atomic_inc(&pi_state->refcount);  in attach_to_pi_state()
955 	*ps = pi_state;  in attach_to_pi_state()
967 	struct futex_pi_state *pi_state;  in attach_to_pi_owner()  local
1008 	pi_state = alloc_pi_state();  in attach_to_pi_owner()
1014 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);  in attach_to_pi_owner()
1017 	pi_state->key = *key;  in attach_to_pi_owner()
1019 	WARN_ON(!list_empty(&pi_state->list));  in attach_to_pi_owner()
1020 	list_add(&pi_state->list, &p->pi_state_list);  in attach_to_pi_owner()
1021 	pi_state->owner = p;  in attach_to_pi_owner()
1026 	*ps = pi_state;  in attach_to_pi_owner()
1041 		return attach_to_pi_state(uval, match->pi_state, ps);  in lookup_pi_state()
1116 		return attach_to_pi_state(uval, match->pi_state, ps);  in futex_lock_pi_atomic()
1187 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in mark_wake_futex()
1210 	struct futex_pi_state *pi_state = this->pi_state;  in wake_futex_pi()  local
1216 	if (!pi_state)  in wake_futex_pi()
1223 	if (pi_state->owner != current)  in wake_futex_pi()
1226 	raw_spin_lock(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1227 	new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);  in wake_futex_pi()
1262 		raw_spin_unlock(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1266 	raw_spin_lock_irq(&pi_state->owner->pi_lock);  in wake_futex_pi()
1267 	WARN_ON(list_empty(&pi_state->list));  in wake_futex_pi()
1268 	list_del_init(&pi_state->list);  in wake_futex_pi()
1269 	raw_spin_unlock_irq(&pi_state->owner->pi_lock);  in wake_futex_pi()
1272 	WARN_ON(!list_empty(&pi_state->list));  in wake_futex_pi()
1273 	list_add(&pi_state->list, &new_owner->pi_state_list);  in wake_futex_pi()
1274 	pi_state->owner = new_owner;  in wake_futex_pi()
1277 	raw_spin_unlock(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1279 	deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);  in wake_futex_pi()
1348 			if (this->pi_state || this->rt_waiter) {  in futex_wake()
1431 			if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1445 				if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1619 	struct futex_pi_state *pi_state = NULL;  in futex_requeue()  local
1711 						 &key2, &pi_state, nr_requeue);  in futex_requeue()
1721 			WARN_ON(pi_state);  in futex_requeue()
1735 			ret = lookup_pi_state(ret, hb2, &key2, &pi_state);  in futex_requeue()
1742 			free_pi_state(pi_state);  in futex_requeue()
1743 			pi_state = NULL;  in futex_requeue()
1759 			free_pi_state(pi_state);  in futex_requeue()
1760 			pi_state = NULL;  in futex_requeue()
1788 		    this->pi_state) {  in futex_requeue()
1815 			atomic_inc(&pi_state->refcount);  in futex_requeue()
1816 			this->pi_state = pi_state;  in futex_requeue()
1817 			ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,  in futex_requeue()
1827 				this->pi_state = NULL;  in futex_requeue()
1828 				free_pi_state(pi_state);  in futex_requeue()
1837 	free_pi_state(pi_state);  in futex_requeue()
1965 		BUG_ON(q->pi_state);  in unqueue_me()
1985 	BUG_ON(!q->pi_state);  in unqueue_me_pi()
1986 	free_pi_state(q->pi_state);  in unqueue_me_pi()
1987 	q->pi_state = NULL;  in unqueue_me_pi()
2002 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()  local
2003 	struct task_struct *oldowner = pi_state->owner;  in fixup_pi_state_owner()
2008 	if (!pi_state->owner)  in fixup_pi_state_owner()
2046 	if (pi_state->owner != NULL) {  in fixup_pi_state_owner()
2047 		raw_spin_lock_irq(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
2048 		WARN_ON(list_empty(&pi_state->list));  in fixup_pi_state_owner()
2049 		list_del_init(&pi_state->list);  in fixup_pi_state_owner()
2050 		raw_spin_unlock_irq(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
2053 	pi_state->owner = newowner;  in fixup_pi_state_owner()
2056 	WARN_ON(!list_empty(&pi_state->list));  in fixup_pi_state_owner()
2057 	list_add(&pi_state->list, &newowner->pi_state_list);  in fixup_pi_state_owner()
2081 	if (pi_state->owner != oldowner)  in fixup_pi_state_owner()
2117 		if (q->pi_state->owner != current)  in fixup_owner()
2126 	if (q->pi_state->owner == current) {  in fixup_owner()
2132 		if (rt_mutex_trylock(&q->pi_state->pi_mutex)) {  in fixup_owner()
2142 		raw_spin_lock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2143 		owner = rt_mutex_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2145 			owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2146 		raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2155 	if (rt_mutex_owner(&q->pi_state->pi_mutex) == current)  in fixup_owner()
2158 				q->pi_state->pi_mutex.owner,  in fixup_owner()
2159 				q->pi_state->owner);  in fixup_owner()
2406 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);  in futex_lock_pi()
2440 	WARN_ON(!q.pi_state);  in futex_lock_pi()
2445 		ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);  in futex_lock_pi()
2447 		ret = rt_mutex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2469 	if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))  in futex_lock_pi()
2470 		rt_mutex_unlock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2774 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
2781 			free_pi_state(q.pi_state);  in futex_wait_requeue_pi()
2790 		WARN_ON(!q.pi_state);  in futex_wait_requeue_pi()
2791 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()