234 	struct futex_pi_state *pi_state;  member
619 	struct futex_pi_state *pi_state;  in refill_pi_state_cache()  local
624 	pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);  in refill_pi_state_cache()
626 	if (!pi_state)  in refill_pi_state_cache()
629 	INIT_LIST_HEAD(&pi_state->list);  in refill_pi_state_cache()
631 	pi_state->owner = NULL;  in refill_pi_state_cache()
632 	atomic_set(&pi_state->refcount, 1);  in refill_pi_state_cache()
633 	pi_state->key = FUTEX_KEY_INIT;  in refill_pi_state_cache()
635 	current->pi_state_cache = pi_state;  in refill_pi_state_cache()
642 	struct futex_pi_state *pi_state = current->pi_state_cache;  in alloc_pi_state()  local
644 	WARN_ON(!pi_state);  in alloc_pi_state()
647 	return pi_state;  in alloc_pi_state()
653 static void free_pi_state(struct futex_pi_state *pi_state)  in free_pi_state()  argument
655 	if (!pi_state)  in free_pi_state()
658 	if (!atomic_dec_and_test(&pi_state->refcount))  in free_pi_state()
665 	if (pi_state->owner) {  in free_pi_state()
666 		raw_spin_lock_irq(&pi_state->owner->pi_lock);  in free_pi_state()
667 		list_del_init(&pi_state->list);  in free_pi_state()
668 		raw_spin_unlock_irq(&pi_state->owner->pi_lock);  in free_pi_state()
670 		rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner);  in free_pi_state()
674 		kfree(pi_state);  in free_pi_state()
681 		pi_state->owner = NULL;  in free_pi_state()
682 		atomic_set(&pi_state->refcount, 1);  in free_pi_state()
683 		current->pi_state_cache = pi_state;  in free_pi_state()
713 	struct futex_pi_state *pi_state;  in exit_pi_state_list()  local
728 		pi_state = list_entry(next, struct futex_pi_state, list);  in exit_pi_state_list()
729 		key = pi_state->key;  in exit_pi_state_list()
745 		WARN_ON(pi_state->owner != curr);  in exit_pi_state_list()
746 		WARN_ON(list_empty(&pi_state->list));  in exit_pi_state_list()
747 		list_del_init(&pi_state->list);  in exit_pi_state_list()
748 		pi_state->owner = NULL;  in exit_pi_state_list()
751 		rt_mutex_unlock(&pi_state->pi_mutex);  in exit_pi_state_list()
815 static int attach_to_pi_state(u32 uval, struct futex_pi_state *pi_state,  in attach_to_pi_state()  argument
823 	if (unlikely(!pi_state))  in attach_to_pi_state()
826 	WARN_ON(!atomic_read(&pi_state->refcount));  in attach_to_pi_state()
837 		if (!pi_state->owner) {  in attach_to_pi_state()
865 		if (!pi_state->owner)  in attach_to_pi_state()
874 	if (pid != task_pid_vnr(pi_state->owner))  in attach_to_pi_state()
877 	atomic_inc(&pi_state->refcount);  in attach_to_pi_state()
878 	*ps = pi_state;  in attach_to_pi_state()
890 	struct futex_pi_state *pi_state;  in attach_to_pi_owner()  local
931 	pi_state = alloc_pi_state();  in attach_to_pi_owner()
937 	rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);  in attach_to_pi_owner()
940 	pi_state->key = *key;  in attach_to_pi_owner()
942 	WARN_ON(!list_empty(&pi_state->list));  in attach_to_pi_owner()
943 	list_add(&pi_state->list, &p->pi_state_list);  in attach_to_pi_owner()
944 	pi_state->owner = p;  in attach_to_pi_owner()
949 	*ps = pi_state;  in attach_to_pi_owner()
964 		return attach_to_pi_state(uval, match->pi_state, ps);  in lookup_pi_state()
1030 		return attach_to_pi_state(uval, match->pi_state, ps);  in futex_lock_pi_atomic()
1099 	if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))  in wake_futex()
1128 	struct futex_pi_state *pi_state = this->pi_state;  in wake_futex_pi()  local
1132 	if (!pi_state)  in wake_futex_pi()
1139 	if (pi_state->owner != current)  in wake_futex_pi()
1142 	raw_spin_lock(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1143 	new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);  in wake_futex_pi()
1175 		raw_spin_unlock(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1179 	raw_spin_lock_irq(&pi_state->owner->pi_lock);  in wake_futex_pi()
1180 	WARN_ON(list_empty(&pi_state->list));  in wake_futex_pi()
1181 	list_del_init(&pi_state->list);  in wake_futex_pi()
1182 	raw_spin_unlock_irq(&pi_state->owner->pi_lock);  in wake_futex_pi()
1185 	WARN_ON(!list_empty(&pi_state->list));  in wake_futex_pi()
1186 	list_add(&pi_state->list, &new_owner->pi_state_list);  in wake_futex_pi()
1187 	pi_state->owner = new_owner;  in wake_futex_pi()
1190 	raw_spin_unlock(&pi_state->pi_mutex.wait_lock);  in wake_futex_pi()
1191 	rt_mutex_unlock(&pi_state->pi_mutex);  in wake_futex_pi()
1248 			if (this->pi_state || this->rt_waiter) {  in futex_wake()
1329 			if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1343 				if (this->pi_state || this->rt_waiter) {  in futex_wake_op()
1513 	struct futex_pi_state *pi_state = NULL;  in futex_requeue()  local
1604 						 &key2, &pi_state, nr_requeue);  in futex_requeue()
1614 			WARN_ON(pi_state);  in futex_requeue()
1628 			ret = lookup_pi_state(ret, hb2, &key2, &pi_state);  in futex_requeue()
1635 			free_pi_state(pi_state);  in futex_requeue()
1636 			pi_state = NULL;  in futex_requeue()
1652 			free_pi_state(pi_state);  in futex_requeue()
1653 			pi_state = NULL;  in futex_requeue()
1681 		    this->pi_state) {  in futex_requeue()
1708 			atomic_inc(&pi_state->refcount);  in futex_requeue()
1709 			this->pi_state = pi_state;  in futex_requeue()
1710 			ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,  in futex_requeue()
1720 				this->pi_state = NULL;  in futex_requeue()
1721 				free_pi_state(pi_state);  in futex_requeue()
1730 	free_pi_state(pi_state);  in futex_requeue()
1857 		BUG_ON(q->pi_state);  in unqueue_me()
1877 	BUG_ON(!q->pi_state);  in unqueue_me_pi()
1878 	free_pi_state(q->pi_state);  in unqueue_me_pi()
1879 	q->pi_state = NULL;  in unqueue_me_pi()
1894 	struct futex_pi_state *pi_state = q->pi_state;  in fixup_pi_state_owner()  local
1895 	struct task_struct *oldowner = pi_state->owner;  in fixup_pi_state_owner()
1900 	if (!pi_state->owner)  in fixup_pi_state_owner()
1938 	if (pi_state->owner != NULL) {  in fixup_pi_state_owner()
1939 		raw_spin_lock_irq(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
1940 		WARN_ON(list_empty(&pi_state->list));  in fixup_pi_state_owner()
1941 		list_del_init(&pi_state->list);  in fixup_pi_state_owner()
1942 		raw_spin_unlock_irq(&pi_state->owner->pi_lock);  in fixup_pi_state_owner()
1945 	pi_state->owner = newowner;  in fixup_pi_state_owner()
1948 	WARN_ON(!list_empty(&pi_state->list));  in fixup_pi_state_owner()
1949 	list_add(&pi_state->list, &newowner->pi_state_list);  in fixup_pi_state_owner()
1973 	if (pi_state->owner != oldowner)  in fixup_pi_state_owner()
2009 		if (q->pi_state->owner != current)  in fixup_owner()
2018 	if (q->pi_state->owner == current) {  in fixup_owner()
2024 		if (rt_mutex_trylock(&q->pi_state->pi_mutex)) {  in fixup_owner()
2034 		raw_spin_lock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2035 		owner = rt_mutex_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2037 			owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);  in fixup_owner()
2038 		raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock);  in fixup_owner()
2047 	if (rt_mutex_owner(&q->pi_state->pi_mutex) == current)  in fixup_owner()
2050 				q->pi_state->pi_mutex.owner,  in fixup_owner()
2051 				q->pi_state->owner);  in fixup_owner()
2298 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, 0);  in futex_lock_pi()
2328 	WARN_ON(!q.pi_state);  in futex_lock_pi()
2333 		ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);  in futex_lock_pi()
2335 		ret = rt_mutex_trylock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2357 	if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))  in futex_lock_pi()
2358 		rt_mutex_unlock(&q.pi_state->pi_mutex);  in futex_lock_pi()
2651 		if (q.pi_state && (q.pi_state->owner != current)) {  in futex_wait_requeue_pi()
2658 			free_pi_state(q.pi_state);  in futex_wait_requeue_pi()
2667 		WARN_ON(!q.pi_state);  in futex_wait_requeue_pi()
2668 		pi_mutex = &q.pi_state->pi_mutex;  in futex_wait_requeue_pi()