pi_state 236 kernel/futex.c struct futex_pi_state *pi_state; pi_state 829 kernel/futex.c struct futex_pi_state *pi_state; pi_state 834 kernel/futex.c pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL); pi_state 836 kernel/futex.c if (!pi_state) pi_state 839 kernel/futex.c INIT_LIST_HEAD(&pi_state->list); pi_state 841 kernel/futex.c pi_state->owner = NULL; pi_state 842 kernel/futex.c refcount_set(&pi_state->refcount, 1); pi_state 843 kernel/futex.c pi_state->key = FUTEX_KEY_INIT; pi_state 845 kernel/futex.c current->pi_state_cache = pi_state; pi_state 852 kernel/futex.c struct futex_pi_state *pi_state = current->pi_state_cache; pi_state 854 kernel/futex.c WARN_ON(!pi_state); pi_state 857 kernel/futex.c return pi_state; pi_state 860 kernel/futex.c static void get_pi_state(struct futex_pi_state *pi_state) pi_state 862 kernel/futex.c WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount)); pi_state 869 kernel/futex.c static void put_pi_state(struct futex_pi_state *pi_state) pi_state 871 kernel/futex.c if (!pi_state) pi_state 874 kernel/futex.c if (!refcount_dec_and_test(&pi_state->refcount)) pi_state 881 kernel/futex.c if (pi_state->owner) { pi_state 884 kernel/futex.c raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); pi_state 885 kernel/futex.c owner = pi_state->owner; pi_state 888 kernel/futex.c list_del_init(&pi_state->list); pi_state 891 kernel/futex.c rt_mutex_proxy_unlock(&pi_state->pi_mutex, owner); pi_state 892 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 896 kernel/futex.c kfree(pi_state); pi_state 903 kernel/futex.c pi_state->owner = NULL; pi_state 904 kernel/futex.c refcount_set(&pi_state->refcount, 1); pi_state 905 kernel/futex.c current->pi_state_cache = pi_state; pi_state 919 kernel/futex.c struct futex_pi_state *pi_state; pi_state 933 kernel/futex.c pi_state = list_entry(next, struct futex_pi_state, list); pi_state 934 kernel/futex.c key = pi_state->key; pi_state 947 kernel/futex.c if (!refcount_inc_not_zero(&pi_state->refcount)) { pi_state 956 kernel/futex.c raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); pi_state 964 kernel/futex.c raw_spin_unlock(&pi_state->pi_mutex.wait_lock); pi_state 966 kernel/futex.c put_pi_state(pi_state); pi_state 970 kernel/futex.c WARN_ON(pi_state->owner != curr); pi_state 971 kernel/futex.c WARN_ON(list_empty(&pi_state->list)); pi_state 972 kernel/futex.c list_del_init(&pi_state->list); pi_state 973 kernel/futex.c pi_state->owner = NULL; pi_state 976 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 979 kernel/futex.c rt_mutex_futex_unlock(&pi_state->pi_mutex); pi_state 980 kernel/futex.c put_pi_state(pi_state); pi_state 1079 kernel/futex.c struct futex_pi_state *pi_state, pi_state 1089 kernel/futex.c if (unlikely(!pi_state)) pi_state 1104 kernel/futex.c WARN_ON(!refcount_read(&pi_state->refcount)); pi_state 1110 kernel/futex.c raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); pi_state 1133 kernel/futex.c if (!pi_state->owner) { pi_state 1161 kernel/futex.c if (!pi_state->owner) pi_state 1170 kernel/futex.c if (pid != task_pid_vnr(pi_state->owner)) pi_state 1174 kernel/futex.c get_pi_state(pi_state); pi_state 1175 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 1176 kernel/futex.c *ps = pi_state; pi_state 1192 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 1291 kernel/futex.c struct futex_pi_state *pi_state; pi_state 1349 kernel/futex.c pi_state = alloc_pi_state(); pi_state 1355 kernel/futex.c rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); pi_state 1358 kernel/futex.c pi_state->key = *key; pi_state 1360 kernel/futex.c WARN_ON(!list_empty(&pi_state->list)); pi_state 1361 kernel/futex.c list_add(&pi_state->list, &p->pi_state_list); pi_state 1366 kernel/futex.c pi_state->owner = p; pi_state 1371 kernel/futex.c *ps = pi_state; pi_state 1388 kernel/futex.c return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps); pi_state 1473 kernel/futex.c return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps); pi_state 1544 kernel/futex.c if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n")) pi_state 1568 kernel/futex.c static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state) pi_state 1576 kernel/futex.c new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); pi_state 1622 kernel/futex.c raw_spin_lock(&pi_state->owner->pi_lock); pi_state 1623 kernel/futex.c WARN_ON(list_empty(&pi_state->list)); pi_state 1624 kernel/futex.c list_del_init(&pi_state->list); pi_state 1625 kernel/futex.c raw_spin_unlock(&pi_state->owner->pi_lock); pi_state 1628 kernel/futex.c WARN_ON(!list_empty(&pi_state->list)); pi_state 1629 kernel/futex.c list_add(&pi_state->list, &new_owner->pi_state_list); pi_state 1630 kernel/futex.c pi_state->owner = new_owner; pi_state 1633 kernel/futex.c postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q); pi_state 1636 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 1697 kernel/futex.c if (this->pi_state || this->rt_waiter) { pi_state 1827 kernel/futex.c if (this->pi_state || this->rt_waiter) { pi_state 1841 kernel/futex.c if (this->pi_state || this->rt_waiter) { pi_state 2021 kernel/futex.c struct futex_pi_state *pi_state = NULL; pi_state 2127 kernel/futex.c &key2, &pi_state, pi_state 2140 kernel/futex.c WARN_ON(pi_state); pi_state 2156 kernel/futex.c &pi_state, &exiting); pi_state 2215 kernel/futex.c this->pi_state) { pi_state 2246 kernel/futex.c get_pi_state(pi_state); pi_state 2247 kernel/futex.c this->pi_state = pi_state; pi_state 2248 kernel/futex.c ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, pi_state 2272 kernel/futex.c this->pi_state = NULL; pi_state 2273 kernel/futex.c put_pi_state(pi_state); pi_state 2290 kernel/futex.c put_pi_state(pi_state); pi_state 2429 kernel/futex.c BUG_ON(q->pi_state); pi_state 2449 kernel/futex.c BUG_ON(!q->pi_state); pi_state 2450 kernel/futex.c put_pi_state(q->pi_state); pi_state 2451 kernel/futex.c q->pi_state = NULL; pi_state 2459 kernel/futex.c struct futex_pi_state *pi_state = q->pi_state; pi_state 2467 kernel/futex.c raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); pi_state 2469 kernel/futex.c oldowner = pi_state->owner; pi_state 2505 kernel/futex.c if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) { pi_state 2514 kernel/futex.c newowner = rt_mutex_owner(&pi_state->pi_mutex); pi_state 2531 kernel/futex.c if (!pi_state->owner) pi_state 2554 kernel/futex.c if (pi_state->owner != NULL) { pi_state 2555 kernel/futex.c raw_spin_lock(&pi_state->owner->pi_lock); pi_state 2556 kernel/futex.c WARN_ON(list_empty(&pi_state->list)); pi_state 2557 kernel/futex.c list_del_init(&pi_state->list); pi_state 2558 kernel/futex.c raw_spin_unlock(&pi_state->owner->pi_lock); pi_state 2561 kernel/futex.c pi_state->owner = newowner; pi_state 2564 kernel/futex.c WARN_ON(!list_empty(&pi_state->list)); pi_state 2565 kernel/futex.c list_add(&pi_state->list, &newowner->pi_state_list); pi_state 2567 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 2585 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 2605 kernel/futex.c raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); pi_state 2610 kernel/futex.c if (pi_state->owner != oldowner) { pi_state 2621 kernel/futex.c raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); pi_state 2655 kernel/futex.c if (q->pi_state->owner != current) pi_state 2668 kernel/futex.c if (q->pi_state->owner == current) { pi_state 2677 kernel/futex.c if (rt_mutex_owner(&q->pi_state->pi_mutex) == current) { pi_state 2680 kernel/futex.c q->pi_state->pi_mutex.owner, pi_state 2681 kernel/futex.c q->pi_state->owner); pi_state 2897 kernel/futex.c struct futex_pi_state *pi_state = NULL; pi_state 2920 kernel/futex.c ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current, pi_state 2957 kernel/futex.c WARN_ON(!q.pi_state); pi_state 2965 kernel/futex.c ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex); pi_state 2986 kernel/futex.c raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock); pi_state 2993 kernel/futex.c ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current); pi_state 2994 kernel/futex.c raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock); pi_state 3005 kernel/futex.c ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter); pi_state 3018 kernel/futex.c if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter)) pi_state 3038 kernel/futex.c if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) { pi_state 3039 kernel/futex.c pi_state = q.pi_state; pi_state 3040 kernel/futex.c get_pi_state(pi_state); pi_state 3046 kernel/futex.c if (pi_state) { pi_state 3047 kernel/futex.c rt_mutex_futex_unlock(&pi_state->pi_mutex); pi_state 3048 kernel/futex.c put_pi_state(pi_state); pi_state 3118 kernel/futex.c struct futex_pi_state *pi_state = top_waiter->pi_state; pi_state 3121 kernel/futex.c if (!pi_state) pi_state 3128 kernel/futex.c if (pi_state->owner != current) pi_state 3131 kernel/futex.c get_pi_state(pi_state); pi_state 3142 kernel/futex.c raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock); pi_state 3146 kernel/futex.c ret = wake_futex_pi(uaddr, uval, pi_state); pi_state 3148 kernel/futex.c put_pi_state(pi_state); pi_state 3316 kernel/futex.c struct futex_pi_state *pi_state = NULL; pi_state 3391 kernel/futex.c if (q.pi_state && (q.pi_state->owner != current)) { pi_state 3394 kernel/futex.c if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) { pi_state 3395 kernel/futex.c pi_state = q.pi_state; pi_state 3396 kernel/futex.c get_pi_state(pi_state); pi_state 3402 kernel/futex.c put_pi_state(q.pi_state); pi_state 3413 kernel/futex.c WARN_ON(!q.pi_state); pi_state 3414 kernel/futex.c pi_mutex = &q.pi_state->pi_mutex; pi_state 3439 kernel/futex.c if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) { pi_state 3440 kernel/futex.c pi_state = q.pi_state; pi_state 3441 kernel/futex.c get_pi_state(pi_state); pi_state 3448 kernel/futex.c if (pi_state) { pi_state 3449 kernel/futex.c rt_mutex_futex_unlock(&pi_state->pi_mutex); pi_state 3450 kernel/futex.c put_pi_state(pi_state);