selem 94 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem) selem 96 net/core/bpf_sk_storage.c return &smap->buckets[hash_ptr(selem, smap->bucket_log)]; selem 111 net/core/bpf_sk_storage.c static bool selem_linked_to_sk(const struct bpf_sk_storage_elem *selem) selem 113 net/core/bpf_sk_storage.c return !hlist_unhashed(&selem->snode); selem 116 net/core/bpf_sk_storage.c static bool selem_linked_to_map(const struct bpf_sk_storage_elem *selem) selem 118 net/core/bpf_sk_storage.c return !hlist_unhashed(&selem->map_node); selem 125 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem; selem 130 net/core/bpf_sk_storage.c selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN); selem 131 net/core/bpf_sk_storage.c if (selem) { selem 133 net/core/bpf_sk_storage.c memcpy(SDATA(selem)->data, value, smap->map.value_size); selem 134 net/core/bpf_sk_storage.c return selem; selem 148 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem, selem 155 net/core/bpf_sk_storage.c smap = rcu_dereference(SDATA(selem)->smap); selem 164 net/core/bpf_sk_storage.c free_sk_storage = hlist_is_singular_node(&selem->snode, selem 186 net/core/bpf_sk_storage.c hlist_del_init_rcu(&selem->snode); selem 188 net/core/bpf_sk_storage.c SDATA(selem)) selem 191 net/core/bpf_sk_storage.c kfree_rcu(selem, rcu); selem 196 net/core/bpf_sk_storage.c static void selem_unlink_sk(struct bpf_sk_storage_elem *selem) selem 201 net/core/bpf_sk_storage.c if (unlikely(!selem_linked_to_sk(selem))) selem 205 net/core/bpf_sk_storage.c sk_storage = rcu_dereference(selem->sk_storage); selem 207 net/core/bpf_sk_storage.c if (likely(selem_linked_to_sk(selem))) selem 208 net/core/bpf_sk_storage.c free_sk_storage = __selem_unlink_sk(sk_storage, selem, true); selem 216 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem) selem 218 net/core/bpf_sk_storage.c RCU_INIT_POINTER(selem->sk_storage, sk_storage); selem 219 net/core/bpf_sk_storage.c hlist_add_head(&selem->snode, &sk_storage->list); selem 222 net/core/bpf_sk_storage.c static void selem_unlink_map(struct bpf_sk_storage_elem *selem) selem 227 net/core/bpf_sk_storage.c if (unlikely(!selem_linked_to_map(selem))) selem 231 net/core/bpf_sk_storage.c smap = rcu_dereference(SDATA(selem)->smap); selem 232 net/core/bpf_sk_storage.c b = select_bucket(smap, selem); selem 234 net/core/bpf_sk_storage.c if (likely(selem_linked_to_map(selem))) selem 235 net/core/bpf_sk_storage.c hlist_del_init_rcu(&selem->map_node); selem 240 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem) selem 242 net/core/bpf_sk_storage.c struct bucket *b = select_bucket(smap, selem); selem 245 net/core/bpf_sk_storage.c RCU_INIT_POINTER(SDATA(selem)->smap, smap); selem 246 net/core/bpf_sk_storage.c hlist_add_head_rcu(&selem->map_node, &b->list); selem 250 net/core/bpf_sk_storage.c static void selem_unlink(struct bpf_sk_storage_elem *selem) selem 256 net/core/bpf_sk_storage.c selem_unlink_map(selem); selem 257 net/core/bpf_sk_storage.c selem_unlink_sk(selem); selem 266 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem; selem 274 net/core/bpf_sk_storage.c hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) selem 275 net/core/bpf_sk_storage.c if (rcu_access_pointer(SDATA(selem)->smap) == smap) selem 278 net/core/bpf_sk_storage.c if (!selem) selem 281 net/core/bpf_sk_storage.c sdata = SDATA(selem); selem 289 net/core/bpf_sk_storage.c if (selem_linked_to_sk(selem)) selem 393 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem; selem 412 net/core/bpf_sk_storage.c selem = selem_alloc(smap, sk, value, true); selem 413 net/core/bpf_sk_storage.c if (!selem) selem 416 net/core/bpf_sk_storage.c err = sk_storage_alloc(sk, smap, selem); selem 418 net/core/bpf_sk_storage.c kfree(selem); selem 423 net/core/bpf_sk_storage.c return SDATA(selem); selem 462 net/core/bpf_sk_storage.c selem = SELEM(old_sdata); selem 474 net/core/bpf_sk_storage.c selem = selem_alloc(smap, sk, value, !old_sdata); selem 475 net/core/bpf_sk_storage.c if (!selem) { selem 481 net/core/bpf_sk_storage.c selem_link_map(smap, selem); selem 484 net/core/bpf_sk_storage.c __selem_link_sk(sk_storage, selem); selem 494 net/core/bpf_sk_storage.c return SDATA(selem); selem 517 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem; selem 539 net/core/bpf_sk_storage.c hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) { selem 543 net/core/bpf_sk_storage.c selem_unlink_map(selem); selem 544 net/core/bpf_sk_storage.c free_sk_storage = __selem_unlink_sk(sk_storage, selem, true); selem 555 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem; selem 582 net/core/bpf_sk_storage.c while ((selem = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(&b->list)), selem 585 net/core/bpf_sk_storage.c selem_unlink(selem); selem 755 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem) selem 765 net/core/bpf_sk_storage.c SDATA(selem)->data, true); selem 768 net/core/bpf_sk_storage.c SDATA(selem)->data); selem 777 net/core/bpf_sk_storage.c struct bpf_sk_storage_elem *selem; selem 788 net/core/bpf_sk_storage.c hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { selem 793 net/core/bpf_sk_storage.c smap = rcu_dereference(SDATA(selem)->smap); selem 806 net/core/bpf_sk_storage.c copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);