selem              94 net/core/bpf_sk_storage.c 				    struct bpf_sk_storage_elem *selem)
selem              96 net/core/bpf_sk_storage.c 	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];
selem             111 net/core/bpf_sk_storage.c static bool selem_linked_to_sk(const struct bpf_sk_storage_elem *selem)
selem             113 net/core/bpf_sk_storage.c 	return !hlist_unhashed(&selem->snode);
selem             116 net/core/bpf_sk_storage.c static bool selem_linked_to_map(const struct bpf_sk_storage_elem *selem)
selem             118 net/core/bpf_sk_storage.c 	return !hlist_unhashed(&selem->map_node);
selem             125 net/core/bpf_sk_storage.c 	struct bpf_sk_storage_elem *selem;
selem             130 net/core/bpf_sk_storage.c 	selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN);
selem             131 net/core/bpf_sk_storage.c 	if (selem) {
selem             133 net/core/bpf_sk_storage.c 			memcpy(SDATA(selem)->data, value, smap->map.value_size);
selem             134 net/core/bpf_sk_storage.c 		return selem;
selem             148 net/core/bpf_sk_storage.c 			      struct bpf_sk_storage_elem *selem,
selem             155 net/core/bpf_sk_storage.c 	smap = rcu_dereference(SDATA(selem)->smap);
selem             164 net/core/bpf_sk_storage.c 	free_sk_storage = hlist_is_singular_node(&selem->snode,
selem             186 net/core/bpf_sk_storage.c 	hlist_del_init_rcu(&selem->snode);
selem             188 net/core/bpf_sk_storage.c 	    SDATA(selem))
selem             191 net/core/bpf_sk_storage.c 	kfree_rcu(selem, rcu);
selem             196 net/core/bpf_sk_storage.c static void selem_unlink_sk(struct bpf_sk_storage_elem *selem)
selem             201 net/core/bpf_sk_storage.c 	if (unlikely(!selem_linked_to_sk(selem)))
selem             205 net/core/bpf_sk_storage.c 	sk_storage = rcu_dereference(selem->sk_storage);
selem             207 net/core/bpf_sk_storage.c 	if (likely(selem_linked_to_sk(selem)))
selem             208 net/core/bpf_sk_storage.c 		free_sk_storage = __selem_unlink_sk(sk_storage, selem, true);
selem             216 net/core/bpf_sk_storage.c 			    struct bpf_sk_storage_elem *selem)
selem             218 net/core/bpf_sk_storage.c 	RCU_INIT_POINTER(selem->sk_storage, sk_storage);
selem             219 net/core/bpf_sk_storage.c 	hlist_add_head(&selem->snode, &sk_storage->list);
selem             222 net/core/bpf_sk_storage.c static void selem_unlink_map(struct bpf_sk_storage_elem *selem)
selem             227 net/core/bpf_sk_storage.c 	if (unlikely(!selem_linked_to_map(selem)))
selem             231 net/core/bpf_sk_storage.c 	smap = rcu_dereference(SDATA(selem)->smap);
selem             232 net/core/bpf_sk_storage.c 	b = select_bucket(smap, selem);
selem             234 net/core/bpf_sk_storage.c 	if (likely(selem_linked_to_map(selem)))
selem             235 net/core/bpf_sk_storage.c 		hlist_del_init_rcu(&selem->map_node);
selem             240 net/core/bpf_sk_storage.c 			   struct bpf_sk_storage_elem *selem)
selem             242 net/core/bpf_sk_storage.c 	struct bucket *b = select_bucket(smap, selem);
selem             245 net/core/bpf_sk_storage.c 	RCU_INIT_POINTER(SDATA(selem)->smap, smap);
selem             246 net/core/bpf_sk_storage.c 	hlist_add_head_rcu(&selem->map_node, &b->list);
selem             250 net/core/bpf_sk_storage.c static void selem_unlink(struct bpf_sk_storage_elem *selem)
selem             256 net/core/bpf_sk_storage.c 	selem_unlink_map(selem);
selem             257 net/core/bpf_sk_storage.c 	selem_unlink_sk(selem);
selem             266 net/core/bpf_sk_storage.c 	struct bpf_sk_storage_elem *selem;
selem             274 net/core/bpf_sk_storage.c 	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode)
selem             275 net/core/bpf_sk_storage.c 		if (rcu_access_pointer(SDATA(selem)->smap) == smap)
selem             278 net/core/bpf_sk_storage.c 	if (!selem)
selem             281 net/core/bpf_sk_storage.c 	sdata = SDATA(selem);
selem             289 net/core/bpf_sk_storage.c 		if (selem_linked_to_sk(selem))
selem             393 net/core/bpf_sk_storage.c 	struct bpf_sk_storage_elem *selem;
selem             412 net/core/bpf_sk_storage.c 		selem = selem_alloc(smap, sk, value, true);
selem             413 net/core/bpf_sk_storage.c 		if (!selem)
selem             416 net/core/bpf_sk_storage.c 		err = sk_storage_alloc(sk, smap, selem);
selem             418 net/core/bpf_sk_storage.c 			kfree(selem);
selem             423 net/core/bpf_sk_storage.c 		return SDATA(selem);
selem             462 net/core/bpf_sk_storage.c 		selem = SELEM(old_sdata);
selem             474 net/core/bpf_sk_storage.c 	selem = selem_alloc(smap, sk, value, !old_sdata);
selem             475 net/core/bpf_sk_storage.c 	if (!selem) {
selem             481 net/core/bpf_sk_storage.c 	selem_link_map(smap, selem);
selem             484 net/core/bpf_sk_storage.c 	__selem_link_sk(sk_storage, selem);
selem             494 net/core/bpf_sk_storage.c 	return SDATA(selem);
selem             517 net/core/bpf_sk_storage.c 	struct bpf_sk_storage_elem *selem;
selem             539 net/core/bpf_sk_storage.c 	hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) {
selem             543 net/core/bpf_sk_storage.c 		selem_unlink_map(selem);
selem             544 net/core/bpf_sk_storage.c 		free_sk_storage = __selem_unlink_sk(sk_storage, selem, true);
selem             555 net/core/bpf_sk_storage.c 	struct bpf_sk_storage_elem *selem;
selem             582 net/core/bpf_sk_storage.c 		while ((selem = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(&b->list)),
selem             585 net/core/bpf_sk_storage.c 			selem_unlink(selem);
selem             755 net/core/bpf_sk_storage.c 			  struct bpf_sk_storage_elem *selem)
selem             765 net/core/bpf_sk_storage.c 				      SDATA(selem)->data, true);
selem             768 net/core/bpf_sk_storage.c 			       SDATA(selem)->data);
selem             777 net/core/bpf_sk_storage.c 	struct bpf_sk_storage_elem *selem;
selem             788 net/core/bpf_sk_storage.c 	hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
selem             793 net/core/bpf_sk_storage.c 		smap = rcu_dereference(SDATA(selem)->smap);
selem             806 net/core/bpf_sk_storage.c 		copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);