37 	struct bpf_htab *htab;  in htab_map_alloc()  local
40 	htab = kzalloc(sizeof(*htab), GFP_USER);  in htab_map_alloc()
41 	if (!htab)  in htab_map_alloc()
45 	htab->map.key_size = attr->key_size;  in htab_map_alloc()
46 	htab->map.value_size = attr->value_size;  in htab_map_alloc()
47 	htab->map.max_entries = attr->max_entries;  in htab_map_alloc()
53 	if (htab->map.max_entries == 0 || htab->map.key_size == 0 ||  in htab_map_alloc()
54 	    htab->map.value_size == 0)  in htab_map_alloc()
58 	htab->n_buckets = roundup_pow_of_two(htab->map.max_entries);  in htab_map_alloc()
61 	if (htab->map.key_size > MAX_BPF_STACK)  in htab_map_alloc()
67 	if (htab->map.value_size >= (1 << (KMALLOC_SHIFT_MAX - 1)) -  in htab_map_alloc()
76 	htab->elem_size = sizeof(struct htab_elem) +  in htab_map_alloc()
77 			  round_up(htab->map.key_size, 8) +  in htab_map_alloc()
78 			  htab->map.value_size;  in htab_map_alloc()
81 	if (htab->n_buckets == 0 ||  in htab_map_alloc()
82 	    htab->n_buckets > U32_MAX / sizeof(struct hlist_head))  in htab_map_alloc()
85 	if ((u64) htab->n_buckets * sizeof(struct hlist_head) +  in htab_map_alloc()
86 	    (u64) htab->elem_size * htab->map.max_entries >=  in htab_map_alloc()
91 	htab->map.pages = round_up(htab->n_buckets * sizeof(struct hlist_head) +  in htab_map_alloc()
92 				   htab->elem_size * htab->map.max_entries,  in htab_map_alloc()
96 	htab->buckets = kmalloc_array(htab->n_buckets, sizeof(struct hlist_head),  in htab_map_alloc()
99 	if (!htab->buckets) {  in htab_map_alloc()
100 		htab->buckets = vmalloc(htab->n_buckets * sizeof(struct hlist_head));  in htab_map_alloc()
101 		if (!htab->buckets)  in htab_map_alloc()
105 	for (i = 0; i < htab->n_buckets; i++)  in htab_map_alloc()
106 		INIT_HLIST_HEAD(&htab->buckets[i]);  in htab_map_alloc()
108 	raw_spin_lock_init(&htab->lock);  in htab_map_alloc()
109 	htab->count = 0;  in htab_map_alloc()
111 	return &htab->map;  in htab_map_alloc()
114 	kfree(htab);  in htab_map_alloc()
123 static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash)  in select_bucket()  argument
125 	return &htab->buckets[hash & (htab->n_buckets - 1)];  in select_bucket()
143 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_lookup_elem()  local
155 	head = select_bucket(htab, hash);  in htab_map_lookup_elem()
168 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_get_next_key()  local
180 	head = select_bucket(htab, hash);  in htab_map_get_next_key()
201 	i = hash & (htab->n_buckets - 1);  in htab_map_get_next_key()
206 	for (; i < htab->n_buckets; i++) {  in htab_map_get_next_key()
207 		head = select_bucket(htab, i);  in htab_map_get_next_key()
227 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_update_elem()  local
241 	l_new = kmalloc(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN);  in htab_map_update_elem()
253 	raw_spin_lock_irqsave(&htab->lock, flags);  in htab_map_update_elem()
255 	head = select_bucket(htab, l_new->hash);  in htab_map_update_elem()
259 	if (!l_old && unlikely(htab->count >= map->max_entries)) {  in htab_map_update_elem()
287 		htab->count++;  in htab_map_update_elem()
289 	raw_spin_unlock_irqrestore(&htab->lock, flags);  in htab_map_update_elem()
293 	raw_spin_unlock_irqrestore(&htab->lock, flags);  in htab_map_update_elem()
301 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_delete_elem()  local
314 	raw_spin_lock_irqsave(&htab->lock, flags);  in htab_map_delete_elem()
316 	head = select_bucket(htab, hash);  in htab_map_delete_elem()
322 		htab->count--;  in htab_map_delete_elem()
327 	raw_spin_unlock_irqrestore(&htab->lock, flags);  in htab_map_delete_elem()
331 static void delete_all_elements(struct bpf_htab *htab)  in delete_all_elements()  argument
335 	for (i = 0; i < htab->n_buckets; i++) {  in delete_all_elements()
336 		struct hlist_head *head = select_bucket(htab, i);  in delete_all_elements()
342 			htab->count--;  in delete_all_elements()
351 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_free()  local
363 	delete_all_elements(htab);  in htab_map_free()
364 	kvfree(htab->buckets);  in htab_map_free()
365 	kfree(htab);  in htab_map_free()