37 	struct bpf_htab *htab;  in htab_map_alloc()  local
40 	htab = kzalloc(sizeof(*htab), GFP_USER);  in htab_map_alloc()
41 	if (!htab)  in htab_map_alloc()
45 	htab->map.key_size = attr->key_size;  in htab_map_alloc()
46 	htab->map.value_size = attr->value_size;  in htab_map_alloc()
47 	htab->map.max_entries = attr->max_entries;  in htab_map_alloc()
53 	if (htab->map.max_entries == 0 || htab->map.key_size == 0 ||  in htab_map_alloc()
54 	    htab->map.value_size == 0)  in htab_map_alloc()
58 	htab->n_buckets = roundup_pow_of_two(htab->map.max_entries);  in htab_map_alloc()
61 	if (htab->map.key_size > MAX_BPF_STACK)  in htab_map_alloc()
69 	if (htab->n_buckets == 0 ||  in htab_map_alloc()
70 	    htab->n_buckets > U32_MAX / sizeof(struct hlist_head))  in htab_map_alloc()
73 	htab->buckets = kmalloc_array(htab->n_buckets, sizeof(struct hlist_head),  in htab_map_alloc()
76 	if (!htab->buckets) {  in htab_map_alloc()
77 		htab->buckets = vmalloc(htab->n_buckets * sizeof(struct hlist_head));  in htab_map_alloc()
78 		if (!htab->buckets)  in htab_map_alloc()
82 	for (i = 0; i < htab->n_buckets; i++)  in htab_map_alloc()
83 		INIT_HLIST_HEAD(&htab->buckets[i]);  in htab_map_alloc()
85 	spin_lock_init(&htab->lock);  in htab_map_alloc()
86 	htab->count = 0;  in htab_map_alloc()
88 	htab->elem_size = sizeof(struct htab_elem) +  in htab_map_alloc()
89 			  round_up(htab->map.key_size, 8) +  in htab_map_alloc()
90 			  htab->map.value_size;  in htab_map_alloc()
91 	return &htab->map;  in htab_map_alloc()
94 	kfree(htab);  in htab_map_alloc()
103 static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash)  in select_bucket()  argument
105 	return &htab->buckets[hash & (htab->n_buckets - 1)];  in select_bucket()
123 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_lookup_elem()  local
135 	head = select_bucket(htab, hash);  in htab_map_lookup_elem()
148 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_get_next_key()  local
160 	head = select_bucket(htab, hash);  in htab_map_get_next_key()
181 	i = hash & (htab->n_buckets - 1);  in htab_map_get_next_key()
186 	for (; i < htab->n_buckets; i++) {  in htab_map_get_next_key()
187 		head = select_bucket(htab, i);  in htab_map_get_next_key()
207 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_update_elem()  local
221 	l_new = kmalloc(htab->elem_size, GFP_ATOMIC);  in htab_map_update_elem()
233 	spin_lock_irqsave(&htab->lock, flags);  in htab_map_update_elem()
235 	head = select_bucket(htab, l_new->hash);  in htab_map_update_elem()
239 	if (!l_old && unlikely(htab->count >= map->max_entries)) {  in htab_map_update_elem()
267 		htab->count++;  in htab_map_update_elem()
269 	spin_unlock_irqrestore(&htab->lock, flags);  in htab_map_update_elem()
273 	spin_unlock_irqrestore(&htab->lock, flags);  in htab_map_update_elem()
281 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_delete_elem()  local
294 	spin_lock_irqsave(&htab->lock, flags);  in htab_map_delete_elem()
296 	head = select_bucket(htab, hash);  in htab_map_delete_elem()
302 		htab->count--;  in htab_map_delete_elem()
307 	spin_unlock_irqrestore(&htab->lock, flags);  in htab_map_delete_elem()
311 static void delete_all_elements(struct bpf_htab *htab)  in delete_all_elements()  argument
315 	for (i = 0; i < htab->n_buckets; i++) {  in delete_all_elements()
316 		struct hlist_head *head = select_bucket(htab, i);  in delete_all_elements()
322 			htab->count--;  in delete_all_elements()
331 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in htab_map_free()  local
343 	delete_all_elements(htab);  in htab_map_free()
344 	kvfree(htab->buckets);  in htab_map_free()
345 	kfree(htab);  in htab_map_free()