28 	struct bpf_map *map;  in find_and_alloc_map()  local
32 			map = tl->ops->map_alloc(attr);  in find_and_alloc_map()
33 			if (IS_ERR(map))  in find_and_alloc_map()
34 				return map;  in find_and_alloc_map()
35 			map->ops = tl->ops;  in find_and_alloc_map()
36 			map->map_type = attr->map_type;  in find_and_alloc_map()
37 			return map;  in find_and_alloc_map()
49 static int bpf_map_charge_memlock(struct bpf_map *map)  in bpf_map_charge_memlock()  argument
56 	atomic_long_add(map->pages, &user->locked_vm);  in bpf_map_charge_memlock()
59 		atomic_long_sub(map->pages, &user->locked_vm);  in bpf_map_charge_memlock()
63 	map->user = user;  in bpf_map_charge_memlock()
67 static void bpf_map_uncharge_memlock(struct bpf_map *map)  in bpf_map_uncharge_memlock()  argument
69 	struct user_struct *user = map->user;  in bpf_map_uncharge_memlock()
71 	atomic_long_sub(map->pages, &user->locked_vm);  in bpf_map_uncharge_memlock()
78 	struct bpf_map *map = container_of(work, struct bpf_map, work);  in bpf_map_free_deferred()  local
80 	bpf_map_uncharge_memlock(map);  in bpf_map_free_deferred()
82 	map->ops->map_free(map);  in bpf_map_free_deferred()
85 static void bpf_map_put_uref(struct bpf_map *map)  in bpf_map_put_uref()  argument
87 	if (atomic_dec_and_test(&map->usercnt)) {  in bpf_map_put_uref()
88 		if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)  in bpf_map_put_uref()
89 			bpf_fd_array_map_clear(map);  in bpf_map_put_uref()
96 void bpf_map_put(struct bpf_map *map)  in bpf_map_put()  argument
98 	if (atomic_dec_and_test(&map->refcnt)) {  in bpf_map_put()
99 		INIT_WORK(&map->work, bpf_map_free_deferred);  in bpf_map_put()
100 		schedule_work(&map->work);  in bpf_map_put()
104 void bpf_map_put_with_uref(struct bpf_map *map)  in bpf_map_put_with_uref()  argument
106 	bpf_map_put_uref(map);  in bpf_map_put_with_uref()
107 	bpf_map_put(map);  in bpf_map_put_with_uref()
120 int bpf_map_new_fd(struct bpf_map *map)  in bpf_map_new_fd()  argument
122 	return anon_inode_getfd("bpf-map", &bpf_map_fops, map,  in bpf_map_new_fd()
138 	struct bpf_map *map;  in map_create()  local
146 	map = find_and_alloc_map(attr);  in map_create()
147 	if (IS_ERR(map))  in map_create()
148 		return PTR_ERR(map);  in map_create()
150 	atomic_set(&map->refcnt, 1);  in map_create()
151 	atomic_set(&map->usercnt, 1);  in map_create()
153 	err = bpf_map_charge_memlock(map);  in map_create()
157 	err = bpf_map_new_fd(map);  in map_create()
165 	map->ops->map_free(map);  in map_create()
187 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)  in bpf_map_inc()  argument
189 	if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {  in bpf_map_inc()
190 		atomic_dec(&map->refcnt);  in bpf_map_inc()
194 		atomic_inc(&map->usercnt);  in bpf_map_inc()
195 	return map;  in bpf_map_inc()
201 	struct bpf_map *map;  in bpf_map_get_with_uref()  local
203 	map = __bpf_map_get(f);  in bpf_map_get_with_uref()
204 	if (IS_ERR(map))  in bpf_map_get_with_uref()
205 		return map;  in bpf_map_get_with_uref()
207 	map = bpf_map_inc(map, true);  in bpf_map_get_with_uref()
210 	return map;  in bpf_map_get_with_uref()
227 	struct bpf_map *map;  in map_lookup_elem()  local
236 	map = __bpf_map_get(f);  in map_lookup_elem()
237 	if (IS_ERR(map))  in map_lookup_elem()
238 		return PTR_ERR(map);  in map_lookup_elem()
241 	key = kmalloc(map->key_size, GFP_USER);  in map_lookup_elem()
246 	if (copy_from_user(key, ukey, map->key_size) != 0)  in map_lookup_elem()
250 	value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);  in map_lookup_elem()
255 	ptr = map->ops->map_lookup_elem(map, key);  in map_lookup_elem()
257 		memcpy(value, ptr, map->value_size);  in map_lookup_elem()
265 	if (copy_to_user(uvalue, value, map->value_size) != 0)  in map_lookup_elem()
286 	struct bpf_map *map;  in map_update_elem()  local
295 	map = __bpf_map_get(f);  in map_update_elem()
296 	if (IS_ERR(map))  in map_update_elem()
297 		return PTR_ERR(map);  in map_update_elem()
300 	key = kmalloc(map->key_size, GFP_USER);  in map_update_elem()
305 	if (copy_from_user(key, ukey, map->key_size) != 0)  in map_update_elem()
309 	value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);  in map_update_elem()
314 	if (copy_from_user(value, uvalue, map->value_size) != 0)  in map_update_elem()
321 	err = map->ops->map_update_elem(map, key, value, attr->flags);  in map_update_elem()
339 	struct bpf_map *map;  in map_delete_elem()  local
348 	map = __bpf_map_get(f);  in map_delete_elem()
349 	if (IS_ERR(map))  in map_delete_elem()
350 		return PTR_ERR(map);  in map_delete_elem()
353 	key = kmalloc(map->key_size, GFP_USER);  in map_delete_elem()
358 	if (copy_from_user(key, ukey, map->key_size) != 0)  in map_delete_elem()
362 	err = map->ops->map_delete_elem(map, key);  in map_delete_elem()
380 	struct bpf_map *map;  in map_get_next_key()  local
389 	map = __bpf_map_get(f);  in map_get_next_key()
390 	if (IS_ERR(map))  in map_get_next_key()
391 		return PTR_ERR(map);  in map_get_next_key()
394 	key = kmalloc(map->key_size, GFP_USER);  in map_get_next_key()
399 	if (copy_from_user(key, ukey, map->key_size) != 0)  in map_get_next_key()
403 	next_key = kmalloc(map->key_size, GFP_USER);  in map_get_next_key()
408 	err = map->ops->map_get_next_key(map, key, next_key);  in map_get_next_key()
414 	if (copy_to_user(unext_key, next_key, map->key_size) != 0)  in map_get_next_key()