430 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)  in find_prog_type()  argument
436 			prog->aux->ops = tl->ops;  in find_prog_type()
437 			prog->type = type;  in find_prog_type()
459 static void fixup_bpf_calls(struct bpf_prog *prog)  in fixup_bpf_calls()  argument
464 	for (i = 0; i < prog->len; i++) {  in fixup_bpf_calls()
465 		struct bpf_insn *insn = &prog->insnsi[i];  in fixup_bpf_calls()
472 			BUG_ON(!prog->aux->ops->get_func_proto);  in fixup_bpf_calls()
475 				prog->dst_needed = 1;  in fixup_bpf_calls()
491 			fn = prog->aux->ops->get_func_proto(insn->imm);  in fixup_bpf_calls()
512 static int bpf_prog_charge_memlock(struct bpf_prog *prog)  in bpf_prog_charge_memlock()  argument
519 	atomic_long_add(prog->pages, &user->locked_vm);  in bpf_prog_charge_memlock()
521 		atomic_long_sub(prog->pages, &user->locked_vm);  in bpf_prog_charge_memlock()
525 	prog->aux->user = user;  in bpf_prog_charge_memlock()
529 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)  in bpf_prog_uncharge_memlock()  argument
531 	struct user_struct *user = prog->aux->user;  in bpf_prog_uncharge_memlock()
533 	atomic_long_sub(prog->pages, &user->locked_vm);  in bpf_prog_uncharge_memlock()
542 	bpf_prog_uncharge_memlock(aux->prog);  in __prog_put_common()
543 	bpf_prog_free(aux->prog);  in __prog_put_common()
547 void bpf_prog_put_rcu(struct bpf_prog *prog)  in bpf_prog_put_rcu()  argument
549 	if (atomic_dec_and_test(&prog->aux->refcnt))  in bpf_prog_put_rcu()
550 		call_rcu(&prog->aux->rcu, __prog_put_common);  in bpf_prog_put_rcu()
553 void bpf_prog_put(struct bpf_prog *prog)  in bpf_prog_put()  argument
555 	if (atomic_dec_and_test(&prog->aux->refcnt))  in bpf_prog_put()
556 		__prog_put_common(&prog->aux->rcu);  in bpf_prog_put()
562 	struct bpf_prog *prog = filp->private_data;  in bpf_prog_release()  local
564 	bpf_prog_put_rcu(prog);  in bpf_prog_release()
572 int bpf_prog_new_fd(struct bpf_prog *prog)  in bpf_prog_new_fd()  argument
574 	return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,  in bpf_prog_new_fd()
590 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)  in bpf_prog_inc()  argument
592 	if (atomic_inc_return(&prog->aux->refcnt) > BPF_MAX_REFCNT) {  in bpf_prog_inc()
593 		atomic_dec(&prog->aux->refcnt);  in bpf_prog_inc()
596 	return prog;  in bpf_prog_inc()
605 	struct bpf_prog *prog;  in bpf_prog_get()  local
607 	prog = __bpf_prog_get(f);  in bpf_prog_get()
608 	if (IS_ERR(prog))  in bpf_prog_get()
609 		return prog;  in bpf_prog_get()
611 	prog = bpf_prog_inc(prog);  in bpf_prog_get()
614 	return prog;  in bpf_prog_get()
624 	struct bpf_prog *prog;  in bpf_prog_load()  local
652 	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);  in bpf_prog_load()
653 	if (!prog)  in bpf_prog_load()
656 	err = bpf_prog_charge_memlock(prog);  in bpf_prog_load()
660 	prog->len = attr->insn_cnt;  in bpf_prog_load()
663 	if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),  in bpf_prog_load()
664 			   prog->len * sizeof(struct bpf_insn)) != 0)  in bpf_prog_load()
667 	prog->orig_prog = NULL;  in bpf_prog_load()
668 	prog->jited = 0;  in bpf_prog_load()
670 	atomic_set(&prog->aux->refcnt, 1);  in bpf_prog_load()
671 	prog->gpl_compatible = is_gpl ? 1 : 0;  in bpf_prog_load()
674 	err = find_prog_type(type, prog);  in bpf_prog_load()
679 	err = bpf_check(&prog, attr);  in bpf_prog_load()
684 	fixup_bpf_calls(prog);  in bpf_prog_load()
687 	err = bpf_prog_select_runtime(prog);  in bpf_prog_load()
691 	err = bpf_prog_new_fd(prog);  in bpf_prog_load()
699 	free_used_maps(prog->aux);  in bpf_prog_load()
701 	bpf_prog_uncharge_memlock(prog);  in bpf_prog_load()
703 	bpf_prog_free(prog);  in bpf_prog_load()