315 static void print_bpf_insn(struct bpf_insn *insn)  in print_bpf_insn()  argument
317 	u8 class = BPF_CLASS(insn->code);  in print_bpf_insn()
320 		if (BPF_SRC(insn->code) == BPF_X)  in print_bpf_insn()
322 				insn->code, class == BPF_ALU ? "(u32) " : "",  in print_bpf_insn()
323 				insn->dst_reg,  in print_bpf_insn()
324 				bpf_alu_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
326 				insn->src_reg);  in print_bpf_insn()
329 				insn->code, class == BPF_ALU ? "(u32) " : "",  in print_bpf_insn()
330 				insn->dst_reg,  in print_bpf_insn()
331 				bpf_alu_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
333 				insn->imm);  in print_bpf_insn()
335 		if (BPF_MODE(insn->code) == BPF_MEM)  in print_bpf_insn()
337 				insn->code,  in print_bpf_insn()
338 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
339 				insn->dst_reg,  in print_bpf_insn()
340 				insn->off, insn->src_reg);  in print_bpf_insn()
341 		else if (BPF_MODE(insn->code) == BPF_XADD)  in print_bpf_insn()
343 				insn->code,  in print_bpf_insn()
344 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
345 				insn->dst_reg, insn->off,  in print_bpf_insn()
346 				insn->src_reg);  in print_bpf_insn()
348 			verbose("BUG_%02x\n", insn->code);  in print_bpf_insn()
350 		if (BPF_MODE(insn->code) != BPF_MEM) {  in print_bpf_insn()
351 			verbose("BUG_st_%02x\n", insn->code);  in print_bpf_insn()
355 			insn->code,  in print_bpf_insn()
356 			bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
357 			insn->dst_reg,  in print_bpf_insn()
358 			insn->off, insn->imm);  in print_bpf_insn()
360 		if (BPF_MODE(insn->code) != BPF_MEM) {  in print_bpf_insn()
361 			verbose("BUG_ldx_%02x\n", insn->code);  in print_bpf_insn()
365 			insn->code, insn->dst_reg,  in print_bpf_insn()
366 			bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
367 			insn->src_reg, insn->off);  in print_bpf_insn()
369 		if (BPF_MODE(insn->code) == BPF_ABS) {  in print_bpf_insn()
371 				insn->code,  in print_bpf_insn()
372 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
373 				insn->imm);  in print_bpf_insn()
374 		} else if (BPF_MODE(insn->code) == BPF_IND) {  in print_bpf_insn()
376 				insn->code,  in print_bpf_insn()
377 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
378 				insn->src_reg, insn->imm);  in print_bpf_insn()
379 		} else if (BPF_MODE(insn->code) == BPF_IMM) {  in print_bpf_insn()
381 				insn->code, insn->dst_reg, insn->imm);  in print_bpf_insn()
383 			verbose("BUG_ld_%02x\n", insn->code);  in print_bpf_insn()
387 		u8 opcode = BPF_OP(insn->code);  in print_bpf_insn()
390 			verbose("(%02x) call %d\n", insn->code, insn->imm);  in print_bpf_insn()
391 		} else if (insn->code == (BPF_JMP | BPF_JA)) {  in print_bpf_insn()
393 				insn->code, insn->off);  in print_bpf_insn()
394 		} else if (insn->code == (BPF_JMP | BPF_EXIT)) {  in print_bpf_insn()
395 			verbose("(%02x) exit\n", insn->code);  in print_bpf_insn()
396 		} else if (BPF_SRC(insn->code) == BPF_X) {  in print_bpf_insn()
398 				insn->code, insn->dst_reg,  in print_bpf_insn()
399 				bpf_jmp_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
400 				insn->src_reg, insn->off);  in print_bpf_insn()
403 				insn->code, insn->dst_reg,  in print_bpf_insn()
404 				bpf_jmp_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
405 				insn->imm, insn->off);  in print_bpf_insn()
408 		verbose("(%02x) %s\n", insn->code, bpf_class_string[class]);  in print_bpf_insn()
687 static int check_xadd(struct verifier_env *env, struct bpf_insn *insn)  in check_xadd()  argument
692 	if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) ||  in check_xadd()
693 	    insn->imm != 0) {  in check_xadd()
699 	err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_xadd()
704 	err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_xadd()
709 	err = check_mem_access(env, insn->dst_reg, insn->off,  in check_xadd()
710 			       BPF_SIZE(insn->code), BPF_READ, -1);  in check_xadd()
715 	return check_mem_access(env, insn->dst_reg, insn->off,  in check_xadd()
716 				BPF_SIZE(insn->code), BPF_WRITE, -1);  in check_xadd()
914 static int check_alu_op(struct reg_state *regs, struct bpf_insn *insn)  in check_alu_op()  argument
916 	u8 opcode = BPF_OP(insn->code);  in check_alu_op()
921 			if (BPF_SRC(insn->code) != 0 ||  in check_alu_op()
922 			    insn->src_reg != BPF_REG_0 ||  in check_alu_op()
923 			    insn->off != 0 || insn->imm != 0) {  in check_alu_op()
928 			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||  in check_alu_op()
929 			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64)) {  in check_alu_op()
936 		err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_alu_op()
941 		err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_alu_op()
947 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
948 			if (insn->imm != 0 || insn->off != 0) {  in check_alu_op()
954 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_alu_op()
958 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {  in check_alu_op()
965 		err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_alu_op()
969 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
970 			if (BPF_CLASS(insn->code) == BPF_ALU64) {  in check_alu_op()
974 				regs[insn->dst_reg] = regs[insn->src_reg];  in check_alu_op()
976 				regs[insn->dst_reg].type = UNKNOWN_VALUE;  in check_alu_op()
977 				regs[insn->dst_reg].map_ptr = NULL;  in check_alu_op()
983 			regs[insn->dst_reg].type = CONST_IMM;  in check_alu_op()
984 			regs[insn->dst_reg].imm = insn->imm;  in check_alu_op()
995 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
996 			if (insn->imm != 0 || insn->off != 0) {  in check_alu_op()
1001 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_alu_op()
1005 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {  in check_alu_op()
1012 		err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_alu_op()
1017 		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {  in check_alu_op()
1023 		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {  in check_alu_op()
1024 			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;  in check_alu_op()
1026 			if (insn->imm < 0 || insn->imm >= size) {  in check_alu_op()
1027 				verbose("invalid shift %d\n", insn->imm);  in check_alu_op()
1033 		if (opcode == BPF_ADD && BPF_CLASS(insn->code) == BPF_ALU64 &&  in check_alu_op()
1034 		    regs[insn->dst_reg].type == FRAME_PTR &&  in check_alu_op()
1035 		    BPF_SRC(insn->code) == BPF_K)  in check_alu_op()
1039 		err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_alu_op()
1044 			regs[insn->dst_reg].type = PTR_TO_STACK;  in check_alu_op()
1045 			regs[insn->dst_reg].imm = insn->imm;  in check_alu_op()
1053 			     struct bpf_insn *insn, int *insn_idx)  in check_cond_jmp_op()  argument
1057 	u8 opcode = BPF_OP(insn->code);  in check_cond_jmp_op()
1065 	if (BPF_SRC(insn->code) == BPF_X) {  in check_cond_jmp_op()
1066 		if (insn->imm != 0) {  in check_cond_jmp_op()
1072 		err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_cond_jmp_op()
1076 		if (insn->src_reg != BPF_REG_0) {  in check_cond_jmp_op()
1083 	err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_cond_jmp_op()
1088 	if (BPF_SRC(insn->code) == BPF_K &&  in check_cond_jmp_op()
1090 	    regs[insn->dst_reg].type == CONST_IMM &&  in check_cond_jmp_op()
1091 	    regs[insn->dst_reg].imm == insn->imm) {  in check_cond_jmp_op()
1096 			*insn_idx += insn->off;  in check_cond_jmp_op()
1107 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx);  in check_cond_jmp_op()
1112 	if (BPF_SRC(insn->code) == BPF_K &&  in check_cond_jmp_op()
1113 	    insn->imm == 0 && (opcode == BPF_JEQ ||  in check_cond_jmp_op()
1115 	    regs[insn->dst_reg].type == PTR_TO_MAP_VALUE_OR_NULL) {  in check_cond_jmp_op()
1120 			regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;  in check_cond_jmp_op()
1122 			other_branch->regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1123 			other_branch->regs[insn->dst_reg].imm = 0;  in check_cond_jmp_op()
1125 			other_branch->regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;  in check_cond_jmp_op()
1126 			regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1127 			regs[insn->dst_reg].imm = 0;  in check_cond_jmp_op()
1129 	} else if (BPF_SRC(insn->code) == BPF_K &&  in check_cond_jmp_op()
1136 			other_branch->regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1137 			other_branch->regs[insn->dst_reg].imm = insn->imm;  in check_cond_jmp_op()
1142 			regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1143 			regs[insn->dst_reg].imm = insn->imm;  in check_cond_jmp_op()
1152 static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn)  in ld_imm64_to_map_ptr()  argument
1154 	u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32;  in ld_imm64_to_map_ptr()
1160 static int check_ld_imm(struct verifier_env *env, struct bpf_insn *insn)  in check_ld_imm()  argument
1165 	if (BPF_SIZE(insn->code) != BPF_DW) {  in check_ld_imm()
1169 	if (insn->off != 0) {  in check_ld_imm()
1174 	err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_ld_imm()
1178 	if (insn->src_reg == 0)  in check_ld_imm()
1183 	BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD);  in check_ld_imm()
1185 	regs[insn->dst_reg].type = CONST_PTR_TO_MAP;  in check_ld_imm()
1186 	regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn);  in check_ld_imm()
1217 static int check_ld_abs(struct verifier_env *env, struct bpf_insn *insn)  in check_ld_abs()  argument
1220 	u8 mode = BPF_MODE(insn->code);  in check_ld_abs()
1229 	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||  in check_ld_abs()
1230 	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {  in check_ld_abs()
1247 		err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_ld_abs()
1586 		struct bpf_insn *insn;  in do_check()  local
1596 		insn = &insns[insn_idx];  in do_check()
1597 		class = BPF_CLASS(insn->code);  in do_check()
1628 			print_bpf_insn(insn);  in do_check()
1632 			err = check_alu_op(regs, insn);  in do_check()
1642 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in do_check()
1646 			err = check_reg_arg(regs, insn->dst_reg, DST_OP_NO_MARK);  in do_check()
1650 			src_reg_type = regs[insn->src_reg].type;  in do_check()
1655 			err = check_mem_access(env, insn->src_reg, insn->off,  in do_check()
1656 					       BPF_SIZE(insn->code), BPF_READ,  in do_check()
1657 					       insn->dst_reg);  in do_check()
1661 			if (BPF_SIZE(insn->code) != BPF_W) {  in do_check()
1666 			if (insn->imm == 0) {  in do_check()
1671 				insn->imm = src_reg_type;  in do_check()
1673 			} else if (src_reg_type != insn->imm &&  in do_check()
1675 				    insn->imm == PTR_TO_CTX)) {  in do_check()
1688 			if (BPF_MODE(insn->code) == BPF_XADD) {  in do_check()
1689 				err = check_xadd(env, insn);  in do_check()
1696 			if (BPF_MODE(insn->code) != BPF_MEM ||  in do_check()
1697 			    insn->imm != 0) {  in do_check()
1702 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in do_check()
1706 			err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in do_check()
1711 			err = check_mem_access(env, insn->dst_reg, insn->off,  in do_check()
1712 					       BPF_SIZE(insn->code), BPF_WRITE,  in do_check()
1713 					       insn->src_reg);  in do_check()
1718 			if (BPF_MODE(insn->code) != BPF_MEM ||  in do_check()
1719 			    insn->src_reg != BPF_REG_0) {  in do_check()
1724 			err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in do_check()
1729 			err = check_mem_access(env, insn->dst_reg, insn->off,  in do_check()
1730 					       BPF_SIZE(insn->code), BPF_WRITE,  in do_check()
1736 			u8 opcode = BPF_OP(insn->code);  in do_check()
1739 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
1740 				    insn->off != 0 ||  in do_check()
1741 				    insn->src_reg != BPF_REG_0 ||  in do_check()
1742 				    insn->dst_reg != BPF_REG_0) {  in do_check()
1747 				err = check_call(env, insn->imm);  in do_check()
1752 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
1753 				    insn->imm != 0 ||  in do_check()
1754 				    insn->src_reg != BPF_REG_0 ||  in do_check()
1755 				    insn->dst_reg != BPF_REG_0) {  in do_check()
1760 				insn_idx += insn->off + 1;  in do_check()
1764 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
1765 				    insn->imm != 0 ||  in do_check()
1766 				    insn->src_reg != BPF_REG_0 ||  in do_check()
1767 				    insn->dst_reg != BPF_REG_0) {  in do_check()
1791 				err = check_cond_jmp_op(env, insn, &insn_idx);  in do_check()
1796 			u8 mode = BPF_MODE(insn->code);  in do_check()
1799 				err = check_ld_abs(env, insn);  in do_check()
1804 				err = check_ld_imm(env, insn);  in do_check()
1829 	struct bpf_insn *insn = env->prog->insnsi;  in replace_map_fd_with_map_ptr()  local
1833 	for (i = 0; i < insn_cnt; i++, insn++) {  in replace_map_fd_with_map_ptr()
1834 		if (BPF_CLASS(insn->code) == BPF_LDX &&  in replace_map_fd_with_map_ptr()
1835 		    (BPF_MODE(insn->code) != BPF_MEM ||  in replace_map_fd_with_map_ptr()
1836 		     insn->imm != 0)) {  in replace_map_fd_with_map_ptr()
1841 		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {  in replace_map_fd_with_map_ptr()
1845 			if (i == insn_cnt - 1 || insn[1].code != 0 ||  in replace_map_fd_with_map_ptr()
1846 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||  in replace_map_fd_with_map_ptr()
1847 			    insn[1].off != 0) {  in replace_map_fd_with_map_ptr()
1852 			if (insn->src_reg == 0)  in replace_map_fd_with_map_ptr()
1856 			if (insn->src_reg != BPF_PSEUDO_MAP_FD) {  in replace_map_fd_with_map_ptr()
1861 			f = fdget(insn->imm);  in replace_map_fd_with_map_ptr()
1866 					insn->imm);  in replace_map_fd_with_map_ptr()
1872 			insn[0].imm = (u32) (unsigned long) map;  in replace_map_fd_with_map_ptr()
1873 			insn[1].imm = ((u64) (unsigned long) map) >> 32;  in replace_map_fd_with_map_ptr()
1899 			insn++;  in replace_map_fd_with_map_ptr()
1923 	struct bpf_insn *insn = env->prog->insnsi;  in convert_pseudo_ld_imm64()  local
1927 	for (i = 0; i < insn_cnt; i++, insn++)  in convert_pseudo_ld_imm64()
1928 		if (insn->code == (BPF_LD | BPF_IMM | BPF_DW))  in convert_pseudo_ld_imm64()
1929 			insn->src_reg = 0;  in convert_pseudo_ld_imm64()
1934 	struct bpf_insn *insn = prog->insnsi;  in adjust_branches()  local
1938 	for (i = 0; i < insn_cnt; i++, insn++) {  in adjust_branches()
1939 		if (BPF_CLASS(insn->code) != BPF_JMP ||  in adjust_branches()
1940 		    BPF_OP(insn->code) == BPF_CALL ||  in adjust_branches()
1941 		    BPF_OP(insn->code) == BPF_EXIT)  in adjust_branches()
1945 		if (i < pos && i + insn->off + 1 > pos)  in adjust_branches()
1946 			insn->off += delta;  in adjust_branches()
1947 		else if (i > pos + delta && i + insn->off + 1 <= pos + delta)  in adjust_branches()
1948 			insn->off -= delta;  in adjust_branches()
1957 	struct bpf_insn *insn = env->prog->insnsi;  in convert_ctx_accesses()  local
1967 	for (i = 0; i < insn_cnt; i++, insn++) {  in convert_ctx_accesses()
1968 		if (insn->code != (BPF_LDX | BPF_MEM | BPF_W))  in convert_ctx_accesses()
1971 		if (insn->imm != PTR_TO_CTX) {  in convert_ctx_accesses()
1973 			insn->imm = 0;  in convert_ctx_accesses()
1978 			convert_ctx_access(insn->dst_reg, insn->src_reg,  in convert_ctx_accesses()
1979 					   insn->off, insn_buf);  in convert_ctx_accesses()
1986 			memcpy(insn, insn_buf, sizeof(*insn));  in convert_ctx_accesses()
2001 			sizeof(*insn) * (insn_cnt - i - cnt));  in convert_ctx_accesses()
2004 		memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);  in convert_ctx_accesses()
2011 		insn = new_prog->insnsi + i + cnt - 1;  in convert_ctx_accesses()