316 static void print_bpf_insn(struct bpf_insn *insn)  in print_bpf_insn()  argument
318 	u8 class = BPF_CLASS(insn->code);  in print_bpf_insn()
321 		if (BPF_SRC(insn->code) == BPF_X)  in print_bpf_insn()
323 				insn->code, class == BPF_ALU ? "(u32) " : "",  in print_bpf_insn()
324 				insn->dst_reg,  in print_bpf_insn()
325 				bpf_alu_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
327 				insn->src_reg);  in print_bpf_insn()
330 				insn->code, class == BPF_ALU ? "(u32) " : "",  in print_bpf_insn()
331 				insn->dst_reg,  in print_bpf_insn()
332 				bpf_alu_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
334 				insn->imm);  in print_bpf_insn()
336 		if (BPF_MODE(insn->code) == BPF_MEM)  in print_bpf_insn()
338 				insn->code,  in print_bpf_insn()
339 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
340 				insn->dst_reg,  in print_bpf_insn()
341 				insn->off, insn->src_reg);  in print_bpf_insn()
342 		else if (BPF_MODE(insn->code) == BPF_XADD)  in print_bpf_insn()
344 				insn->code,  in print_bpf_insn()
345 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
346 				insn->dst_reg, insn->off,  in print_bpf_insn()
347 				insn->src_reg);  in print_bpf_insn()
349 			verbose("BUG_%02x\n", insn->code);  in print_bpf_insn()
351 		if (BPF_MODE(insn->code) != BPF_MEM) {  in print_bpf_insn()
352 			verbose("BUG_st_%02x\n", insn->code);  in print_bpf_insn()
356 			insn->code,  in print_bpf_insn()
357 			bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
358 			insn->dst_reg,  in print_bpf_insn()
359 			insn->off, insn->imm);  in print_bpf_insn()
361 		if (BPF_MODE(insn->code) != BPF_MEM) {  in print_bpf_insn()
362 			verbose("BUG_ldx_%02x\n", insn->code);  in print_bpf_insn()
366 			insn->code, insn->dst_reg,  in print_bpf_insn()
367 			bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
368 			insn->src_reg, insn->off);  in print_bpf_insn()
370 		if (BPF_MODE(insn->code) == BPF_ABS) {  in print_bpf_insn()
372 				insn->code,  in print_bpf_insn()
373 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
374 				insn->imm);  in print_bpf_insn()
375 		} else if (BPF_MODE(insn->code) == BPF_IND) {  in print_bpf_insn()
377 				insn->code,  in print_bpf_insn()
378 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],  in print_bpf_insn()
379 				insn->src_reg, insn->imm);  in print_bpf_insn()
380 		} else if (BPF_MODE(insn->code) == BPF_IMM) {  in print_bpf_insn()
382 				insn->code, insn->dst_reg, insn->imm);  in print_bpf_insn()
384 			verbose("BUG_ld_%02x\n", insn->code);  in print_bpf_insn()
388 		u8 opcode = BPF_OP(insn->code);  in print_bpf_insn()
391 			verbose("(%02x) call %d\n", insn->code, insn->imm);  in print_bpf_insn()
392 		} else if (insn->code == (BPF_JMP | BPF_JA)) {  in print_bpf_insn()
394 				insn->code, insn->off);  in print_bpf_insn()
395 		} else if (insn->code == (BPF_JMP | BPF_EXIT)) {  in print_bpf_insn()
396 			verbose("(%02x) exit\n", insn->code);  in print_bpf_insn()
397 		} else if (BPF_SRC(insn->code) == BPF_X) {  in print_bpf_insn()
399 				insn->code, insn->dst_reg,  in print_bpf_insn()
400 				bpf_jmp_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
401 				insn->src_reg, insn->off);  in print_bpf_insn()
404 				insn->code, insn->dst_reg,  in print_bpf_insn()
405 				bpf_jmp_string[BPF_OP(insn->code) >> 4],  in print_bpf_insn()
406 				insn->imm, insn->off);  in print_bpf_insn()
409 		verbose("(%02x) %s\n", insn->code, bpf_class_string[class]);  in print_bpf_insn()
736 static int check_xadd(struct verifier_env *env, struct bpf_insn *insn)  in check_xadd()  argument
741 	if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) ||  in check_xadd()
742 	    insn->imm != 0) {  in check_xadd()
748 	err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_xadd()
753 	err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_xadd()
758 	err = check_mem_access(env, insn->dst_reg, insn->off,  in check_xadd()
759 			       BPF_SIZE(insn->code), BPF_READ, -1);  in check_xadd()
764 	return check_mem_access(env, insn->dst_reg, insn->off,  in check_xadd()
765 				BPF_SIZE(insn->code), BPF_WRITE, -1);  in check_xadd()
1015 static int check_alu_op(struct verifier_env *env, struct bpf_insn *insn)  in check_alu_op()  argument
1018 	u8 opcode = BPF_OP(insn->code);  in check_alu_op()
1023 			if (BPF_SRC(insn->code) != 0 ||  in check_alu_op()
1024 			    insn->src_reg != BPF_REG_0 ||  in check_alu_op()
1025 			    insn->off != 0 || insn->imm != 0) {  in check_alu_op()
1030 			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||  in check_alu_op()
1031 			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64)) {  in check_alu_op()
1038 		err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_alu_op()
1042 		if (is_pointer_value(env, insn->dst_reg)) {  in check_alu_op()
1044 				insn->dst_reg);  in check_alu_op()
1049 		err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_alu_op()
1055 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
1056 			if (insn->imm != 0 || insn->off != 0) {  in check_alu_op()
1062 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_alu_op()
1066 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {  in check_alu_op()
1073 		err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_alu_op()
1077 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
1078 			if (BPF_CLASS(insn->code) == BPF_ALU64) {  in check_alu_op()
1082 				regs[insn->dst_reg] = regs[insn->src_reg];  in check_alu_op()
1084 				if (is_pointer_value(env, insn->src_reg)) {  in check_alu_op()
1086 						insn->src_reg);  in check_alu_op()
1089 				regs[insn->dst_reg].type = UNKNOWN_VALUE;  in check_alu_op()
1090 				regs[insn->dst_reg].map_ptr = NULL;  in check_alu_op()
1096 			regs[insn->dst_reg].type = CONST_IMM;  in check_alu_op()
1097 			regs[insn->dst_reg].imm = insn->imm;  in check_alu_op()
1108 		if (BPF_SRC(insn->code) == BPF_X) {  in check_alu_op()
1109 			if (insn->imm != 0 || insn->off != 0) {  in check_alu_op()
1114 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_alu_op()
1118 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {  in check_alu_op()
1125 		err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_alu_op()
1130 		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {  in check_alu_op()
1136 		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {  in check_alu_op()
1137 			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;  in check_alu_op()
1139 			if (insn->imm < 0 || insn->imm >= size) {  in check_alu_op()
1140 				verbose("invalid shift %d\n", insn->imm);  in check_alu_op()
1146 		if (opcode == BPF_ADD && BPF_CLASS(insn->code) == BPF_ALU64 &&  in check_alu_op()
1147 		    regs[insn->dst_reg].type == FRAME_PTR &&  in check_alu_op()
1148 		    BPF_SRC(insn->code) == BPF_K) {  in check_alu_op()
1150 		} else if (is_pointer_value(env, insn->dst_reg)) {  in check_alu_op()
1152 				insn->dst_reg);  in check_alu_op()
1154 		} else if (BPF_SRC(insn->code) == BPF_X &&  in check_alu_op()
1155 			   is_pointer_value(env, insn->src_reg)) {  in check_alu_op()
1157 				insn->src_reg);  in check_alu_op()
1162 		err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_alu_op()
1167 			regs[insn->dst_reg].type = PTR_TO_STACK;  in check_alu_op()
1168 			regs[insn->dst_reg].imm = insn->imm;  in check_alu_op()
1176 			     struct bpf_insn *insn, int *insn_idx)  in check_cond_jmp_op()  argument
1180 	u8 opcode = BPF_OP(insn->code);  in check_cond_jmp_op()
1188 	if (BPF_SRC(insn->code) == BPF_X) {  in check_cond_jmp_op()
1189 		if (insn->imm != 0) {  in check_cond_jmp_op()
1195 		err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_cond_jmp_op()
1199 		if (is_pointer_value(env, insn->src_reg)) {  in check_cond_jmp_op()
1201 				insn->src_reg);  in check_cond_jmp_op()
1205 		if (insn->src_reg != BPF_REG_0) {  in check_cond_jmp_op()
1212 	err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in check_cond_jmp_op()
1217 	if (BPF_SRC(insn->code) == BPF_K &&  in check_cond_jmp_op()
1219 	    regs[insn->dst_reg].type == CONST_IMM &&  in check_cond_jmp_op()
1220 	    regs[insn->dst_reg].imm == insn->imm) {  in check_cond_jmp_op()
1225 			*insn_idx += insn->off;  in check_cond_jmp_op()
1236 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx);  in check_cond_jmp_op()
1241 	if (BPF_SRC(insn->code) == BPF_K &&  in check_cond_jmp_op()
1242 	    insn->imm == 0 && (opcode == BPF_JEQ ||  in check_cond_jmp_op()
1244 	    regs[insn->dst_reg].type == PTR_TO_MAP_VALUE_OR_NULL) {  in check_cond_jmp_op()
1249 			regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;  in check_cond_jmp_op()
1251 			other_branch->regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1252 			other_branch->regs[insn->dst_reg].imm = 0;  in check_cond_jmp_op()
1254 			other_branch->regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;  in check_cond_jmp_op()
1255 			regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1256 			regs[insn->dst_reg].imm = 0;  in check_cond_jmp_op()
1258 	} else if (is_pointer_value(env, insn->dst_reg)) {  in check_cond_jmp_op()
1259 		verbose("R%d pointer comparison prohibited\n", insn->dst_reg);  in check_cond_jmp_op()
1261 	} else if (BPF_SRC(insn->code) == BPF_K &&  in check_cond_jmp_op()
1268 			other_branch->regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1269 			other_branch->regs[insn->dst_reg].imm = insn->imm;  in check_cond_jmp_op()
1274 			regs[insn->dst_reg].type = CONST_IMM;  in check_cond_jmp_op()
1275 			regs[insn->dst_reg].imm = insn->imm;  in check_cond_jmp_op()
1284 static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn)  in ld_imm64_to_map_ptr()  argument
1286 	u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32;  in ld_imm64_to_map_ptr()
1292 static int check_ld_imm(struct verifier_env *env, struct bpf_insn *insn)  in check_ld_imm()  argument
1297 	if (BPF_SIZE(insn->code) != BPF_DW) {  in check_ld_imm()
1301 	if (insn->off != 0) {  in check_ld_imm()
1306 	err = check_reg_arg(regs, insn->dst_reg, DST_OP);  in check_ld_imm()
1310 	if (insn->src_reg == 0)  in check_ld_imm()
1315 	BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD);  in check_ld_imm()
1317 	regs[insn->dst_reg].type = CONST_PTR_TO_MAP;  in check_ld_imm()
1318 	regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn);  in check_ld_imm()
1349 static int check_ld_abs(struct verifier_env *env, struct bpf_insn *insn)  in check_ld_abs()  argument
1352 	u8 mode = BPF_MODE(insn->code);  in check_ld_abs()
1361 	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||  in check_ld_abs()
1362 	    BPF_SIZE(insn->code) == BPF_DW ||  in check_ld_abs()
1363 	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {  in check_ld_abs()
1380 		err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in check_ld_abs()
1719 		struct bpf_insn *insn;  in do_check()  local
1729 		insn = &insns[insn_idx];  in do_check()
1730 		class = BPF_CLASS(insn->code);  in do_check()
1761 			print_bpf_insn(insn);  in do_check()
1765 			err = check_alu_op(env, insn);  in do_check()
1775 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in do_check()
1779 			err = check_reg_arg(regs, insn->dst_reg, DST_OP_NO_MARK);  in do_check()
1783 			src_reg_type = regs[insn->src_reg].type;  in do_check()
1788 			err = check_mem_access(env, insn->src_reg, insn->off,  in do_check()
1789 					       BPF_SIZE(insn->code), BPF_READ,  in do_check()
1790 					       insn->dst_reg);  in do_check()
1794 			if (BPF_SIZE(insn->code) != BPF_W) {  in do_check()
1799 			if (insn->imm == 0) {  in do_check()
1804 				insn->imm = src_reg_type;  in do_check()
1806 			} else if (src_reg_type != insn->imm &&  in do_check()
1808 				    insn->imm == PTR_TO_CTX)) {  in do_check()
1823 			if (BPF_MODE(insn->code) == BPF_XADD) {  in do_check()
1824 				err = check_xadd(env, insn);  in do_check()
1832 			err = check_reg_arg(regs, insn->src_reg, SRC_OP);  in do_check()
1836 			err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in do_check()
1840 			dst_reg_type = regs[insn->dst_reg].type;  in do_check()
1843 			err = check_mem_access(env, insn->dst_reg, insn->off,  in do_check()
1844 					       BPF_SIZE(insn->code), BPF_WRITE,  in do_check()
1845 					       insn->src_reg);  in do_check()
1849 			if (insn->imm == 0) {  in do_check()
1850 				insn->imm = dst_reg_type;  in do_check()
1851 			} else if (dst_reg_type != insn->imm &&  in do_check()
1853 				    insn->imm == PTR_TO_CTX)) {  in do_check()
1859 			if (BPF_MODE(insn->code) != BPF_MEM ||  in do_check()
1860 			    insn->src_reg != BPF_REG_0) {  in do_check()
1865 			err = check_reg_arg(regs, insn->dst_reg, SRC_OP);  in do_check()
1870 			err = check_mem_access(env, insn->dst_reg, insn->off,  in do_check()
1871 					       BPF_SIZE(insn->code), BPF_WRITE,  in do_check()
1877 			u8 opcode = BPF_OP(insn->code);  in do_check()
1880 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
1881 				    insn->off != 0 ||  in do_check()
1882 				    insn->src_reg != BPF_REG_0 ||  in do_check()
1883 				    insn->dst_reg != BPF_REG_0) {  in do_check()
1888 				err = check_call(env, insn->imm);  in do_check()
1893 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
1894 				    insn->imm != 0 ||  in do_check()
1895 				    insn->src_reg != BPF_REG_0 ||  in do_check()
1896 				    insn->dst_reg != BPF_REG_0) {  in do_check()
1901 				insn_idx += insn->off + 1;  in do_check()
1905 				if (BPF_SRC(insn->code) != BPF_K ||  in do_check()
1906 				    insn->imm != 0 ||  in do_check()
1907 				    insn->src_reg != BPF_REG_0 ||  in do_check()
1908 				    insn->dst_reg != BPF_REG_0) {  in do_check()
1937 				err = check_cond_jmp_op(env, insn, &insn_idx);  in do_check()
1942 			u8 mode = BPF_MODE(insn->code);  in do_check()
1945 				err = check_ld_abs(env, insn);  in do_check()
1950 				err = check_ld_imm(env, insn);  in do_check()
1975 	struct bpf_insn *insn = env->prog->insnsi;  in replace_map_fd_with_map_ptr()  local
1979 	for (i = 0; i < insn_cnt; i++, insn++) {  in replace_map_fd_with_map_ptr()
1980 		if (BPF_CLASS(insn->code) == BPF_LDX &&  in replace_map_fd_with_map_ptr()
1981 		    (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {  in replace_map_fd_with_map_ptr()
1986 		if (BPF_CLASS(insn->code) == BPF_STX &&  in replace_map_fd_with_map_ptr()
1987 		    ((BPF_MODE(insn->code) != BPF_MEM &&  in replace_map_fd_with_map_ptr()
1988 		      BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) {  in replace_map_fd_with_map_ptr()
1993 		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {  in replace_map_fd_with_map_ptr()
1997 			if (i == insn_cnt - 1 || insn[1].code != 0 ||  in replace_map_fd_with_map_ptr()
1998 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||  in replace_map_fd_with_map_ptr()
1999 			    insn[1].off != 0) {  in replace_map_fd_with_map_ptr()
2004 			if (insn->src_reg == 0)  in replace_map_fd_with_map_ptr()
2008 			if (insn->src_reg != BPF_PSEUDO_MAP_FD) {  in replace_map_fd_with_map_ptr()
2013 			f = fdget(insn->imm);  in replace_map_fd_with_map_ptr()
2017 					insn->imm);  in replace_map_fd_with_map_ptr()
2022 			insn[0].imm = (u32) (unsigned long) map;  in replace_map_fd_with_map_ptr()
2023 			insn[1].imm = ((u64) (unsigned long) map) >> 32;  in replace_map_fd_with_map_ptr()
2051 			insn++;  in replace_map_fd_with_map_ptr()
2075 	struct bpf_insn *insn = env->prog->insnsi;  in convert_pseudo_ld_imm64()  local
2079 	for (i = 0; i < insn_cnt; i++, insn++)  in convert_pseudo_ld_imm64()
2080 		if (insn->code == (BPF_LD | BPF_IMM | BPF_DW))  in convert_pseudo_ld_imm64()
2081 			insn->src_reg = 0;  in convert_pseudo_ld_imm64()
2086 	struct bpf_insn *insn = prog->insnsi;  in adjust_branches()  local
2090 	for (i = 0; i < insn_cnt; i++, insn++) {  in adjust_branches()
2091 		if (BPF_CLASS(insn->code) != BPF_JMP ||  in adjust_branches()
2092 		    BPF_OP(insn->code) == BPF_CALL ||  in adjust_branches()
2093 		    BPF_OP(insn->code) == BPF_EXIT)  in adjust_branches()
2097 		if (i < pos && i + insn->off + 1 > pos)  in adjust_branches()
2098 			insn->off += delta;  in adjust_branches()
2099 		else if (i > pos + delta && i + insn->off + 1 <= pos + delta)  in adjust_branches()
2100 			insn->off -= delta;  in adjust_branches()
2109 	struct bpf_insn *insn = env->prog->insnsi;  in convert_ctx_accesses()  local
2120 	for (i = 0; i < insn_cnt; i++, insn++) {  in convert_ctx_accesses()
2121 		if (insn->code == (BPF_LDX | BPF_MEM | BPF_W))  in convert_ctx_accesses()
2123 		else if (insn->code == (BPF_STX | BPF_MEM | BPF_W))  in convert_ctx_accesses()
2128 		if (insn->imm != PTR_TO_CTX) {  in convert_ctx_accesses()
2130 			insn->imm = 0;  in convert_ctx_accesses()
2135 			convert_ctx_access(type, insn->dst_reg, insn->src_reg,  in convert_ctx_accesses()
2136 					   insn->off, insn_buf, env->prog);  in convert_ctx_accesses()
2143 			memcpy(insn, insn_buf, sizeof(*insn));  in convert_ctx_accesses()
2158 			sizeof(*insn) * (insn_cnt - i - cnt));  in convert_ctx_accesses()
2161 		memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);  in convert_ctx_accesses()
2168 		insn = new_prog->insnsi + i + cnt - 1;  in convert_ctx_accesses()