110 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)  in reg_set_seen()  argument
114 	if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15)  in reg_set_seen()
115 		jit->seen_reg[r1] = 1;  in reg_set_seen()
120 	reg_set_seen(jit, b1);					\
123 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
131 	if (jit->prg_buf)					\
132 		*(u16 *) (jit->prg_buf + jit->prg) = op;	\
133 	jit->prg += 2;						\
145 	if (jit->prg_buf)					\
146 		*(u32 *) (jit->prg_buf + jit->prg) = op;	\
147 	jit->prg += 4;						\
194 	if (jit->prg_buf) {					\
195 		*(u32 *) (jit->prg_buf + jit->prg) = op1;	\
196 		*(u16 *) (jit->prg_buf + jit->prg + 4) = op2;	\
198 	jit->prg += 6;						\
226 	int rel = (jit->labels[label] - jit->prg) >> 1;		\
235 	int rel = (jit->labels[label] - jit->prg) >> 1;		\
266 	ret = jit->lit - jit->base_ip;				\
267 	jit->seen |= SEEN_LITERAL;				\
268 	if (jit->prg_buf)					\
269 		*(u32 *) (jit->prg_buf + jit->lit) = (u32) val;	\
270 	jit->lit += 4;						\
277 	ret = jit->lit - jit->base_ip;				\
278 	jit->seen |= SEEN_LITERAL;				\
279 	if (jit->prg_buf)					\
280 		*(u64 *) (jit->prg_buf + jit->lit) = (u64) val;	\
281 	jit->lit += 8;						\
303 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)  in save_regs()  argument
318 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)  in restore_regs()  argument
322 	if (jit->seen & SEEN_STACK)  in restore_regs()
336 static int get_start(struct bpf_jit *jit, int start)  in get_start()  argument
341 		if (jit->seen_reg[i])  in get_start()
350 static int get_end(struct bpf_jit *jit, int start)  in get_end()  argument
355 		if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])  in get_end()
358 	return jit->seen_reg[15] ? 15 : 14;  in get_end()
367 static void save_restore_regs(struct bpf_jit *jit, int op)  in save_restore_regs()  argument
373 		rs = get_start(jit, re);  in save_restore_regs()
376 		re = get_end(jit, rs + 1);  in save_restore_regs()
378 			save_regs(jit, rs, re);  in save_restore_regs()
380 			restore_regs(jit, rs, re);  in save_restore_regs()
390 static void emit_load_skb_data_hlen(struct bpf_jit *jit)  in emit_load_skb_data_hlen()  argument
411 static void bpf_jit_prologue(struct bpf_jit *jit, bool is_classic)  in bpf_jit_prologue()  argument
413 	if (jit->seen & SEEN_TAIL_CALL) {  in bpf_jit_prologue()
422 	jit->tail_call_start = jit->prg;  in bpf_jit_prologue()
424 	save_restore_regs(jit, REGS_SAVE);  in bpf_jit_prologue()
426 	if (jit->seen & SEEN_LITERAL) {  in bpf_jit_prologue()
429 		jit->base_ip = jit->prg;  in bpf_jit_prologue()
432 	if (jit->seen & SEEN_STACK) {  in bpf_jit_prologue()
433 		if (jit->seen & SEEN_FUNC)  in bpf_jit_prologue()
440 		if (jit->seen & SEEN_FUNC)  in bpf_jit_prologue()
445 	if (jit->seen & SEEN_SKB)  in bpf_jit_prologue()
446 		emit_load_skb_data_hlen(jit);  in bpf_jit_prologue()
447 	if (jit->seen & SEEN_SKB_CHANGE)  in bpf_jit_prologue()
465 static void bpf_jit_epilogue(struct bpf_jit *jit)  in bpf_jit_epilogue()  argument
468 	if (jit->seen & SEEN_RET0) {  in bpf_jit_epilogue()
469 		jit->ret0_ip = jit->prg;  in bpf_jit_epilogue()
473 	jit->exit_ip = jit->prg;  in bpf_jit_epilogue()
477 	save_restore_regs(jit, REGS_RESTORE);  in bpf_jit_epilogue()
488 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i)  in bpf_jit_insn()  argument
495 	u32 *addrs = jit->addrs;  in bpf_jit_insn()
617 		jit->seen |= SEEN_RET0;  in bpf_jit_insn()
621 		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);  in bpf_jit_insn()
637 		jit->seen |= SEEN_RET0;  in bpf_jit_insn()
641 		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);  in bpf_jit_insn()
885 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
890 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
895 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
900 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
907 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
914 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
921 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
928 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
937 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
943 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
951 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
956 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
960 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
965 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
979 		jit->seen |= SEEN_FUNC;  in bpf_jit_insn()
988 			jit->seen |= SEEN_SKB_CHANGE;  in bpf_jit_insn()
992 			emit_load_skb_data_hlen(jit);  in bpf_jit_insn()
1003 		jit->seen |= SEEN_TAIL_CALL;  in bpf_jit_insn()
1022 		if (jit->seen & SEEN_STACK)  in bpf_jit_insn()
1051 		save_restore_regs(jit, REGS_RESTORE);  in bpf_jit_insn()
1061 		_EMIT4(0x47f01000 + jit->tail_call_start);  in bpf_jit_insn()
1063 		jit->labels[0] = jit->prg;  in bpf_jit_insn()
1067 		if (last && !(jit->seen & SEEN_RET0))  in bpf_jit_insn()
1070 		EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);  in bpf_jit_insn()
1194 		jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC;  in bpf_jit_insn()
1229 		EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);  in bpf_jit_insn()
1241 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp)  in bpf_jit_prog()  argument
1245 	jit->lit = jit->lit_start;  in bpf_jit_prog()
1246 	jit->prg = 0;  in bpf_jit_prog()
1248 	bpf_jit_prologue(jit, bpf_prog_was_classic(fp));  in bpf_jit_prog()
1250 		insn_count = bpf_jit_insn(jit, fp, i);  in bpf_jit_prog()
1253 		jit->addrs[i + 1] = jit->prg; /* Next instruction address */  in bpf_jit_prog()
1255 	bpf_jit_epilogue(jit);  in bpf_jit_prog()
1257 	jit->lit_start = jit->prg;  in bpf_jit_prog()
1258 	jit->size = jit->lit;  in bpf_jit_prog()
1259 	jit->size_prg = jit->prg;  in bpf_jit_prog()
1277 	struct bpf_jit jit;  in bpf_int_jit_compile()  local
1282 	memset(&jit, 0, sizeof(jit));  in bpf_int_jit_compile()
1283 	jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);  in bpf_int_jit_compile()
1284 	if (jit.addrs == NULL)  in bpf_int_jit_compile()
1292 		if (bpf_jit_prog(&jit, fp))  in bpf_int_jit_compile()
1298 	if (jit.size >= BPF_SIZE_MAX)  in bpf_int_jit_compile()
1300 	header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);  in bpf_int_jit_compile()
1303 	if (bpf_jit_prog(&jit, fp))  in bpf_int_jit_compile()
1306 		bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);  in bpf_int_jit_compile()
1307 		if (jit.prg_buf)  in bpf_int_jit_compile()
1308 			print_fn_code(jit.prg_buf, jit.size_prg);  in bpf_int_jit_compile()
1310 	if (jit.prg_buf) {  in bpf_int_jit_compile()
1312 		fp->bpf_func = (void *) jit.prg_buf;  in bpf_int_jit_compile()
1316 	kfree(jit.addrs);  in bpf_int_jit_compile()