104 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)  in reg_set_seen()  argument
108 	if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15)  in reg_set_seen()
109 		jit->seen_reg[r1] = 1;  in reg_set_seen()
114 	reg_set_seen(jit, b1);					\
117 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
125 	if (jit->prg_buf)					\
126 		*(u16 *) (jit->prg_buf + jit->prg) = op;	\
127 	jit->prg += 2;						\
139 	if (jit->prg_buf)					\
140 		*(u32 *) (jit->prg_buf + jit->prg) = op;	\
141 	jit->prg += 4;						\
188 	if (jit->prg_buf) {					\
189 		*(u32 *) (jit->prg_buf + jit->prg) = op1;	\
190 		*(u16 *) (jit->prg_buf + jit->prg + 4) = op2;	\
192 	jit->prg += 6;						\
250 	ret = jit->lit - jit->base_ip;				\
251 	jit->seen |= SEEN_LITERAL;				\
252 	if (jit->prg_buf)					\
253 		*(u32 *) (jit->prg_buf + jit->lit) = (u32) val;	\
254 	jit->lit += 4;						\
261 	ret = jit->lit - jit->base_ip;				\
262 	jit->seen |= SEEN_LITERAL;				\
263 	if (jit->prg_buf)					\
264 		*(u64 *) (jit->prg_buf + jit->lit) = (u64) val;	\
265 	jit->lit += 8;						\
287 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)  in save_regs()  argument
302 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)  in restore_regs()  argument
306 	if (jit->seen & SEEN_STACK)  in restore_regs()
320 static int get_start(struct bpf_jit *jit, int start)  in get_start()  argument
325 		if (jit->seen_reg[i])  in get_start()
334 static int get_end(struct bpf_jit *jit, int start)  in get_end()  argument
339 		if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])  in get_end()
342 	return jit->seen_reg[15] ? 15 : 14;  in get_end()
351 static void save_restore_regs(struct bpf_jit *jit, int op)  in save_restore_regs()  argument
357 		rs = get_start(jit, re);  in save_restore_regs()
360 		re = get_end(jit, rs + 1);  in save_restore_regs()
362 			save_regs(jit, rs, re);  in save_restore_regs()
364 			restore_regs(jit, rs, re);  in save_restore_regs()
375 static void bpf_jit_prologue(struct bpf_jit *jit)  in bpf_jit_prologue()  argument
378 	save_restore_regs(jit, REGS_SAVE);  in bpf_jit_prologue()
380 	if (jit->seen & SEEN_LITERAL) {  in bpf_jit_prologue()
383 		jit->base_ip = jit->prg;  in bpf_jit_prologue()
386 	if (jit->seen & SEEN_STACK) {  in bpf_jit_prologue()
387 		if (jit->seen & SEEN_FUNC)  in bpf_jit_prologue()
394 		if (jit->seen & SEEN_FUNC)  in bpf_jit_prologue()
404 	if (jit->seen & SEEN_SKB) {  in bpf_jit_prologue()
430 static void bpf_jit_epilogue(struct bpf_jit *jit)  in bpf_jit_epilogue()  argument
433 	if (jit->seen & SEEN_RET0) {  in bpf_jit_epilogue()
434 		jit->ret0_ip = jit->prg;  in bpf_jit_epilogue()
438 	jit->exit_ip = jit->prg;  in bpf_jit_epilogue()
442 	save_restore_regs(jit, REGS_RESTORE);  in bpf_jit_epilogue()
453 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i)  in bpf_jit_insn()  argument
460 	u32 *addrs = jit->addrs;  in bpf_jit_insn()
582 		jit->seen |= SEEN_RET0;  in bpf_jit_insn()
586 		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);  in bpf_jit_insn()
602 		jit->seen |= SEEN_RET0;  in bpf_jit_insn()
606 		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);  in bpf_jit_insn()
850 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
855 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
860 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
865 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
872 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
879 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
886 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
893 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
902 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
908 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
916 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
921 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
925 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
930 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
944 		jit->seen |= SEEN_FUNC;  in bpf_jit_insn()
956 		if (last && !(jit->seen & SEEN_RET0))  in bpf_jit_insn()
959 		EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);  in bpf_jit_insn()
1083 		jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC;  in bpf_jit_insn()
1118 		EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);  in bpf_jit_insn()
1130 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp)  in bpf_jit_prog()  argument
1134 	jit->lit = jit->lit_start;  in bpf_jit_prog()
1135 	jit->prg = 0;  in bpf_jit_prog()
1137 	bpf_jit_prologue(jit);  in bpf_jit_prog()
1139 		insn_count = bpf_jit_insn(jit, fp, i);  in bpf_jit_prog()
1142 		jit->addrs[i + 1] = jit->prg; /* Next instruction address */  in bpf_jit_prog()
1144 	bpf_jit_epilogue(jit);  in bpf_jit_prog()
1146 	jit->lit_start = jit->prg;  in bpf_jit_prog()
1147 	jit->size = jit->lit;  in bpf_jit_prog()
1148 	jit->size_prg = jit->prg;  in bpf_jit_prog()
1166 	struct bpf_jit jit;  in bpf_int_jit_compile()  local
1171 	memset(&jit, 0, sizeof(jit));  in bpf_int_jit_compile()
1172 	jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);  in bpf_int_jit_compile()
1173 	if (jit.addrs == NULL)  in bpf_int_jit_compile()
1181 		if (bpf_jit_prog(&jit, fp))  in bpf_int_jit_compile()
1187 	if (jit.size >= BPF_SIZE_MAX)  in bpf_int_jit_compile()
1189 	header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);  in bpf_int_jit_compile()
1192 	if (bpf_jit_prog(&jit, fp))  in bpf_int_jit_compile()
1195 		bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);  in bpf_int_jit_compile()
1196 		if (jit.prg_buf)  in bpf_int_jit_compile()
1197 			print_fn_code(jit.prg_buf, jit.size_prg);  in bpf_int_jit_compile()
1199 	if (jit.prg_buf) {  in bpf_int_jit_compile()
1201 		fp->bpf_func = (void *) jit.prg_buf;  in bpf_int_jit_compile()
1205 	kfree(jit.addrs);  in bpf_int_jit_compile()