Lines Matching refs:ctx
119 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx);
122 #define emit_instr(ctx, func, ...) \ argument
124 if ((ctx)->target != NULL) { \
125 u32 *p = &(ctx)->target[ctx->idx]; \
128 (ctx)->idx++; \
135 #define emit_long_instr(ctx, func, ...) \ argument
137 if ((ctx)->target != NULL) { \
138 u32 *p = &(ctx)->target[ctx->idx]; \
141 (ctx)->idx++; \
151 unsigned int src2, struct jit_ctx *ctx) in emit_addu() argument
153 emit_instr(ctx, addu, dst, src1, src2); in emit_addu()
156 static inline void emit_nop(struct jit_ctx *ctx) in emit_nop() argument
158 emit_instr(ctx, nop); in emit_nop()
162 static inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx) in emit_load_imm() argument
164 if (ctx->target != NULL) { in emit_load_imm()
167 u32 *p = &ctx->target[ctx->idx]; in emit_load_imm()
169 p = &ctx->target[ctx->idx + 1]; in emit_load_imm()
172 u32 *p = &ctx->target[ctx->idx]; in emit_load_imm()
176 ctx->idx++; in emit_load_imm()
179 ctx->idx++; in emit_load_imm()
183 unsigned int src2, struct jit_ctx *ctx) in emit_or() argument
185 emit_instr(ctx, or, dst, src1, src2); in emit_or()
189 struct jit_ctx *ctx) in emit_ori() argument
192 emit_load_imm(r_tmp, imm, ctx); in emit_ori()
193 emit_or(dst, src, r_tmp, ctx); in emit_ori()
195 emit_instr(ctx, ori, dst, src, imm); in emit_ori()
200 int imm, struct jit_ctx *ctx) in emit_daddiu() argument
206 emit_instr(ctx, daddiu, dst, src, imm); in emit_daddiu()
210 u32 imm, struct jit_ctx *ctx) in emit_addiu() argument
213 emit_load_imm(r_tmp, imm, ctx); in emit_addiu()
214 emit_addu(dst, r_tmp, src, ctx); in emit_addiu()
216 emit_instr(ctx, addiu, dst, src, imm); in emit_addiu()
221 unsigned int src2, struct jit_ctx *ctx) in emit_and() argument
223 emit_instr(ctx, and, dst, src1, src2); in emit_and()
227 u32 imm, struct jit_ctx *ctx) in emit_andi() argument
231 emit_load_imm(r_tmp, imm, ctx); in emit_andi()
232 emit_and(dst, src, r_tmp, ctx); in emit_andi()
234 emit_instr(ctx, andi, dst, src, imm); in emit_andi()
239 unsigned int src2, struct jit_ctx *ctx) in emit_xor() argument
241 emit_instr(ctx, xor, dst, src1, src2); in emit_xor()
244 static inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx) in emit_xori() argument
248 emit_load_imm(r_tmp, imm, ctx); in emit_xori()
249 emit_xor(dst, src, r_tmp, ctx); in emit_xori()
251 emit_instr(ctx, xori, dst, src, imm); in emit_xori()
255 static inline void emit_stack_offset(int offset, struct jit_ctx *ctx) in emit_stack_offset() argument
257 emit_long_instr(ctx, ADDIU, r_sp, r_sp, offset); in emit_stack_offset()
261 unsigned int src2, struct jit_ctx *ctx) in emit_subu() argument
263 emit_instr(ctx, subu, dst, src1, src2); in emit_subu()
266 static inline void emit_neg(unsigned int reg, struct jit_ctx *ctx) in emit_neg() argument
268 emit_subu(reg, r_zero, reg, ctx); in emit_neg()
272 unsigned int sa, struct jit_ctx *ctx) in emit_sllv() argument
274 emit_instr(ctx, sllv, dst, src, sa); in emit_sllv()
278 unsigned int sa, struct jit_ctx *ctx) in emit_sll() argument
283 emit_jit_reg_move(dst, r_zero, ctx); in emit_sll()
285 emit_instr(ctx, sll, dst, src, sa); in emit_sll()
289 unsigned int sa, struct jit_ctx *ctx) in emit_srlv() argument
291 emit_instr(ctx, srlv, dst, src, sa); in emit_srlv()
295 unsigned int sa, struct jit_ctx *ctx) in emit_srl() argument
300 emit_jit_reg_move(dst, r_zero, ctx); in emit_srl()
302 emit_instr(ctx, srl, dst, src, sa); in emit_srl()
306 unsigned int src2, struct jit_ctx *ctx) in emit_slt() argument
308 emit_instr(ctx, slt, dst, src1, src2); in emit_slt()
312 unsigned int src2, struct jit_ctx *ctx) in emit_sltu() argument
314 emit_instr(ctx, sltu, dst, src1, src2); in emit_sltu()
318 unsigned int imm, struct jit_ctx *ctx) in emit_sltiu() argument
322 emit_load_imm(r_tmp, imm, ctx); in emit_sltiu()
323 emit_sltu(dst, src, r_tmp, ctx); in emit_sltiu()
325 emit_instr(ctx, sltiu, dst, src, imm); in emit_sltiu()
333 struct jit_ctx *ctx) in emit_store_stack_reg() argument
335 emit_long_instr(ctx, SW, reg, offset, base); in emit_store_stack_reg()
339 struct jit_ctx *ctx) in emit_store() argument
341 emit_instr(ctx, sw, reg, offset, base); in emit_store()
346 struct jit_ctx *ctx) in emit_load_stack_reg() argument
348 emit_long_instr(ctx, LW, reg, offset, base); in emit_load_stack_reg()
352 unsigned int offset, struct jit_ctx *ctx) in emit_load() argument
354 emit_instr(ctx, lw, reg, offset, base); in emit_load()
358 unsigned int offset, struct jit_ctx *ctx) in emit_load_byte() argument
360 emit_instr(ctx, lb, reg, offset, base); in emit_load_byte()
364 unsigned int offset, struct jit_ctx *ctx) in emit_half_load() argument
366 emit_instr(ctx, lh, reg, offset, base); in emit_half_load()
370 unsigned int src2, struct jit_ctx *ctx) in emit_mul() argument
372 emit_instr(ctx, mul, dst, src1, src2); in emit_mul()
376 struct jit_ctx *ctx) in emit_div() argument
378 if (ctx->target != NULL) { in emit_div()
379 u32 *p = &ctx->target[ctx->idx]; in emit_div()
381 p = &ctx->target[ctx->idx + 1]; in emit_div()
384 ctx->idx += 2; /* 2 insts */ in emit_div()
388 struct jit_ctx *ctx) in emit_mod() argument
390 if (ctx->target != NULL) { in emit_mod()
391 u32 *p = &ctx->target[ctx->idx]; in emit_mod()
393 p = &ctx->target[ctx->idx + 1]; in emit_mod()
396 ctx->idx += 2; /* 2 insts */ in emit_mod()
400 unsigned int sa, struct jit_ctx *ctx) in emit_dsll() argument
402 emit_instr(ctx, dsll, dst, src, sa); in emit_dsll()
406 unsigned int sa, struct jit_ctx *ctx) in emit_dsrl32() argument
408 emit_instr(ctx, dsrl32, dst, src, sa); in emit_dsrl32()
412 struct jit_ctx *ctx) in emit_wsbh() argument
414 emit_instr(ctx, wsbh, dst, src); in emit_wsbh()
419 int imm, struct jit_ctx *ctx) in emit_load_ptr() argument
422 emit_long_instr(ctx, LW, dst, imm, src); in emit_load_ptr()
427 struct jit_ctx *ctx) in emit_load_func() argument
431 emit_load_imm(r_tmp, (u64)imm >> 32, ctx); in emit_load_func()
432 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ in emit_load_func()
433 emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx); in emit_load_func()
434 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ in emit_load_func()
435 emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx); in emit_load_func()
437 emit_load_imm(reg, imm, ctx); in emit_load_func()
442 static inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) in emit_reg_move() argument
444 emit_long_instr(ctx, ADDU, dst, src, r_zero); in emit_reg_move()
448 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) in emit_jit_reg_move() argument
450 emit_addu(dst, src, r_zero, ctx); in emit_jit_reg_move()
454 static inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx) in b_imm() argument
456 if (ctx->target == NULL) in b_imm()
472 return ctx->offsets[tgt] - in b_imm()
473 (ctx->idx * 4 - ctx->prologue_bytes) - 4; in b_imm()
477 unsigned int imm, struct jit_ctx *ctx) in emit_bcond() argument
479 if (ctx->target != NULL) { in emit_bcond()
480 u32 *p = &ctx->target[ctx->idx]; in emit_bcond()
497 ctx->idx++; in emit_bcond()
500 static inline void emit_b(unsigned int imm, struct jit_ctx *ctx) in emit_b() argument
502 emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx); in emit_b()
506 struct jit_ctx *ctx) in emit_jalr() argument
508 emit_instr(ctx, jalr, link, reg); in emit_jalr()
511 static inline void emit_jr(unsigned int reg, struct jit_ctx *ctx) in emit_jr() argument
513 emit_instr(ctx, jr, reg); in emit_jr()
524 static void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset) in save_bpf_jit_regs() argument
530 emit_stack_offset(-align_sp(offset), ctx); in save_bpf_jit_regs()
532 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; in save_bpf_jit_regs()
537 ctx); in save_bpf_jit_regs()
545 if (ctx->flags & SEEN_CALL) { in save_bpf_jit_regs()
546 emit_store_stack_reg(r_ra, r_sp, real_off, ctx); in save_bpf_jit_regs()
551 if (ctx->flags & SEEN_MEM) { in save_bpf_jit_regs()
554 emit_long_instr(ctx, ADDIU, r_M, r_sp, real_off); in save_bpf_jit_regs()
558 static void restore_bpf_jit_regs(struct jit_ctx *ctx, in restore_bpf_jit_regs() argument
564 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; in restore_bpf_jit_regs()
570 ctx); in restore_bpf_jit_regs()
578 if (ctx->flags & SEEN_CALL) in restore_bpf_jit_regs()
579 emit_load_stack_reg(r_ra, r_sp, real_off, ctx); in restore_bpf_jit_regs()
582 emit_stack_offset(align_sp(offset), ctx); in restore_bpf_jit_regs()
585 static unsigned int get_stack_depth(struct jit_ctx *ctx) in get_stack_depth() argument
591 sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * SZREG; in get_stack_depth()
593 if (ctx->flags & SEEN_MEM) in get_stack_depth()
596 if (ctx->flags & SEEN_CALL) in get_stack_depth()
602 static void build_prologue(struct jit_ctx *ctx) in build_prologue() argument
607 sp_off = get_stack_depth(ctx); in build_prologue()
608 save_bpf_jit_regs(ctx, sp_off); in build_prologue()
610 if (ctx->flags & SEEN_SKB) in build_prologue()
611 emit_reg_move(r_skb, MIPS_R_A0, ctx); in build_prologue()
613 if (ctx->flags & SEEN_SKB_DATA) { in build_prologue()
616 ctx); in build_prologue()
618 ctx); in build_prologue()
621 offsetof(struct sk_buff, data), ctx); in build_prologue()
623 emit_subu(r_skb_hl, r_skb_len, r_tmp, ctx); in build_prologue()
626 if (ctx->flags & SEEN_X) in build_prologue()
627 emit_jit_reg_move(r_X, r_zero, ctx); in build_prologue()
630 if (bpf_needs_clear_a(&ctx->skf->insns[0])) in build_prologue()
631 emit_jit_reg_move(r_A, r_zero, ctx); in build_prologue()
634 static void build_epilogue(struct jit_ctx *ctx) in build_epilogue() argument
640 sp_off = get_stack_depth(ctx); in build_epilogue()
641 restore_bpf_jit_regs(ctx, sp_off); in build_epilogue()
644 emit_jr(r_ra, ctx); in build_epilogue()
645 emit_nop(ctx); in build_epilogue()
652 static int build_body(struct jit_ctx *ctx) in build_body() argument
654 const struct bpf_prog *prog = ctx->skf; in build_body()
669 if (ctx->target == NULL) in build_body()
670 ctx->offsets[i] = ctx->idx * 4; in build_body()
675 ctx->flags |= SEEN_A; in build_body()
676 emit_load_imm(r_A, k, ctx); in build_body()
681 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
683 emit_load(r_A, r_skb, off, ctx); in build_body()
687 ctx->flags |= SEEN_MEM | SEEN_A; in build_body()
688 emit_load(r_A, r_M, SCRATCH_OFF(k), ctx); in build_body()
702 emit_load_imm(r_off, k, ctx); in build_body()
704 ctx->flags |= SEEN_CALL | SEEN_OFF | in build_body()
707 emit_load_func(r_s0, (ptr)sk_load_func, ctx); in build_body()
708 emit_reg_move(MIPS_R_A0, r_skb, ctx); in build_body()
709 emit_jalr(MIPS_R_RA, r_s0, ctx); in build_body()
711 emit_reg_move(MIPS_R_A1, r_off, ctx); in build_body()
713 emit_bcond(MIPS_COND_EQ, r_ret, 0, b_imm(i + 1, ctx), in build_body()
714 ctx); in build_body()
716 emit_reg_move(r_ret, r_zero, ctx); in build_body()
718 emit_b(b_imm(prog->len, ctx), ctx); in build_body()
719 emit_nop(ctx); in build_body()
733 ctx->flags |= SEEN_OFF | SEEN_X; in build_body()
734 emit_addiu(r_off, r_X, k, ctx); in build_body()
738 ctx->flags |= SEEN_X; in build_body()
739 emit_load_imm(r_X, k, ctx); in build_body()
743 ctx->flags |= SEEN_X | SEEN_MEM; in build_body()
744 emit_load(r_X, r_M, SCRATCH_OFF(k), ctx); in build_body()
748 ctx->flags |= SEEN_X | SEEN_SKB; in build_body()
750 emit_load(r_X, r_skb, off, ctx); in build_body()
754 ctx->flags |= SEEN_X | SEEN_CALL | SEEN_SKB; in build_body()
756 emit_load_func(r_s0, (ptr)sk_load_byte, ctx); in build_body()
761 emit_load_imm(MIPS_R_A1, k, ctx); in build_body()
762 emit_jalr(MIPS_R_RA, r_s0, ctx); in build_body()
763 emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */ in build_body()
766 b_imm(prog->len, ctx), ctx); in build_body()
767 emit_reg_move(r_ret, r_zero, ctx); in build_body()
770 emit_andi(r_X, r_A, 0xf, ctx); in build_body()
772 emit_b(b_imm(i + 1, ctx), ctx); in build_body()
773 emit_sll(r_X, r_X, 2, ctx); /* delay slot */ in build_body()
777 ctx->flags |= SEEN_MEM | SEEN_A; in build_body()
778 emit_store(r_A, r_M, SCRATCH_OFF(k), ctx); in build_body()
782 ctx->flags |= SEEN_MEM | SEEN_X; in build_body()
783 emit_store(r_X, r_M, SCRATCH_OFF(k), ctx); in build_body()
787 ctx->flags |= SEEN_A; in build_body()
788 emit_addiu(r_A, r_A, k, ctx); in build_body()
792 ctx->flags |= SEEN_A | SEEN_X; in build_body()
793 emit_addu(r_A, r_A, r_X, ctx); in build_body()
797 ctx->flags |= SEEN_A; in build_body()
798 emit_addiu(r_A, r_A, -k, ctx); in build_body()
802 ctx->flags |= SEEN_A | SEEN_X; in build_body()
803 emit_subu(r_A, r_A, r_X, ctx); in build_body()
808 ctx->flags |= SEEN_A; in build_body()
809 emit_load_imm(r_s0, k, ctx); in build_body()
810 emit_mul(r_A, r_A, r_s0, ctx); in build_body()
814 ctx->flags |= SEEN_A | SEEN_X; in build_body()
815 emit_mul(r_A, r_A, r_X, ctx); in build_body()
822 ctx->flags |= SEEN_A; in build_body()
823 emit_srl(r_A, r_A, k, ctx); in build_body()
826 ctx->flags |= SEEN_A; in build_body()
827 emit_load_imm(r_s0, k, ctx); in build_body()
828 emit_div(r_A, r_s0, ctx); in build_body()
833 ctx->flags |= SEEN_A; in build_body()
834 emit_jit_reg_move(r_A, r_zero, ctx); in build_body()
836 ctx->flags |= SEEN_A; in build_body()
837 emit_load_imm(r_s0, k, ctx); in build_body()
838 emit_mod(r_A, r_s0, ctx); in build_body()
843 ctx->flags |= SEEN_X | SEEN_A; in build_body()
846 b_imm(prog->len, ctx), ctx); in build_body()
847 emit_load_imm(r_ret, 0, ctx); /* delay slot */ in build_body()
848 emit_div(r_A, r_X, ctx); in build_body()
852 ctx->flags |= SEEN_X | SEEN_A; in build_body()
855 b_imm(prog->len, ctx), ctx); in build_body()
856 emit_load_imm(r_ret, 0, ctx); /* delay slot */ in build_body()
857 emit_mod(r_A, r_X, ctx); in build_body()
861 ctx->flags |= SEEN_A; in build_body()
862 emit_ori(r_A, r_A, k, ctx); in build_body()
866 ctx->flags |= SEEN_A; in build_body()
867 emit_ori(r_A, r_A, r_X, ctx); in build_body()
871 ctx->flags |= SEEN_A; in build_body()
872 emit_xori(r_A, r_A, k, ctx); in build_body()
877 ctx->flags |= SEEN_A; in build_body()
878 emit_xor(r_A, r_A, r_X, ctx); in build_body()
882 ctx->flags |= SEEN_A; in build_body()
883 emit_andi(r_A, r_A, k, ctx); in build_body()
887 ctx->flags |= SEEN_A | SEEN_X; in build_body()
888 emit_and(r_A, r_A, r_X, ctx); in build_body()
892 ctx->flags |= SEEN_A; in build_body()
893 emit_sll(r_A, r_A, k, ctx); in build_body()
897 ctx->flags |= SEEN_A | SEEN_X; in build_body()
898 emit_sllv(r_A, r_A, r_X, ctx); in build_body()
902 ctx->flags |= SEEN_A; in build_body()
903 emit_srl(r_A, r_A, k, ctx); in build_body()
906 ctx->flags |= SEEN_A | SEEN_X; in build_body()
907 emit_srlv(r_A, r_A, r_X, ctx); in build_body()
911 ctx->flags |= SEEN_A; in build_body()
912 emit_neg(r_A, ctx); in build_body()
916 emit_b(b_imm(i + k + 1, ctx), ctx); in build_body()
917 emit_nop(ctx); in build_body()
924 ctx->flags |= SEEN_X; in build_body()
933 ctx->flags |= SEEN_X; in build_body()
942 ctx->flags |= SEEN_X; in build_body()
950 ctx->flags |= SEEN_A; in build_body()
951 emit_sltiu(r_s0, r_A, k, ctx); in build_body()
953 ctx->flags |= SEEN_A | in build_body()
955 emit_sltu(r_s0, r_A, r_X, ctx); in build_body()
958 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
960 ctx); in build_body()
961 emit_nop(ctx); in build_body()
965 ctx->flags |= SEEN_A | SEEN_X; in build_body()
967 emit_load_imm(r_s0, k, ctx); in build_body()
970 ctx); in build_body()
971 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
973 b_off, ctx); in build_body()
974 emit_nop(ctx); in build_body()
976 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
977 emit_b(b_off, ctx); in build_body()
978 emit_nop(ctx); in build_body()
981 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
982 emit_b(b_off, ctx); in build_body()
983 emit_nop(ctx); in build_body()
988 ctx->flags |= SEEN_A; in build_body()
989 emit_load_imm(r_s0, k, ctx); in build_body()
991 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
993 b_off, ctx); in build_body()
994 emit_nop(ctx); in build_body()
997 ctx); in build_body()
999 b_off, ctx); in build_body()
1000 emit_nop(ctx); in build_body()
1003 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1005 ctx); in build_body()
1007 b_off, ctx); in build_body()
1008 emit_nop(ctx); in build_body()
1010 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1012 b_off, ctx); in build_body()
1013 emit_nop(ctx); in build_body()
1018 ctx->flags |= SEEN_A; in build_body()
1020 emit_load_imm(r_s1, k, ctx); in build_body()
1021 emit_and(r_s0, r_A, r_s1, ctx); in build_body()
1023 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1024 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); in build_body()
1025 emit_nop(ctx); in build_body()
1027 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1028 emit_b(b_off, ctx); in build_body()
1029 emit_nop(ctx); in build_body()
1032 ctx->flags |= SEEN_X | SEEN_A; in build_body()
1034 emit_and(r_s0, r_A, r_X, ctx); in build_body()
1036 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1037 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); in build_body()
1038 emit_nop(ctx); in build_body()
1040 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1041 emit_b(b_off, ctx); in build_body()
1042 emit_nop(ctx); in build_body()
1045 ctx->flags |= SEEN_A; in build_body()
1051 emit_b(b_imm(prog->len, ctx), ctx); in build_body()
1052 emit_reg_move(r_ret, r_A, ctx); /* delay slot */ in build_body()
1059 emit_load_imm(r_ret, k, ctx); in build_body()
1065 emit_b(b_imm(prog->len, ctx), ctx); in build_body()
1066 emit_nop(ctx); in build_body()
1071 ctx->flags |= SEEN_X | SEEN_A; in build_body()
1072 emit_jit_reg_move(r_X, r_A, ctx); in build_body()
1076 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1077 emit_jit_reg_move(r_A, r_X, ctx); in build_body()
1082 ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A; in build_body()
1086 emit_half_load(r_A, r_skb, off, ctx); in build_body()
1091 emit_wsbh(r_A, r_A, ctx); in build_body()
1094 emit_andi(r_tmp_imm, r_A, 0xff, ctx); in build_body()
1096 emit_sll(r_tmp, r_tmp_imm, 8, ctx); in build_body()
1098 emit_srl(r_tmp_imm, r_A, 8, ctx); in build_body()
1099 emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx); in build_body()
1101 emit_or(r_A, r_tmp, r_tmp_imm, ctx); in build_body()
1106 ctx->flags |= SEEN_A | SEEN_OFF; in build_body()
1112 emit_load(r_A, 28, off, ctx); in build_body()
1116 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1119 emit_load_ptr(r_s0, r_skb, off, ctx); in build_body()
1122 b_imm(prog->len, ctx), ctx); in build_body()
1123 emit_reg_move(r_ret, r_zero, ctx); in build_body()
1127 emit_load(r_A, r_s0, off, ctx); in build_body()
1130 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1133 emit_load(r_A, r_skb, off, ctx); in build_body()
1136 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1139 emit_load(r_A, r_skb, off, ctx); in build_body()
1143 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1147 emit_half_load(r_s0, r_skb, off, ctx); in build_body()
1149 emit_andi(r_A, r_s0, (u16)~VLAN_TAG_PRESENT, ctx); in build_body()
1151 emit_andi(r_A, r_s0, VLAN_TAG_PRESENT, ctx); in build_body()
1153 emit_sltu(r_A, r_zero, r_A, ctx); in build_body()
1157 ctx->flags |= SEEN_SKB; in build_body()
1159 emit_load_byte(r_tmp, r_skb, PKT_TYPE_OFFSET(), ctx); in build_body()
1161 emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx); in build_body()
1164 emit_srl(r_A, r_A, 5, ctx); in build_body()
1168 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1174 emit_half_load(r_A, r_skb, off, ctx); in build_body()
1184 if (ctx->target == NULL) in build_body()
1185 ctx->offsets[i] = ctx->idx * 4; in build_body()
1194 struct jit_ctx ctx; in bpf_jit_compile() local
1200 memset(&ctx, 0, sizeof(ctx)); in bpf_jit_compile()
1202 ctx.offsets = kcalloc(fp->len, sizeof(*ctx.offsets), GFP_KERNEL); in bpf_jit_compile()
1203 if (ctx.offsets == NULL) in bpf_jit_compile()
1206 ctx.skf = fp; in bpf_jit_compile()
1208 if (build_body(&ctx)) in bpf_jit_compile()
1211 tmp_idx = ctx.idx; in bpf_jit_compile()
1212 build_prologue(&ctx); in bpf_jit_compile()
1213 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; in bpf_jit_compile()
1215 build_epilogue(&ctx); in bpf_jit_compile()
1217 alloc_size = 4 * ctx.idx; in bpf_jit_compile()
1218 ctx.target = module_alloc(alloc_size); in bpf_jit_compile()
1219 if (ctx.target == NULL) in bpf_jit_compile()
1223 memset(ctx.target, 0, alloc_size); in bpf_jit_compile()
1225 ctx.idx = 0; in bpf_jit_compile()
1228 build_prologue(&ctx); in bpf_jit_compile()
1229 build_body(&ctx); in bpf_jit_compile()
1230 build_epilogue(&ctx); in bpf_jit_compile()
1233 flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx)); in bpf_jit_compile()
1237 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); in bpf_jit_compile()
1239 fp->bpf_func = (void *)ctx.target; in bpf_jit_compile()
1243 kfree(ctx.offsets); in bpf_jit_compile()