Lines Matching refs:ctx
154 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx);
157 #define emit_instr(ctx, func, ...) \ argument
159 if ((ctx)->target != NULL) { \
160 u32 *p = &(ctx)->target[ctx->idx]; \
163 (ctx)->idx++; \
170 #define emit_long_instr(ctx, func, ...) \ argument
172 if ((ctx)->target != NULL) { \
173 u32 *p = &(ctx)->target[ctx->idx]; \
176 (ctx)->idx++; \
186 unsigned int src2, struct jit_ctx *ctx) in emit_addu() argument
188 emit_instr(ctx, addu, dst, src1, src2); in emit_addu()
191 static inline void emit_nop(struct jit_ctx *ctx) in emit_nop() argument
193 emit_instr(ctx, nop); in emit_nop()
197 static inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx) in emit_load_imm() argument
199 if (ctx->target != NULL) { in emit_load_imm()
202 u32 *p = &ctx->target[ctx->idx]; in emit_load_imm()
204 p = &ctx->target[ctx->idx + 1]; in emit_load_imm()
207 u32 *p = &ctx->target[ctx->idx]; in emit_load_imm()
211 ctx->idx++; in emit_load_imm()
214 ctx->idx++; in emit_load_imm()
218 unsigned int src2, struct jit_ctx *ctx) in emit_or() argument
220 emit_instr(ctx, or, dst, src1, src2); in emit_or()
224 struct jit_ctx *ctx) in emit_ori() argument
227 emit_load_imm(r_tmp, imm, ctx); in emit_ori()
228 emit_or(dst, src, r_tmp, ctx); in emit_ori()
230 emit_instr(ctx, ori, dst, src, imm); in emit_ori()
235 int imm, struct jit_ctx *ctx) in emit_daddiu() argument
241 emit_instr(ctx, daddiu, dst, src, imm); in emit_daddiu()
245 u32 imm, struct jit_ctx *ctx) in emit_addiu() argument
248 emit_load_imm(r_tmp, imm, ctx); in emit_addiu()
249 emit_addu(dst, r_tmp, src, ctx); in emit_addiu()
251 emit_instr(ctx, addiu, dst, src, imm); in emit_addiu()
256 unsigned int src2, struct jit_ctx *ctx) in emit_and() argument
258 emit_instr(ctx, and, dst, src1, src2); in emit_and()
262 u32 imm, struct jit_ctx *ctx) in emit_andi() argument
266 emit_load_imm(r_tmp, imm, ctx); in emit_andi()
267 emit_and(dst, src, r_tmp, ctx); in emit_andi()
269 emit_instr(ctx, andi, dst, src, imm); in emit_andi()
274 unsigned int src2, struct jit_ctx *ctx) in emit_xor() argument
276 emit_instr(ctx, xor, dst, src1, src2); in emit_xor()
279 static inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx) in emit_xori() argument
283 emit_load_imm(r_tmp, imm, ctx); in emit_xori()
284 emit_xor(dst, src, r_tmp, ctx); in emit_xori()
286 emit_instr(ctx, xori, dst, src, imm); in emit_xori()
290 static inline void emit_stack_offset(int offset, struct jit_ctx *ctx) in emit_stack_offset() argument
292 emit_long_instr(ctx, ADDIU, r_sp, r_sp, offset); in emit_stack_offset()
296 unsigned int src2, struct jit_ctx *ctx) in emit_subu() argument
298 emit_instr(ctx, subu, dst, src1, src2); in emit_subu()
301 static inline void emit_neg(unsigned int reg, struct jit_ctx *ctx) in emit_neg() argument
303 emit_subu(reg, r_zero, reg, ctx); in emit_neg()
307 unsigned int sa, struct jit_ctx *ctx) in emit_sllv() argument
309 emit_instr(ctx, sllv, dst, src, sa); in emit_sllv()
313 unsigned int sa, struct jit_ctx *ctx) in emit_sll() argument
318 emit_jit_reg_move(dst, r_zero, ctx); in emit_sll()
320 emit_instr(ctx, sll, dst, src, sa); in emit_sll()
324 unsigned int sa, struct jit_ctx *ctx) in emit_srlv() argument
326 emit_instr(ctx, srlv, dst, src, sa); in emit_srlv()
330 unsigned int sa, struct jit_ctx *ctx) in emit_srl() argument
335 emit_jit_reg_move(dst, r_zero, ctx); in emit_srl()
337 emit_instr(ctx, srl, dst, src, sa); in emit_srl()
341 unsigned int src2, struct jit_ctx *ctx) in emit_slt() argument
343 emit_instr(ctx, slt, dst, src1, src2); in emit_slt()
347 unsigned int src2, struct jit_ctx *ctx) in emit_sltu() argument
349 emit_instr(ctx, sltu, dst, src1, src2); in emit_sltu()
353 unsigned int imm, struct jit_ctx *ctx) in emit_sltiu() argument
357 emit_load_imm(r_tmp, imm, ctx); in emit_sltiu()
358 emit_sltu(dst, src, r_tmp, ctx); in emit_sltiu()
360 emit_instr(ctx, sltiu, dst, src, imm); in emit_sltiu()
368 struct jit_ctx *ctx) in emit_store_stack_reg() argument
370 emit_long_instr(ctx, SW, reg, offset, base); in emit_store_stack_reg()
374 struct jit_ctx *ctx) in emit_store() argument
376 emit_instr(ctx, sw, reg, offset, base); in emit_store()
381 struct jit_ctx *ctx) in emit_load_stack_reg() argument
383 emit_long_instr(ctx, LW, reg, offset, base); in emit_load_stack_reg()
387 unsigned int offset, struct jit_ctx *ctx) in emit_load() argument
389 emit_instr(ctx, lw, reg, offset, base); in emit_load()
393 unsigned int offset, struct jit_ctx *ctx) in emit_load_byte() argument
395 emit_instr(ctx, lb, reg, offset, base); in emit_load_byte()
399 unsigned int offset, struct jit_ctx *ctx) in emit_half_load() argument
401 emit_instr(ctx, lh, reg, offset, base); in emit_half_load()
405 unsigned int src2, struct jit_ctx *ctx) in emit_mul() argument
407 emit_instr(ctx, mul, dst, src1, src2); in emit_mul()
411 struct jit_ctx *ctx) in emit_div() argument
413 if (ctx->target != NULL) { in emit_div()
414 u32 *p = &ctx->target[ctx->idx]; in emit_div()
416 p = &ctx->target[ctx->idx + 1]; in emit_div()
419 ctx->idx += 2; /* 2 insts */ in emit_div()
423 struct jit_ctx *ctx) in emit_mod() argument
425 if (ctx->target != NULL) { in emit_mod()
426 u32 *p = &ctx->target[ctx->idx]; in emit_mod()
428 p = &ctx->target[ctx->idx + 1]; in emit_mod()
431 ctx->idx += 2; /* 2 insts */ in emit_mod()
435 unsigned int sa, struct jit_ctx *ctx) in emit_dsll() argument
437 emit_instr(ctx, dsll, dst, src, sa); in emit_dsll()
441 unsigned int sa, struct jit_ctx *ctx) in emit_dsrl32() argument
443 emit_instr(ctx, dsrl32, dst, src, sa); in emit_dsrl32()
447 struct jit_ctx *ctx) in emit_wsbh() argument
449 emit_instr(ctx, wsbh, dst, src); in emit_wsbh()
454 int imm, struct jit_ctx *ctx) in emit_load_ptr() argument
457 emit_long_instr(ctx, LW, dst, imm, src); in emit_load_ptr()
462 struct jit_ctx *ctx) in emit_load_func() argument
466 emit_load_imm(r_tmp, (u64)imm >> 32, ctx); in emit_load_func()
467 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ in emit_load_func()
468 emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx); in emit_load_func()
469 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ in emit_load_func()
470 emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx); in emit_load_func()
472 emit_load_imm(reg, imm, ctx); in emit_load_func()
477 static inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) in emit_reg_move() argument
479 emit_long_instr(ctx, ADDU, dst, src, r_zero); in emit_reg_move()
483 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) in emit_jit_reg_move() argument
485 emit_addu(dst, src, r_zero, ctx); in emit_jit_reg_move()
489 static inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx) in b_imm() argument
491 if (ctx->target == NULL) in b_imm()
507 return ctx->offsets[tgt] - in b_imm()
508 (ctx->idx * 4 - ctx->prologue_bytes) - 4; in b_imm()
512 unsigned int imm, struct jit_ctx *ctx) in emit_bcond() argument
514 if (ctx->target != NULL) { in emit_bcond()
515 u32 *p = &ctx->target[ctx->idx]; in emit_bcond()
532 ctx->idx++; in emit_bcond()
535 static inline void emit_b(unsigned int imm, struct jit_ctx *ctx) in emit_b() argument
537 emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx); in emit_b()
541 struct jit_ctx *ctx) in emit_jalr() argument
543 emit_instr(ctx, jalr, link, reg); in emit_jalr()
546 static inline void emit_jr(unsigned int reg, struct jit_ctx *ctx) in emit_jr() argument
548 emit_instr(ctx, jr, reg); in emit_jr()
559 static void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset) in save_bpf_jit_regs() argument
565 emit_stack_offset(-align_sp(offset), ctx); in save_bpf_jit_regs()
567 if (ctx->flags & SEEN_CALL) { in save_bpf_jit_regs()
575 emit_store_stack_reg(MIPS_R_A0, r_sp, real_off, ctx); in save_bpf_jit_regs()
576 emit_store_stack_reg(MIPS_R_A1, r_sp, real_off + RSIZE, ctx); in save_bpf_jit_regs()
581 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; in save_bpf_jit_regs()
586 ctx); in save_bpf_jit_regs()
594 if (ctx->flags & SEEN_CALL) { in save_bpf_jit_regs()
595 emit_store_stack_reg(r_ra, r_sp, real_off, ctx); in save_bpf_jit_regs()
600 if (ctx->flags & SEEN_MEM) { in save_bpf_jit_regs()
603 emit_long_instr(ctx, ADDIU, r_M, r_sp, real_off); in save_bpf_jit_regs()
607 static void restore_bpf_jit_regs(struct jit_ctx *ctx, in restore_bpf_jit_regs() argument
613 if (ctx->flags & SEEN_CALL) { in restore_bpf_jit_regs()
620 emit_load_stack_reg(MIPS_R_A0, r_sp, real_off, ctx); in restore_bpf_jit_regs()
621 emit_load_stack_reg(MIPS_R_A1, r_sp, real_off + RSIZE, ctx); in restore_bpf_jit_regs()
626 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; in restore_bpf_jit_regs()
632 ctx); in restore_bpf_jit_regs()
640 if (ctx->flags & SEEN_CALL) in restore_bpf_jit_regs()
641 emit_load_stack_reg(r_ra, r_sp, real_off, ctx); in restore_bpf_jit_regs()
644 emit_stack_offset(align_sp(offset), ctx); in restore_bpf_jit_regs()
647 static unsigned int get_stack_depth(struct jit_ctx *ctx) in get_stack_depth() argument
653 sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * RSIZE; in get_stack_depth()
655 if (ctx->flags & SEEN_MEM) in get_stack_depth()
658 if (ctx->flags & SEEN_CALL) in get_stack_depth()
674 static void build_prologue(struct jit_ctx *ctx) in build_prologue() argument
679 sp_off = get_stack_depth(ctx); in build_prologue()
680 save_bpf_jit_regs(ctx, sp_off); in build_prologue()
682 if (ctx->flags & SEEN_SKB) in build_prologue()
683 emit_reg_move(r_skb, MIPS_R_A0, ctx); in build_prologue()
685 if (ctx->flags & SEEN_X) in build_prologue()
686 emit_jit_reg_move(r_X, r_zero, ctx); in build_prologue()
689 if (bpf_needs_clear_a(&ctx->skf->insns[0])) in build_prologue()
690 emit_jit_reg_move(r_A, r_zero, ctx); in build_prologue()
693 static void build_epilogue(struct jit_ctx *ctx) in build_epilogue() argument
699 sp_off = get_stack_depth(ctx); in build_epilogue()
700 restore_bpf_jit_regs(ctx, sp_off); in build_epilogue()
703 emit_jr(r_ra, ctx); in build_epilogue()
704 emit_nop(ctx); in build_epilogue()
737 static int build_body(struct jit_ctx *ctx) in build_body() argument
740 const struct bpf_prog *prog = ctx->skf; in build_body()
754 if (ctx->target == NULL) in build_body()
755 ctx->offsets[i] = ctx->idx * 4; in build_body()
760 ctx->flags |= SEEN_A; in build_body()
761 emit_load_imm(r_A, k, ctx); in build_body()
766 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
768 emit_load(r_A, r_skb, off, ctx); in build_body()
772 ctx->flags |= SEEN_MEM | SEEN_A; in build_body()
773 emit_load(r_A, r_M, SCRATCH_OFF(k), ctx); in build_body()
791 emit_load_imm(r_off, k, ctx); in build_body()
797 emit_slt(r_s0, r_off, r_zero, ctx); in build_body()
799 b_imm(prog->len, ctx), ctx); in build_body()
800 emit_reg_move(r_ret, r_zero, ctx); in build_body()
802 ctx->flags |= SEEN_CALL | SEEN_OFF | SEEN_S0 | in build_body()
806 ctx); in build_body()
807 emit_reg_move(MIPS_R_A0, r_skb, ctx); in build_body()
808 emit_jalr(MIPS_R_RA, r_s0, ctx); in build_body()
810 emit_reg_move(MIPS_R_A1, r_off, ctx); in build_body()
814 emit_dsrl32(r_s0, r_val, 0, ctx); in build_body()
817 ctx); in build_body()
821 ctx); in build_body()
823 emit_nop(ctx); in build_body()
825 emit_b(b_imm(i + 1, ctx), ctx); in build_body()
826 emit_jit_reg_move(r_A, r_val, ctx); in build_body()
828 emit_b(b_imm(prog->len, ctx), ctx); in build_body()
829 emit_reg_move(r_ret, r_zero, ctx); in build_body()
843 ctx->flags |= SEEN_OFF | SEEN_X; in build_body()
844 emit_addiu(r_off, r_X, k, ctx); in build_body()
848 ctx->flags |= SEEN_X; in build_body()
849 emit_load_imm(r_X, k, ctx); in build_body()
853 ctx->flags |= SEEN_X | SEEN_MEM; in build_body()
854 emit_load(r_X, r_M, SCRATCH_OFF(k), ctx); in build_body()
858 ctx->flags |= SEEN_X | SEEN_SKB; in build_body()
860 emit_load(r_X, r_skb, off, ctx); in build_body()
868 ctx->flags |= SEEN_X | SEEN_CALL | SEEN_S0 | SEEN_SKB; in build_body()
870 emit_load_func(r_s0, (ptr)jit_get_skb_b, ctx); in build_body()
875 emit_load_imm(MIPS_R_A1, k, ctx); in build_body()
876 emit_jalr(MIPS_R_RA, r_s0, ctx); in build_body()
877 emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */ in build_body()
881 emit_dsrl32(r_s0, r_val, 0, ctx); in build_body()
883 3 << 2, ctx); in build_body()
886 3 << 2, ctx); in build_body()
891 emit_andi(r_X, r_val, 0xf, ctx); in build_body()
893 emit_b(b_imm(i + 1, ctx), ctx); in build_body()
894 emit_sll(r_X, r_X, 2, ctx); /* delay slot */ in build_body()
896 emit_b(b_imm(prog->len, ctx), ctx); in build_body()
897 emit_load_imm(r_ret, 0, ctx); /* delay slot */ in build_body()
901 ctx->flags |= SEEN_MEM | SEEN_A; in build_body()
902 emit_store(r_A, r_M, SCRATCH_OFF(k), ctx); in build_body()
906 ctx->flags |= SEEN_MEM | SEEN_X; in build_body()
907 emit_store(r_X, r_M, SCRATCH_OFF(k), ctx); in build_body()
911 ctx->flags |= SEEN_A; in build_body()
912 emit_addiu(r_A, r_A, k, ctx); in build_body()
916 ctx->flags |= SEEN_A | SEEN_X; in build_body()
917 emit_addu(r_A, r_A, r_X, ctx); in build_body()
921 ctx->flags |= SEEN_A; in build_body()
922 emit_addiu(r_A, r_A, -k, ctx); in build_body()
926 ctx->flags |= SEEN_A | SEEN_X; in build_body()
927 emit_subu(r_A, r_A, r_X, ctx); in build_body()
932 ctx->flags |= SEEN_A | SEEN_S0; in build_body()
933 emit_load_imm(r_s0, k, ctx); in build_body()
934 emit_mul(r_A, r_A, r_s0, ctx); in build_body()
938 ctx->flags |= SEEN_A | SEEN_X; in build_body()
939 emit_mul(r_A, r_A, r_X, ctx); in build_body()
946 ctx->flags |= SEEN_A; in build_body()
947 emit_srl(r_A, r_A, k, ctx); in build_body()
950 ctx->flags |= SEEN_A | SEEN_S0; in build_body()
951 emit_load_imm(r_s0, k, ctx); in build_body()
952 emit_div(r_A, r_s0, ctx); in build_body()
957 ctx->flags |= SEEN_A; in build_body()
958 emit_jit_reg_move(r_A, r_zero, ctx); in build_body()
960 ctx->flags |= SEEN_A | SEEN_S0; in build_body()
961 emit_load_imm(r_s0, k, ctx); in build_body()
962 emit_mod(r_A, r_s0, ctx); in build_body()
967 ctx->flags |= SEEN_X | SEEN_A; in build_body()
970 b_imm(prog->len, ctx), ctx); in build_body()
971 emit_load_imm(r_val, 0, ctx); /* delay slot */ in build_body()
972 emit_div(r_A, r_X, ctx); in build_body()
976 ctx->flags |= SEEN_X | SEEN_A; in build_body()
979 b_imm(prog->len, ctx), ctx); in build_body()
980 emit_load_imm(r_val, 0, ctx); /* delay slot */ in build_body()
981 emit_mod(r_A, r_X, ctx); in build_body()
985 ctx->flags |= SEEN_A; in build_body()
986 emit_ori(r_A, r_A, k, ctx); in build_body()
990 ctx->flags |= SEEN_A; in build_body()
991 emit_ori(r_A, r_A, r_X, ctx); in build_body()
995 ctx->flags |= SEEN_A; in build_body()
996 emit_xori(r_A, r_A, k, ctx); in build_body()
1001 ctx->flags |= SEEN_A; in build_body()
1002 emit_xor(r_A, r_A, r_X, ctx); in build_body()
1006 ctx->flags |= SEEN_A; in build_body()
1007 emit_andi(r_A, r_A, k, ctx); in build_body()
1011 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1012 emit_and(r_A, r_A, r_X, ctx); in build_body()
1016 ctx->flags |= SEEN_A; in build_body()
1017 emit_sll(r_A, r_A, k, ctx); in build_body()
1021 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1022 emit_sllv(r_A, r_A, r_X, ctx); in build_body()
1026 ctx->flags |= SEEN_A; in build_body()
1027 emit_srl(r_A, r_A, k, ctx); in build_body()
1030 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1031 emit_srlv(r_A, r_A, r_X, ctx); in build_body()
1035 ctx->flags |= SEEN_A; in build_body()
1036 emit_neg(r_A, ctx); in build_body()
1040 emit_b(b_imm(i + k + 1, ctx), ctx); in build_body()
1041 emit_nop(ctx); in build_body()
1048 ctx->flags |= SEEN_X; in build_body()
1057 ctx->flags |= SEEN_X; in build_body()
1066 ctx->flags |= SEEN_X; in build_body()
1074 ctx->flags |= SEEN_S0 | SEEN_A; in build_body()
1075 emit_sltiu(r_s0, r_A, k, ctx); in build_body()
1077 ctx->flags |= SEEN_S0 | SEEN_A | in build_body()
1079 emit_sltu(r_s0, r_A, r_X, ctx); in build_body()
1082 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1084 ctx); in build_body()
1085 emit_nop(ctx); in build_body()
1089 ctx->flags |= SEEN_S0 | SEEN_A | SEEN_X; in build_body()
1091 emit_load_imm(r_s0, k, ctx); in build_body()
1094 ctx); in build_body()
1095 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1097 b_off, ctx); in build_body()
1098 emit_nop(ctx); in build_body()
1100 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1101 emit_b(b_off, ctx); in build_body()
1102 emit_nop(ctx); in build_body()
1105 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1106 emit_b(b_off, ctx); in build_body()
1107 emit_nop(ctx); in build_body()
1112 ctx->flags |= SEEN_S0 | SEEN_A; in build_body()
1113 emit_load_imm(r_s0, k, ctx); in build_body()
1115 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1117 b_off, ctx); in build_body()
1118 emit_nop(ctx); in build_body()
1121 ctx); in build_body()
1123 b_off, ctx); in build_body()
1124 emit_nop(ctx); in build_body()
1127 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1129 ctx); in build_body()
1131 b_off, ctx); in build_body()
1132 emit_nop(ctx); in build_body()
1134 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1136 b_off, ctx); in build_body()
1137 emit_nop(ctx); in build_body()
1142 ctx->flags |= SEEN_S0 | SEEN_S1 | SEEN_A; in build_body()
1144 emit_load_imm(r_s1, k, ctx); in build_body()
1145 emit_and(r_s0, r_A, r_s1, ctx); in build_body()
1147 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1148 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); in build_body()
1149 emit_nop(ctx); in build_body()
1151 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1152 emit_b(b_off, ctx); in build_body()
1153 emit_nop(ctx); in build_body()
1156 ctx->flags |= SEEN_S0 | SEEN_X | SEEN_A; in build_body()
1158 emit_and(r_s0, r_A, r_X, ctx); in build_body()
1160 b_off = b_imm(i + inst->jt + 1, ctx); in build_body()
1161 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); in build_body()
1162 emit_nop(ctx); in build_body()
1164 b_off = b_imm(i + inst->jf + 1, ctx); in build_body()
1165 emit_b(b_off, ctx); in build_body()
1166 emit_nop(ctx); in build_body()
1169 ctx->flags |= SEEN_A; in build_body()
1175 emit_b(b_imm(prog->len, ctx), ctx); in build_body()
1176 emit_reg_move(r_ret, r_A, ctx); /* delay slot */ in build_body()
1183 emit_load_imm(r_ret, k, ctx); in build_body()
1189 emit_b(b_imm(prog->len, ctx), ctx); in build_body()
1190 emit_nop(ctx); in build_body()
1195 ctx->flags |= SEEN_X | SEEN_A; in build_body()
1196 emit_jit_reg_move(r_X, r_A, ctx); in build_body()
1200 ctx->flags |= SEEN_A | SEEN_X; in build_body()
1201 emit_jit_reg_move(r_A, r_X, ctx); in build_body()
1206 ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A; in build_body()
1210 emit_half_load(r_A, r_skb, off, ctx); in build_body()
1215 emit_wsbh(r_A, r_A, ctx); in build_body()
1218 emit_andi(r_tmp_imm, r_A, 0xff, ctx); in build_body()
1220 emit_sll(r_tmp, r_tmp_imm, 8, ctx); in build_body()
1222 emit_srl(r_tmp_imm, r_A, 8, ctx); in build_body()
1223 emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx); in build_body()
1225 emit_or(r_A, r_tmp, r_tmp_imm, ctx); in build_body()
1230 ctx->flags |= SEEN_A | SEEN_OFF; in build_body()
1236 emit_load(r_A, 28, off, ctx); in build_body()
1240 ctx->flags |= SEEN_SKB | SEEN_A | SEEN_S0; in build_body()
1243 emit_load_ptr(r_s0, r_skb, off, ctx); in build_body()
1246 b_imm(prog->len, ctx), ctx); in build_body()
1247 emit_reg_move(r_ret, r_zero, ctx); in build_body()
1251 emit_load(r_A, r_s0, off, ctx); in build_body()
1254 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1257 emit_load(r_A, r_skb, off, ctx); in build_body()
1260 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1263 emit_load(r_A, r_skb, off, ctx); in build_body()
1267 ctx->flags |= SEEN_SKB | SEEN_S0 | SEEN_A; in build_body()
1271 emit_half_load(r_s0, r_skb, off, ctx); in build_body()
1273 emit_andi(r_A, r_s0, (u16)~VLAN_TAG_PRESENT, ctx); in build_body()
1275 emit_andi(r_A, r_s0, VLAN_TAG_PRESENT, ctx); in build_body()
1277 emit_sltu(r_A, r_zero, r_A, ctx); in build_body()
1281 ctx->flags |= SEEN_SKB; in build_body()
1283 emit_load_byte(r_tmp, r_skb, PKT_TYPE_OFFSET(), ctx); in build_body()
1285 emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx); in build_body()
1288 emit_srl(r_A, r_A, 5, ctx); in build_body()
1292 ctx->flags |= SEEN_SKB | SEEN_A; in build_body()
1298 emit_half_load(r_A, r_skb, off, ctx); in build_body()
1308 if (ctx->target == NULL) in build_body()
1309 ctx->offsets[i] = ctx->idx * 4; in build_body()
1318 struct jit_ctx ctx; in bpf_jit_compile() local
1324 memset(&ctx, 0, sizeof(ctx)); in bpf_jit_compile()
1326 ctx.offsets = kcalloc(fp->len, sizeof(*ctx.offsets), GFP_KERNEL); in bpf_jit_compile()
1327 if (ctx.offsets == NULL) in bpf_jit_compile()
1330 ctx.skf = fp; in bpf_jit_compile()
1332 if (build_body(&ctx)) in bpf_jit_compile()
1335 tmp_idx = ctx.idx; in bpf_jit_compile()
1336 build_prologue(&ctx); in bpf_jit_compile()
1337 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; in bpf_jit_compile()
1339 build_epilogue(&ctx); in bpf_jit_compile()
1341 alloc_size = 4 * ctx.idx; in bpf_jit_compile()
1342 ctx.target = module_alloc(alloc_size); in bpf_jit_compile()
1343 if (ctx.target == NULL) in bpf_jit_compile()
1347 memset(ctx.target, 0, alloc_size); in bpf_jit_compile()
1349 ctx.idx = 0; in bpf_jit_compile()
1352 build_prologue(&ctx); in bpf_jit_compile()
1353 build_body(&ctx); in bpf_jit_compile()
1354 build_epilogue(&ctx); in bpf_jit_compile()
1357 flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx)); in bpf_jit_compile()
1361 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); in bpf_jit_compile()
1363 fp->bpf_func = (void *)ctx.target; in bpf_jit_compile()
1367 kfree(ctx.offsets); in bpf_jit_compile()