Lines Matching refs:ctx
116 static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx) in _emit() argument
121 if (ctx->target != NULL) in _emit()
122 ctx->target[ctx->idx] = inst; in _emit()
124 ctx->idx++; in _emit()
130 static inline void emit(u32 inst, struct jit_ctx *ctx) in emit() argument
132 _emit(ARM_COND_AL, inst, ctx); in emit()
135 static u16 saved_regs(struct jit_ctx *ctx) in saved_regs() argument
139 if ((ctx->skf->len > 1) || in saved_regs()
140 (ctx->skf->insns[0].code == (BPF_RET | BPF_A))) in saved_regs()
146 if (ctx->seen & SEEN_CALL) in saved_regs()
149 if (ctx->seen & (SEEN_DATA | SEEN_SKB)) in saved_regs()
151 if (ctx->seen & SEEN_DATA) in saved_regs()
153 if (ctx->seen & SEEN_X) in saved_regs()
159 static inline int mem_words_used(struct jit_ctx *ctx) in mem_words_used() argument
162 return fls(ctx->seen & SEEN_MEM); in mem_words_used()
173 static void build_prologue(struct jit_ctx *ctx) in build_prologue() argument
175 u16 reg_set = saved_regs(ctx); in build_prologue()
179 emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); in build_prologue()
180 emit(ARM_PUSH(reg_set), ctx); in build_prologue()
181 emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); in build_prologue()
184 emit(ARM_PUSH(reg_set), ctx); in build_prologue()
187 if (ctx->seen & (SEEN_DATA | SEEN_SKB)) in build_prologue()
188 emit(ARM_MOV_R(r_skb, ARM_R0), ctx); in build_prologue()
190 if (ctx->seen & SEEN_DATA) { in build_prologue()
192 emit(ARM_LDR_I(r_skb_data, r_skb, off), ctx); in build_prologue()
195 emit(ARM_LDR_I(r_skb_hl, r_skb, off), ctx); in build_prologue()
197 emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); in build_prologue()
198 emit(ARM_SUB_R(r_skb_hl, r_skb_hl, r_scratch), ctx); in build_prologue()
201 if (ctx->flags & FLAG_NEED_X_RESET) in build_prologue()
202 emit(ARM_MOV_I(r_X, 0), ctx); in build_prologue()
205 if (bpf_needs_clear_a(&ctx->skf->insns[0])) in build_prologue()
206 emit(ARM_MOV_I(r_A, 0), ctx); in build_prologue()
209 if (ctx->seen & SEEN_MEM) in build_prologue()
210 emit(ARM_SUB_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); in build_prologue()
213 static void build_epilogue(struct jit_ctx *ctx) in build_epilogue() argument
215 u16 reg_set = saved_regs(ctx); in build_epilogue()
217 if (ctx->seen & SEEN_MEM) in build_epilogue()
218 emit(ARM_ADD_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); in build_epilogue()
226 emit(ARM_LDM(ARM_SP, reg_set), ctx); in build_epilogue()
229 if (ctx->seen & SEEN_CALL) in build_epilogue()
231 emit(ARM_POP(reg_set), ctx); in build_epilogue()
234 if (!(ctx->seen & SEEN_CALL)) in build_epilogue()
235 emit(ARM_BX(ARM_LR), ctx); in build_epilogue()
252 static u16 imm_offset(u32 k, struct jit_ctx *ctx) in imm_offset() argument
258 if (ctx->target == NULL) { in imm_offset()
259 ctx->imm_count++; in imm_offset()
263 while ((i < ctx->imm_count) && ctx->imms[i]) { in imm_offset()
264 if (ctx->imms[i] == k) in imm_offset()
269 if (ctx->imms[i] == 0) in imm_offset()
270 ctx->imms[i] = k; in imm_offset()
273 offset = ctx->offsets[ctx->skf->len]; in imm_offset()
274 offset += ctx->prologue_bytes; in imm_offset()
275 offset += ctx->epilogue_bytes; in imm_offset()
278 ctx->target[offset / 4] = k; in imm_offset()
281 imm = offset - (8 + ctx->idx * 4); in imm_offset()
288 ctx->flags |= FLAG_IMM_OVERFLOW; in imm_offset()
300 static inline void emit_mov_i_no8m(int rd, u32 val, struct jit_ctx *ctx) in emit_mov_i_no8m() argument
303 emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx); in emit_mov_i_no8m()
305 emit(ARM_MOVW(rd, val & 0xffff), ctx); in emit_mov_i_no8m()
307 emit(ARM_MOVT(rd, val >> 16), ctx); in emit_mov_i_no8m()
311 static inline void emit_mov_i(int rd, u32 val, struct jit_ctx *ctx) in emit_mov_i() argument
316 emit(ARM_MOV_I(rd, imm12), ctx); in emit_mov_i()
318 emit_mov_i_no8m(rd, val, ctx); in emit_mov_i()
323 static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) in emit_load_be32() argument
325 _emit(cond, ARM_LDRB_I(ARM_R3, r_addr, 1), ctx); in emit_load_be32()
326 _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); in emit_load_be32()
327 _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 3), ctx); in emit_load_be32()
328 _emit(cond, ARM_LSL_I(ARM_R3, ARM_R3, 16), ctx); in emit_load_be32()
329 _emit(cond, ARM_LDRB_I(ARM_R0, r_addr, 2), ctx); in emit_load_be32()
330 _emit(cond, ARM_ORR_S(ARM_R3, ARM_R3, ARM_R1, SRTYPE_LSL, 24), ctx); in emit_load_be32()
331 _emit(cond, ARM_ORR_R(ARM_R3, ARM_R3, ARM_R2), ctx); in emit_load_be32()
332 _emit(cond, ARM_ORR_S(r_res, ARM_R3, ARM_R0, SRTYPE_LSL, 8), ctx); in emit_load_be32()
335 static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) in emit_load_be16() argument
337 _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); in emit_load_be16()
338 _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 1), ctx); in emit_load_be16()
339 _emit(cond, ARM_ORR_S(r_res, ARM_R2, ARM_R1, SRTYPE_LSL, 8), ctx); in emit_load_be16()
342 static inline void emit_swap16(u8 r_dst, u8 r_src, struct jit_ctx *ctx) in emit_swap16() argument
345 emit(ARM_LSL_I(ARM_R1, r_src, 8), ctx); in emit_swap16()
346 emit(ARM_ORR_S(r_dst, ARM_R1, r_src, SRTYPE_LSR, 8), ctx); in emit_swap16()
354 emit(ARM_BIC_I(r_dst, r_dst, 0x8ff), ctx); in emit_swap16()
359 static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) in emit_load_be32() argument
361 _emit(cond, ARM_LDR_I(r_res, r_addr, 0), ctx); in emit_load_be32()
363 _emit(cond, ARM_REV(r_res, r_res), ctx); in emit_load_be32()
367 static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) in emit_load_be16() argument
369 _emit(cond, ARM_LDRH_I(r_res, r_addr, 0), ctx); in emit_load_be16()
371 _emit(cond, ARM_REV16(r_res, r_res), ctx); in emit_load_be16()
377 struct jit_ctx *ctx __maybe_unused) in emit_swap16()
380 emit(ARM_REV16(r_dst, r_src), ctx); in emit_swap16()
388 static inline u32 b_imm(unsigned tgt, struct jit_ctx *ctx) in b_imm() argument
392 if (ctx->target == NULL) in b_imm()
398 imm = ctx->offsets[tgt] + ctx->prologue_bytes - (ctx->idx * 4 + 8); in b_imm()
403 #define OP_IMM3(op, r1, r2, imm_val, ctx) \ argument
407 emit_mov_i_no8m(r_scratch, imm_val, ctx); \
408 emit(op ## _R((r1), (r2), r_scratch), ctx); \
410 emit(op ## _I((r1), (r2), imm12), ctx); \
414 static inline void emit_err_ret(u8 cond, struct jit_ctx *ctx) in emit_err_ret() argument
416 if (ctx->ret0_fp_idx >= 0) { in emit_err_ret()
417 _emit(cond, ARM_B(b_imm(ctx->ret0_fp_idx, ctx)), ctx); in emit_err_ret()
419 emit(ARM_MOV_R(ARM_R0, ARM_R0), ctx); in emit_err_ret()
421 _emit(cond, ARM_MOV_I(ARM_R0, 0), ctx); in emit_err_ret()
422 _emit(cond, ARM_B(b_imm(ctx->skf->len, ctx)), ctx); in emit_err_ret()
426 static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) in emit_blx_r() argument
429 emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); in emit_blx_r()
432 emit(ARM_BX(tgt_reg), ctx); in emit_blx_r()
434 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); in emit_blx_r()
436 emit(ARM_BLX_R(tgt_reg), ctx); in emit_blx_r()
440 static inline void emit_udiv(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx) in emit_udiv() argument
444 emit(ARM_UDIV(rd, rm, rn), ctx); in emit_udiv()
460 emit(ARM_MOV_R(ARM_R1, rn), ctx); in emit_udiv()
462 emit(ARM_MOV_R(ARM_R0, rm), ctx); in emit_udiv()
464 ctx->seen |= SEEN_CALL; in emit_udiv()
465 emit_mov_i(ARM_R3, (u32)jit_udiv, ctx); in emit_udiv()
466 emit_blx_r(ARM_R3, ctx); in emit_udiv()
469 emit(ARM_MOV_R(rd, ARM_R0), ctx); in emit_udiv()
472 static inline void update_on_xread(struct jit_ctx *ctx) in update_on_xread() argument
474 if (!(ctx->seen & SEEN_X)) in update_on_xread()
475 ctx->flags |= FLAG_NEED_X_RESET; in update_on_xread()
477 ctx->seen |= SEEN_X; in update_on_xread()
480 static int build_body(struct jit_ctx *ctx) in build_body() argument
483 const struct bpf_prog *prog = ctx->skf; in build_body()
498 if (ctx->target == NULL) in build_body()
499 ctx->offsets[i] = ctx->idx * 4; in build_body()
503 emit_mov_i(r_A, k, ctx); in build_body()
506 ctx->seen |= SEEN_SKB; in build_body()
509 offsetof(struct sk_buff, len)), ctx); in build_body()
513 ctx->seen |= SEEN_MEM_WORD(k); in build_body()
514 emit(ARM_LDR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); in build_body()
528 emit_mov_i(r_off, k, ctx); in build_body()
530 ctx->seen |= SEEN_DATA | SEEN_CALL; in build_body()
534 1 << load_order), ctx); in build_body()
535 emit(ARM_CMP_R(r_scratch, r_off), ctx); in build_body()
538 emit(ARM_CMP_R(r_skb_hl, r_off), ctx); in build_body()
543 ctx); in build_body()
547 ctx); in build_body()
549 emit_load_be16(condt, r_A, r_scratch, ctx); in build_body()
551 emit_load_be32(condt, r_A, r_scratch, ctx); in build_body()
553 _emit(condt, ARM_B(b_imm(i + 1, ctx)), ctx); in build_body()
556 emit_mov_i(ARM_R3, (u32)load_func[load_order], ctx); in build_body()
557 emit(ARM_MOV_R(ARM_R0, r_skb), ctx); in build_body()
559 emit_blx_r(ARM_R3, ctx); in build_body()
561 emit(ARM_CMP_I(ARM_R1, 0), ctx); in build_body()
562 emit_err_ret(ARM_COND_NE, ctx); in build_body()
563 emit(ARM_MOV_R(r_A, ARM_R0), ctx); in build_body()
574 OP_IMM3(ARM_ADD, r_off, r_X, k, ctx); in build_body()
577 ctx->seen |= SEEN_X; in build_body()
578 emit_mov_i(r_X, k, ctx); in build_body()
581 ctx->seen |= SEEN_X | SEEN_SKB; in build_body()
583 offsetof(struct sk_buff, len)), ctx); in build_body()
586 ctx->seen |= SEEN_X | SEEN_MEM_WORD(k); in build_body()
587 emit(ARM_LDR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); in build_body()
591 ctx->seen |= SEEN_X | SEEN_DATA | SEEN_CALL; in build_body()
596 emit_mov_i(r_off, k, ctx); in build_body()
597 emit(ARM_CMP_R(r_skb_hl, r_off), ctx); in build_body()
601 ARM_R1), ctx); in build_body()
606 _emit(ARM_COND_HI, ARM_B(b_imm(i + 1, ctx) - 2), ctx); in build_body()
608 emit(ARM_MOV_R(ARM_R0, r_skb), ctx); in build_body()
610 emit_mov_i(ARM_R3, (u32)jit_get_skb_b, ctx); in build_body()
611 emit_blx_r(ARM_R3, ctx); in build_body()
613 emit(ARM_CMP_I(ARM_R1, 0), ctx); in build_body()
614 emit_err_ret(ARM_COND_NE, ctx); in build_body()
616 emit(ARM_AND_I(r_X, ARM_R0, 0x00f), ctx); in build_body()
617 emit(ARM_LSL_I(r_X, r_X, 2), ctx); in build_body()
620 ctx->seen |= SEEN_MEM_WORD(k); in build_body()
621 emit(ARM_STR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); in build_body()
624 update_on_xread(ctx); in build_body()
625 ctx->seen |= SEEN_MEM_WORD(k); in build_body()
626 emit(ARM_STR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); in build_body()
630 OP_IMM3(ARM_ADD, r_A, r_A, k, ctx); in build_body()
633 update_on_xread(ctx); in build_body()
634 emit(ARM_ADD_R(r_A, r_A, r_X), ctx); in build_body()
638 OP_IMM3(ARM_SUB, r_A, r_A, k, ctx); in build_body()
641 update_on_xread(ctx); in build_body()
642 emit(ARM_SUB_R(r_A, r_A, r_X), ctx); in build_body()
646 emit_mov_i(r_scratch, k, ctx); in build_body()
647 emit(ARM_MUL(r_A, r_A, r_scratch), ctx); in build_body()
650 update_on_xread(ctx); in build_body()
651 emit(ARM_MUL(r_A, r_A, r_X), ctx); in build_body()
656 emit_mov_i(r_scratch, k, ctx); in build_body()
657 emit_udiv(r_A, r_A, r_scratch, ctx); in build_body()
660 update_on_xread(ctx); in build_body()
661 emit(ARM_CMP_I(r_X, 0), ctx); in build_body()
662 emit_err_ret(ARM_COND_EQ, ctx); in build_body()
663 emit_udiv(r_A, r_A, r_X, ctx); in build_body()
667 OP_IMM3(ARM_ORR, r_A, r_A, k, ctx); in build_body()
670 update_on_xread(ctx); in build_body()
671 emit(ARM_ORR_R(r_A, r_A, r_X), ctx); in build_body()
675 OP_IMM3(ARM_EOR, r_A, r_A, k, ctx); in build_body()
680 update_on_xread(ctx); in build_body()
681 emit(ARM_EOR_R(r_A, r_A, r_X), ctx); in build_body()
685 OP_IMM3(ARM_AND, r_A, r_A, k, ctx); in build_body()
688 update_on_xread(ctx); in build_body()
689 emit(ARM_AND_R(r_A, r_A, r_X), ctx); in build_body()
694 emit(ARM_LSL_I(r_A, r_A, k), ctx); in build_body()
697 update_on_xread(ctx); in build_body()
698 emit(ARM_LSL_R(r_A, r_A, r_X), ctx); in build_body()
703 emit(ARM_LSR_I(r_A, r_A, k), ctx); in build_body()
706 update_on_xread(ctx); in build_body()
707 emit(ARM_LSR_R(r_A, r_A, r_X), ctx); in build_body()
711 emit(ARM_RSB_I(r_A, r_A, 0), ctx); in build_body()
715 emit(ARM_B(b_imm(i + k + 1, ctx)), ctx); in build_body()
731 emit_mov_i_no8m(r_scratch, k, ctx); in build_body()
732 emit(ARM_CMP_R(r_A, r_scratch), ctx); in build_body()
734 emit(ARM_CMP_I(r_A, imm12), ctx); in build_body()
739 ctx)), ctx); in build_body()
742 ctx)), ctx); in build_body()
756 update_on_xread(ctx); in build_body()
757 emit(ARM_CMP_R(r_A, r_X), ctx); in build_body()
766 emit_mov_i_no8m(r_scratch, k, ctx); in build_body()
767 emit(ARM_TST_R(r_A, r_scratch), ctx); in build_body()
769 emit(ARM_TST_I(r_A, imm12), ctx); in build_body()
774 update_on_xread(ctx); in build_body()
776 emit(ARM_TST_R(r_A, r_X), ctx); in build_body()
779 emit(ARM_MOV_R(ARM_R0, r_A), ctx); in build_body()
782 if ((k == 0) && (ctx->ret0_fp_idx < 0)) in build_body()
783 ctx->ret0_fp_idx = i; in build_body()
784 emit_mov_i(ARM_R0, k, ctx); in build_body()
786 if (i != ctx->skf->len - 1) in build_body()
787 emit(ARM_B(b_imm(prog->len, ctx)), ctx); in build_body()
791 ctx->seen |= SEEN_X; in build_body()
792 emit(ARM_MOV_R(r_X, r_A), ctx); in build_body()
796 update_on_xread(ctx); in build_body()
797 emit(ARM_MOV_R(r_A, r_X), ctx); in build_body()
801 ctx->seen |= SEEN_SKB; in build_body()
805 emit(ARM_LDRH_I(r_scratch, r_skb, off), ctx); in build_body()
806 emit_swap16(r_A, r_scratch, ctx); in build_body()
810 OP_IMM3(ARM_BIC, r_scratch, ARM_SP, THREAD_SIZE - 1, ctx); in build_body()
814 emit(ARM_LDR_I(r_A, r_scratch, off), ctx); in build_body()
818 ctx->seen |= SEEN_SKB; in build_body()
820 emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); in build_body()
822 emit(ARM_CMP_I(r_scratch, 0), ctx); in build_body()
823 emit_err_ret(ARM_COND_EQ, ctx); in build_body()
828 emit(ARM_LDR_I(r_A, r_scratch, off), ctx); in build_body()
831 ctx->seen |= SEEN_SKB; in build_body()
834 emit(ARM_LDR_I(r_A, r_skb, off), ctx); in build_body()
837 ctx->seen |= SEEN_SKB; in build_body()
840 emit(ARM_LDR_I(r_A, r_skb, off), ctx); in build_body()
844 ctx->seen |= SEEN_SKB; in build_body()
847 emit(ARM_LDRH_I(r_A, r_skb, off), ctx); in build_body()
849 OP_IMM3(ARM_AND, r_A, r_A, VLAN_VID_MASK, ctx); in build_body()
851 OP_IMM3(ARM_AND, r_A, r_A, VLAN_TAG_PRESENT, ctx); in build_body()
854 ctx->seen |= SEEN_SKB; in build_body()
860 emit(ARM_LDRH_I(r_A, r_skb, off), ctx); in build_body()
866 if (ctx->flags & FLAG_IMM_OVERFLOW) in build_body()
876 if (ctx->target == NULL) in build_body()
877 ctx->offsets[i] = ctx->idx * 4; in build_body()
886 struct jit_ctx ctx; in bpf_jit_compile() local
894 memset(&ctx, 0, sizeof(ctx)); in bpf_jit_compile()
895 ctx.skf = fp; in bpf_jit_compile()
896 ctx.ret0_fp_idx = -1; in bpf_jit_compile()
898 ctx.offsets = kzalloc(4 * (ctx.skf->len + 1), GFP_KERNEL); in bpf_jit_compile()
899 if (ctx.offsets == NULL) in bpf_jit_compile()
903 if (unlikely(build_body(&ctx))) in bpf_jit_compile()
906 tmp_idx = ctx.idx; in bpf_jit_compile()
907 build_prologue(&ctx); in bpf_jit_compile()
908 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; in bpf_jit_compile()
911 tmp_idx = ctx.idx; in bpf_jit_compile()
912 build_epilogue(&ctx); in bpf_jit_compile()
913 ctx.epilogue_bytes = (ctx.idx - tmp_idx) * 4; in bpf_jit_compile()
915 ctx.idx += ctx.imm_count; in bpf_jit_compile()
916 if (ctx.imm_count) { in bpf_jit_compile()
917 ctx.imms = kzalloc(4 * ctx.imm_count, GFP_KERNEL); in bpf_jit_compile()
918 if (ctx.imms == NULL) in bpf_jit_compile()
923 build_epilogue(&ctx); in bpf_jit_compile()
925 alloc_size = 4 * ctx.idx; in bpf_jit_compile()
931 ctx.target = (u32 *) target_ptr; in bpf_jit_compile()
932 ctx.idx = 0; in bpf_jit_compile()
934 build_prologue(&ctx); in bpf_jit_compile()
935 if (build_body(&ctx) < 0) { in bpf_jit_compile()
937 if (ctx.imm_count) in bpf_jit_compile()
938 kfree(ctx.imms); in bpf_jit_compile()
943 build_epilogue(&ctx); in bpf_jit_compile()
945 flush_icache_range((u32)ctx.target, (u32)(ctx.target + ctx.idx)); in bpf_jit_compile()
948 if (ctx.imm_count) in bpf_jit_compile()
949 kfree(ctx.imms); in bpf_jit_compile()
954 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); in bpf_jit_compile()
957 fp->bpf_func = (void *)ctx.target; in bpf_jit_compile()
960 kfree(ctx.offsets); in bpf_jit_compile()