Lines Matching refs:ctx
68 static inline void emit(const u32 insn, struct jit_ctx *ctx) in emit() argument
70 if (ctx->image != NULL) in emit()
71 ctx->image[ctx->idx] = cpu_to_le32(insn); in emit()
73 ctx->idx++; in emit()
77 struct jit_ctx *ctx) in emit_a64_mov_i64() argument
82 emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx); in emit_a64_mov_i64()
87 emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx); in emit_a64_mov_i64()
94 const s32 val, struct jit_ctx *ctx) in emit_a64_mov_i() argument
101 emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx); in emit_a64_mov_i()
103 emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx); in emit_a64_mov_i()
104 emit(A64_MOVK(is64, reg, lo, 0), ctx); in emit_a64_mov_i()
107 emit(A64_MOVZ(is64, reg, lo, 0), ctx); in emit_a64_mov_i()
109 emit(A64_MOVK(is64, reg, hi, 16), ctx); in emit_a64_mov_i()
114 const struct jit_ctx *ctx) in bpf2a64_offset() argument
116 int to = ctx->offset[bpf_to]; in bpf2a64_offset()
118 int from = ctx->offset[bpf_from] - 1; in bpf2a64_offset()
131 static inline int epilogue_offset(const struct jit_ctx *ctx) in epilogue_offset() argument
133 int to = ctx->epilogue_offset; in epilogue_offset()
134 int from = ctx->idx; in epilogue_offset()
148 static void build_prologue(struct jit_ctx *ctx) in build_prologue() argument
186 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); in build_prologue()
187 emit(A64_MOV(1, A64_FP, A64_SP), ctx); in build_prologue()
190 emit(A64_PUSH(r6, r7, A64_SP), ctx); in build_prologue()
191 emit(A64_PUSH(r8, r9, A64_SP), ctx); in build_prologue()
192 if (ctx->tmp_used) in build_prologue()
193 emit(A64_PUSH(tmp1, tmp2, A64_SP), ctx); in build_prologue()
196 emit(A64_PUSH(fp, A64_R(26), A64_SP), ctx); in build_prologue()
199 emit(A64_MOV(1, fp, A64_SP), ctx); in build_prologue()
202 emit(A64_SUB_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); in build_prologue()
205 emit_a64_mov_i64(ra, 0, ctx); in build_prologue()
206 emit_a64_mov_i64(rx, 0, ctx); in build_prologue()
209 static void build_epilogue(struct jit_ctx *ctx) in build_epilogue() argument
221 emit(A64_ADD_I(1, A64_SP, A64_SP, STACK_SIZE), ctx); in build_epilogue()
224 emit(A64_POP(fp, A64_R(26), A64_SP), ctx); in build_epilogue()
227 if (ctx->tmp_used) in build_epilogue()
228 emit(A64_POP(tmp1, tmp2, A64_SP), ctx); in build_epilogue()
229 emit(A64_POP(r8, r9, A64_SP), ctx); in build_epilogue()
230 emit(A64_POP(r6, r7, A64_SP), ctx); in build_epilogue()
233 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); in build_epilogue()
236 emit(A64_MOV(1, A64_R(0), r0), ctx); in build_epilogue()
238 emit(A64_RET(A64_LR), ctx); in build_epilogue()
247 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) in build_insn() argument
256 const int i = insn - ctx->prog->insnsi; in build_insn()
276 emit(A64_MOV(is64, dst, src), ctx); in build_insn()
281 emit(A64_ADD(is64, dst, dst, src), ctx); in build_insn()
285 emit(A64_SUB(is64, dst, dst, src), ctx); in build_insn()
289 emit(A64_AND(is64, dst, dst, src), ctx); in build_insn()
293 emit(A64_ORR(is64, dst, dst, src), ctx); in build_insn()
297 emit(A64_EOR(is64, dst, dst, src), ctx); in build_insn()
301 emit(A64_MUL(is64, dst, dst, src), ctx); in build_insn()
313 emit(A64_CBNZ(is64, src, jmp_offset), ctx); in build_insn()
314 emit(A64_MOVZ(1, r0, 0, 0), ctx); in build_insn()
315 jmp_offset = epilogue_offset(ctx); in build_insn()
317 emit(A64_B(jmp_offset), ctx); in build_insn()
321 emit(A64_UDIV(is64, dst, dst, src), ctx); in build_insn()
324 ctx->tmp_used = 1; in build_insn()
325 emit(A64_UDIV(is64, tmp, dst, src), ctx); in build_insn()
326 emit(A64_MUL(is64, tmp, tmp, src), ctx); in build_insn()
327 emit(A64_SUB(is64, dst, dst, tmp), ctx); in build_insn()
334 emit(A64_LSLV(is64, dst, dst, src), ctx); in build_insn()
338 emit(A64_LSRV(is64, dst, dst, src), ctx); in build_insn()
342 emit(A64_ASRV(is64, dst, dst, src), ctx); in build_insn()
347 emit(A64_NEG(is64, dst, dst), ctx); in build_insn()
361 emit(A64_REV16(is64, dst, dst), ctx); in build_insn()
363 emit(A64_UXTH(is64, dst, dst), ctx); in build_insn()
366 emit(A64_REV32(is64, dst, dst), ctx); in build_insn()
370 emit(A64_REV64(dst, dst), ctx); in build_insn()
378 emit(A64_UXTH(is64, dst, dst), ctx); in build_insn()
382 emit(A64_UXTW(is64, dst, dst), ctx); in build_insn()
392 emit_a64_mov_i(is64, dst, imm, ctx); in build_insn()
397 ctx->tmp_used = 1; in build_insn()
398 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
399 emit(A64_ADD(is64, dst, dst, tmp), ctx); in build_insn()
403 ctx->tmp_used = 1; in build_insn()
404 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
405 emit(A64_SUB(is64, dst, dst, tmp), ctx); in build_insn()
409 ctx->tmp_used = 1; in build_insn()
410 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
411 emit(A64_AND(is64, dst, dst, tmp), ctx); in build_insn()
415 ctx->tmp_used = 1; in build_insn()
416 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
417 emit(A64_ORR(is64, dst, dst, tmp), ctx); in build_insn()
421 ctx->tmp_used = 1; in build_insn()
422 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
423 emit(A64_EOR(is64, dst, dst, tmp), ctx); in build_insn()
427 ctx->tmp_used = 1; in build_insn()
428 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
429 emit(A64_MUL(is64, dst, dst, tmp), ctx); in build_insn()
433 ctx->tmp_used = 1; in build_insn()
434 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
435 emit(A64_UDIV(is64, dst, dst, tmp), ctx); in build_insn()
439 ctx->tmp_used = 1; in build_insn()
440 emit_a64_mov_i(is64, tmp2, imm, ctx); in build_insn()
441 emit(A64_UDIV(is64, tmp, dst, tmp2), ctx); in build_insn()
442 emit(A64_MUL(is64, tmp, tmp, tmp2), ctx); in build_insn()
443 emit(A64_SUB(is64, dst, dst, tmp), ctx); in build_insn()
447 emit(A64_LSL(is64, dst, dst, imm), ctx); in build_insn()
451 emit(A64_LSR(is64, dst, dst, imm), ctx); in build_insn()
455 emit(A64_ASR(is64, dst, dst, imm), ctx); in build_insn()
460 jmp_offset = bpf2a64_offset(i + off, i, ctx); in build_insn()
462 emit(A64_B(jmp_offset), ctx); in build_insn()
471 emit(A64_CMP(1, dst, src), ctx); in build_insn()
473 jmp_offset = bpf2a64_offset(i + off, i, ctx); in build_insn()
497 emit(A64_B_(jmp_cond, jmp_offset), ctx); in build_insn()
500 emit(A64_TST(1, dst, src), ctx); in build_insn()
509 ctx->tmp_used = 1; in build_insn()
510 emit_a64_mov_i(1, tmp, imm, ctx); in build_insn()
511 emit(A64_CMP(1, dst, tmp), ctx); in build_insn()
514 ctx->tmp_used = 1; in build_insn()
515 emit_a64_mov_i(1, tmp, imm, ctx); in build_insn()
516 emit(A64_TST(1, dst, tmp), ctx); in build_insn()
524 ctx->tmp_used = 1; in build_insn()
525 emit_a64_mov_i64(tmp, func, ctx); in build_insn()
526 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
527 emit(A64_MOV(1, A64_FP, A64_SP), ctx); in build_insn()
528 emit(A64_BLR(tmp), ctx); in build_insn()
529 emit(A64_MOV(1, r0, A64_R(0)), ctx); in build_insn()
530 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
537 if (i == ctx->prog->len - 1) in build_insn()
539 jmp_offset = epilogue_offset(ctx); in build_insn()
541 emit(A64_B(jmp_offset), ctx); in build_insn()
560 emit_a64_mov_i64(dst, imm64, ctx); in build_insn()
570 ctx->tmp_used = 1; in build_insn()
571 emit_a64_mov_i(1, tmp, off, ctx); in build_insn()
574 emit(A64_LDR32(dst, src, tmp), ctx); in build_insn()
577 emit(A64_LDRH(dst, src, tmp), ctx); in build_insn()
580 emit(A64_LDRB(dst, src, tmp), ctx); in build_insn()
583 emit(A64_LDR64(dst, src, tmp), ctx); in build_insn()
594 ctx->tmp_used = 1; in build_insn()
595 emit_a64_mov_i(1, tmp2, off, ctx); in build_insn()
596 emit_a64_mov_i(1, tmp, imm, ctx); in build_insn()
599 emit(A64_STR32(tmp, dst, tmp2), ctx); in build_insn()
602 emit(A64_STRH(tmp, dst, tmp2), ctx); in build_insn()
605 emit(A64_STRB(tmp, dst, tmp2), ctx); in build_insn()
608 emit(A64_STR64(tmp, dst, tmp2), ctx); in build_insn()
618 ctx->tmp_used = 1; in build_insn()
619 emit_a64_mov_i(1, tmp, off, ctx); in build_insn()
622 emit(A64_STR32(src, dst, tmp), ctx); in build_insn()
625 emit(A64_STRH(src, dst, tmp), ctx); in build_insn()
628 emit(A64_STRB(src, dst, tmp), ctx); in build_insn()
631 emit(A64_STR64(src, dst, tmp), ctx); in build_insn()
660 emit(A64_MOV(1, r1, r6), ctx); in build_insn()
661 emit_a64_mov_i(0, r2, imm, ctx); in build_insn()
663 emit(A64_ADD(0, r2, r2, src), ctx); in build_insn()
677 emit_a64_mov_i64(r3, size, ctx); in build_insn()
678 emit(A64_SUB_I(1, r4, fp, STACK_SIZE), ctx); in build_insn()
679 emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx); in build_insn()
680 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
681 emit(A64_MOV(1, A64_FP, A64_SP), ctx); in build_insn()
682 emit(A64_BLR(r5), ctx); in build_insn()
683 emit(A64_MOV(1, r0, A64_R(0)), ctx); in build_insn()
684 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
686 jmp_offset = epilogue_offset(ctx); in build_insn()
688 emit(A64_CBZ(1, r0, jmp_offset), ctx); in build_insn()
689 emit(A64_MOV(1, r5, r0), ctx); in build_insn()
692 emit(A64_LDR32(r0, r5, A64_ZR), ctx); in build_insn()
694 emit(A64_REV32(0, r0, r0), ctx); in build_insn()
698 emit(A64_LDRH(r0, r5, A64_ZR), ctx); in build_insn()
700 emit(A64_REV16(0, r0, r0), ctx); in build_insn()
704 emit(A64_LDRB(r0, r5, A64_ZR), ctx); in build_insn()
721 static int build_body(struct jit_ctx *ctx) in build_body() argument
723 const struct bpf_prog *prog = ctx->prog; in build_body()
730 ret = build_insn(insn, ctx); in build_body()
732 if (ctx->image == NULL) in build_body()
733 ctx->offset[i] = ctx->idx; in build_body()
759 struct jit_ctx ctx; in bpf_int_jit_compile() local
769 memset(&ctx, 0, sizeof(ctx)); in bpf_int_jit_compile()
770 ctx.prog = prog; in bpf_int_jit_compile()
772 ctx.offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); in bpf_int_jit_compile()
773 if (ctx.offset == NULL) in bpf_int_jit_compile()
779 if (build_body(&ctx)) in bpf_int_jit_compile()
782 build_prologue(&ctx); in bpf_int_jit_compile()
784 ctx.epilogue_offset = ctx.idx; in bpf_int_jit_compile()
785 build_epilogue(&ctx); in bpf_int_jit_compile()
788 image_size = sizeof(u32) * ctx.idx; in bpf_int_jit_compile()
796 ctx.image = (u32 *)image_ptr; in bpf_int_jit_compile()
797 ctx.idx = 0; in bpf_int_jit_compile()
799 build_prologue(&ctx); in bpf_int_jit_compile()
801 if (build_body(&ctx)) { in bpf_int_jit_compile()
806 build_epilogue(&ctx); in bpf_int_jit_compile()
810 bpf_jit_dump(prog->len, image_size, 2, ctx.image); in bpf_int_jit_compile()
812 bpf_flush_icache(header, ctx.image + ctx.idx); in bpf_int_jit_compile()
815 prog->bpf_func = (void *)ctx.image; in bpf_int_jit_compile()
818 kfree(ctx.offset); in bpf_int_jit_compile()