Lines Matching refs:ctx
68 static inline void emit(const u32 insn, struct jit_ctx *ctx) in emit() argument
70 if (ctx->image != NULL) in emit()
71 ctx->image[ctx->idx] = cpu_to_le32(insn); in emit()
73 ctx->idx++; in emit()
77 struct jit_ctx *ctx) in emit_a64_mov_i64() argument
82 emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx); in emit_a64_mov_i64()
87 emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx); in emit_a64_mov_i64()
94 const s32 val, struct jit_ctx *ctx) in emit_a64_mov_i() argument
101 emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx); in emit_a64_mov_i()
103 emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx); in emit_a64_mov_i()
104 emit(A64_MOVK(is64, reg, lo, 0), ctx); in emit_a64_mov_i()
107 emit(A64_MOVZ(is64, reg, lo, 0), ctx); in emit_a64_mov_i()
109 emit(A64_MOVK(is64, reg, hi, 16), ctx); in emit_a64_mov_i()
114 const struct jit_ctx *ctx) in bpf2a64_offset() argument
116 int to = ctx->offset[bpf_to]; in bpf2a64_offset()
118 int from = ctx->offset[bpf_from] - 1; in bpf2a64_offset()
131 static inline int epilogue_offset(const struct jit_ctx *ctx) in epilogue_offset() argument
133 int to = ctx->epilogue_offset; in epilogue_offset()
134 int from = ctx->idx; in epilogue_offset()
142 static void build_prologue(struct jit_ctx *ctx) in build_prologue() argument
159 emit(A64_PUSH(r6, r7, A64_SP), ctx); in build_prologue()
160 emit(A64_PUSH(r8, r9, A64_SP), ctx); in build_prologue()
161 if (ctx->tmp_used) in build_prologue()
162 emit(A64_PUSH(tmp1, tmp2, A64_SP), ctx); in build_prologue()
165 emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx); in build_prologue()
168 emit(A64_MOV(1, fp, A64_SP), ctx); in build_prologue()
171 emit_a64_mov_i64(ra, 0, ctx); in build_prologue()
172 emit_a64_mov_i64(rx, 0, ctx); in build_prologue()
175 static void build_epilogue(struct jit_ctx *ctx) in build_epilogue() argument
191 emit(A64_ADD_I(1, A64_SP, A64_SP, stack_size), ctx); in build_epilogue()
194 if (ctx->tmp_used) in build_epilogue()
195 emit(A64_POP(tmp1, tmp2, A64_SP), ctx); in build_epilogue()
196 emit(A64_POP(r8, r9, A64_SP), ctx); in build_epilogue()
197 emit(A64_POP(r6, r7, A64_SP), ctx); in build_epilogue()
200 emit(A64_MOV(1, fp, A64_SP), ctx); in build_epilogue()
203 emit(A64_MOV(1, A64_R(0), r0), ctx); in build_epilogue()
205 emit(A64_RET(A64_LR), ctx); in build_epilogue()
214 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) in build_insn() argument
223 const int i = insn - ctx->prog->insnsi; in build_insn()
243 emit(A64_MOV(is64, dst, src), ctx); in build_insn()
248 emit(A64_ADD(is64, dst, dst, src), ctx); in build_insn()
252 emit(A64_SUB(is64, dst, dst, src), ctx); in build_insn()
256 emit(A64_AND(is64, dst, dst, src), ctx); in build_insn()
260 emit(A64_ORR(is64, dst, dst, src), ctx); in build_insn()
264 emit(A64_EOR(is64, dst, dst, src), ctx); in build_insn()
268 emit(A64_MUL(is64, dst, dst, src), ctx); in build_insn()
280 emit(A64_CBNZ(is64, src, jmp_offset), ctx); in build_insn()
281 emit(A64_MOVZ(1, r0, 0, 0), ctx); in build_insn()
282 jmp_offset = epilogue_offset(ctx); in build_insn()
284 emit(A64_B(jmp_offset), ctx); in build_insn()
288 emit(A64_UDIV(is64, dst, dst, src), ctx); in build_insn()
291 ctx->tmp_used = 1; in build_insn()
292 emit(A64_UDIV(is64, tmp, dst, src), ctx); in build_insn()
293 emit(A64_MUL(is64, tmp, tmp, src), ctx); in build_insn()
294 emit(A64_SUB(is64, dst, dst, tmp), ctx); in build_insn()
301 emit(A64_LSLV(is64, dst, dst, src), ctx); in build_insn()
305 emit(A64_LSRV(is64, dst, dst, src), ctx); in build_insn()
309 emit(A64_ASRV(is64, dst, dst, src), ctx); in build_insn()
314 emit(A64_NEG(is64, dst, dst), ctx); in build_insn()
328 emit(A64_REV16(is64, dst, dst), ctx); in build_insn()
330 emit(A64_UXTH(is64, dst, dst), ctx); in build_insn()
333 emit(A64_REV32(is64, dst, dst), ctx); in build_insn()
337 emit(A64_REV64(dst, dst), ctx); in build_insn()
345 emit(A64_UXTH(is64, dst, dst), ctx); in build_insn()
349 emit(A64_UXTW(is64, dst, dst), ctx); in build_insn()
359 emit_a64_mov_i(is64, dst, imm, ctx); in build_insn()
364 ctx->tmp_used = 1; in build_insn()
365 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
366 emit(A64_ADD(is64, dst, dst, tmp), ctx); in build_insn()
370 ctx->tmp_used = 1; in build_insn()
371 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
372 emit(A64_SUB(is64, dst, dst, tmp), ctx); in build_insn()
376 ctx->tmp_used = 1; in build_insn()
377 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
378 emit(A64_AND(is64, dst, dst, tmp), ctx); in build_insn()
382 ctx->tmp_used = 1; in build_insn()
383 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
384 emit(A64_ORR(is64, dst, dst, tmp), ctx); in build_insn()
388 ctx->tmp_used = 1; in build_insn()
389 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
390 emit(A64_EOR(is64, dst, dst, tmp), ctx); in build_insn()
394 ctx->tmp_used = 1; in build_insn()
395 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
396 emit(A64_MUL(is64, dst, dst, tmp), ctx); in build_insn()
400 ctx->tmp_used = 1; in build_insn()
401 emit_a64_mov_i(is64, tmp, imm, ctx); in build_insn()
402 emit(A64_UDIV(is64, dst, dst, tmp), ctx); in build_insn()
406 ctx->tmp_used = 1; in build_insn()
407 emit_a64_mov_i(is64, tmp2, imm, ctx); in build_insn()
408 emit(A64_UDIV(is64, tmp, dst, tmp2), ctx); in build_insn()
409 emit(A64_MUL(is64, tmp, tmp, tmp2), ctx); in build_insn()
410 emit(A64_SUB(is64, dst, dst, tmp), ctx); in build_insn()
414 emit(A64_LSL(is64, dst, dst, imm), ctx); in build_insn()
418 emit(A64_LSR(is64, dst, dst, imm), ctx); in build_insn()
422 emit(A64_ASR(is64, dst, dst, imm), ctx); in build_insn()
427 jmp_offset = bpf2a64_offset(i + off, i, ctx); in build_insn()
429 emit(A64_B(jmp_offset), ctx); in build_insn()
438 emit(A64_CMP(1, dst, src), ctx); in build_insn()
440 jmp_offset = bpf2a64_offset(i + off, i, ctx); in build_insn()
464 emit(A64_B_(jmp_cond, jmp_offset), ctx); in build_insn()
467 emit(A64_TST(1, dst, src), ctx); in build_insn()
476 ctx->tmp_used = 1; in build_insn()
477 emit_a64_mov_i(1, tmp, imm, ctx); in build_insn()
478 emit(A64_CMP(1, dst, tmp), ctx); in build_insn()
481 ctx->tmp_used = 1; in build_insn()
482 emit_a64_mov_i(1, tmp, imm, ctx); in build_insn()
483 emit(A64_TST(1, dst, tmp), ctx); in build_insn()
491 ctx->tmp_used = 1; in build_insn()
492 emit_a64_mov_i64(tmp, func, ctx); in build_insn()
493 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
494 emit(A64_MOV(1, A64_FP, A64_SP), ctx); in build_insn()
495 emit(A64_BLR(tmp), ctx); in build_insn()
496 emit(A64_MOV(1, r0, A64_R(0)), ctx); in build_insn()
497 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
504 if (i == ctx->prog->len - 1) in build_insn()
506 jmp_offset = epilogue_offset(ctx); in build_insn()
508 emit(A64_B(jmp_offset), ctx); in build_insn()
527 emit_a64_mov_i64(dst, imm64, ctx); in build_insn()
537 ctx->tmp_used = 1; in build_insn()
538 emit_a64_mov_i(1, tmp, off, ctx); in build_insn()
541 emit(A64_LDR32(dst, src, tmp), ctx); in build_insn()
544 emit(A64_LDRH(dst, src, tmp), ctx); in build_insn()
547 emit(A64_LDRB(dst, src, tmp), ctx); in build_insn()
550 emit(A64_LDR64(dst, src, tmp), ctx); in build_insn()
567 ctx->tmp_used = 1; in build_insn()
568 emit_a64_mov_i(1, tmp, off, ctx); in build_insn()
571 emit(A64_STR32(src, dst, tmp), ctx); in build_insn()
574 emit(A64_STRH(src, dst, tmp), ctx); in build_insn()
577 emit(A64_STRB(src, dst, tmp), ctx); in build_insn()
580 emit(A64_STR64(src, dst, tmp), ctx); in build_insn()
609 emit(A64_MOV(1, r1, r6), ctx); in build_insn()
610 emit_a64_mov_i(0, r2, imm, ctx); in build_insn()
612 emit(A64_ADD(0, r2, r2, src), ctx); in build_insn()
626 emit_a64_mov_i64(r3, size, ctx); in build_insn()
627 emit(A64_ADD_I(1, r4, fp, MAX_BPF_STACK), ctx); in build_insn()
628 emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx); in build_insn()
629 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
630 emit(A64_MOV(1, A64_FP, A64_SP), ctx); in build_insn()
631 emit(A64_BLR(r5), ctx); in build_insn()
632 emit(A64_MOV(1, r0, A64_R(0)), ctx); in build_insn()
633 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); in build_insn()
635 jmp_offset = epilogue_offset(ctx); in build_insn()
637 emit(A64_CBZ(1, r0, jmp_offset), ctx); in build_insn()
638 emit(A64_MOV(1, r5, r0), ctx); in build_insn()
641 emit(A64_LDR32(r0, r5, A64_ZR), ctx); in build_insn()
643 emit(A64_REV32(0, r0, r0), ctx); in build_insn()
647 emit(A64_LDRH(r0, r5, A64_ZR), ctx); in build_insn()
649 emit(A64_REV16(0, r0, r0), ctx); in build_insn()
653 emit(A64_LDRB(r0, r5, A64_ZR), ctx); in build_insn()
670 static int build_body(struct jit_ctx *ctx) in build_body() argument
672 const struct bpf_prog *prog = ctx->prog; in build_body()
679 ret = build_insn(insn, ctx); in build_body()
681 if (ctx->image == NULL) in build_body()
682 ctx->offset[i] = ctx->idx; in build_body()
708 struct jit_ctx ctx; in bpf_int_jit_compile() local
718 memset(&ctx, 0, sizeof(ctx)); in bpf_int_jit_compile()
719 ctx.prog = prog; in bpf_int_jit_compile()
721 ctx.offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); in bpf_int_jit_compile()
722 if (ctx.offset == NULL) in bpf_int_jit_compile()
728 if (build_body(&ctx)) in bpf_int_jit_compile()
731 build_prologue(&ctx); in bpf_int_jit_compile()
733 ctx.epilogue_offset = ctx.idx; in bpf_int_jit_compile()
734 build_epilogue(&ctx); in bpf_int_jit_compile()
737 image_size = sizeof(u32) * ctx.idx; in bpf_int_jit_compile()
745 ctx.image = (u32 *)image_ptr; in bpf_int_jit_compile()
746 ctx.idx = 0; in bpf_int_jit_compile()
748 build_prologue(&ctx); in bpf_int_jit_compile()
750 if (build_body(&ctx)) { in bpf_int_jit_compile()
755 build_epilogue(&ctx); in bpf_int_jit_compile()
759 bpf_jit_dump(prog->len, image_size, 2, ctx.image); in bpf_int_jit_compile()
761 bpf_flush_icache(ctx.image, ctx.image + ctx.idx); in bpf_int_jit_compile()
764 prog->bpf_func = (void *)ctx.image; in bpf_int_jit_compile()
767 kfree(ctx.offset); in bpf_int_jit_compile()